dfindexeddb 20241031__py3-none-any.whl → 20251109__py3-none-any.whl

dfindexeddb/leveldb/cli.py

@@ -14,29 +14,28 @@
 # limitations under the License.
 """A CLI tool for leveldb files."""
 import argparse
+import csv
 import dataclasses
-from datetime import datetime
 import json
 import pathlib
+import sys
+from datetime import datetime
+from typing import Union
 
-from dfindexeddb import utils
-from dfindexeddb import version
-from dfindexeddb.leveldb import descriptor
-from dfindexeddb.leveldb import ldb
-from dfindexeddb.leveldb import log
-from dfindexeddb.leveldb import record
+from dfindexeddb import utils, version
+from dfindexeddb.leveldb import descriptor, ldb, log, record
 from dfindexeddb.leveldb.plugins import manager
 
-
 _VALID_PRINTABLE_CHARACTERS = (
-    ' abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' +
-    '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~.')
+    " abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+    + "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~."
+)
 
 
 class Encoder(json.JSONEncoder):
   """A JSON encoder class for dfleveldb fields."""
 
-  def default(self, o):
+  def default(self, o):  # type: ignore[no-untyped-def]
     """Returns a serializable object for o."""
     if dataclasses.is_dataclass(o):
       o_dict = utils.asdict(o)
@@ -45,10 +44,10 @@ class Encoder(json.JSONEncoder):
       out = []
       for x in o:
         if chr(x) not in _VALID_PRINTABLE_CHARACTERS:
-          out.append(f'\\x{x:02X}')
+          out.append(f"\\x{x:02X}")
         else:
           out.append(chr(x))
-      return ''.join(out)
+      return "".join(out)
     if isinstance(o, datetime):
       return o.isoformat()
     if isinstance(o, set):
@@ -56,19 +55,34 @@ class Encoder(json.JSONEncoder):
     return json.JSONEncoder.default(self, o)
 
 
-def _Output(structure, output):
+def _Output(
+    structure: Union[
+        record.LevelDBRecord,
+        ldb.Block,
+        ldb.KeyValueRecord,
+        log.Block,
+        log.PhysicalRecord,
+        log.WriteBatch,
+        log.ParsedInternalKey,
+        descriptor.VersionEdit,
+        descriptor.LevelDBVersion,
+    ],
+    output: str,
+) -> None:
   """Helper method to output parsed structure to stdout."""
-  if output == 'json':
+  if output == "csv":
+    csv.writer(sys.stdout).writerow(utils.asdict(structure).values())
+  elif output == "json":
     print(json.dumps(structure, indent=2, cls=Encoder))
-  elif output == 'jsonl':
+  elif output == "jsonl":
     print(json.dumps(structure, cls=Encoder))
-  elif output == 'repr':
+  elif output == "repr":
     print(structure)
 
 
-def DbCommand(args):
+def DbCommand(args: argparse.Namespace) -> None:
   """The CLI for processing leveldb folders."""
-  if args.plugin and args.plugin == 'list':
+  if args.plugin and args.plugin == "list":
     for plugin, _ in manager.LeveldbPluginManager.GetPlugins():
       print(plugin)
     return
@@ -78,10 +92,10 @@ def DbCommand(args):
   else:
     plugin_class = None
 
-  for leveldb_record in record.FolderReader(
-      args.source).GetRecords(
-          use_manifest=args.use_manifest,
-          use_sequence_number=args.use_sequence_number):
+  for leveldb_record in record.FolderReader(args.source).GetRecords(
+      use_manifest=args.use_manifest,
+      use_sequence_number=args.use_sequence_number,
+  ):
     if plugin_class:
       plugin_record = plugin_class.FromLevelDBRecord(leveldb_record)
       _Output(plugin_record, output=args.output)
@@ -89,9 +103,9 @@ def DbCommand(args):
       _Output(leveldb_record, output=args.output)
 
 
-def LdbCommand(args):
+def LdbCommand(args: argparse.Namespace) -> None:
   """The CLI for processing ldb files."""
-  if args.plugin and args.plugin == 'list':
+  if args.plugin and args.plugin == "list":
     for plugin, _ in manager.LeveldbPluginManager.GetPlugins():
       print(plugin)
     return
@@ -103,12 +117,12 @@ def LdbCommand(args):
 
   ldb_file = ldb.FileReader(args.source)
 
-  if args.structure_type == 'blocks':
+  if args.structure_type == "blocks":
     # Prints block information.
     for block in ldb_file.GetBlocks():
       _Output(block, output=args.output)
 
-  elif args.structure_type == 'records' or not args.structure_type:
+  elif args.structure_type == "records" or not args.structure_type:
     # Prints key value record information.
     for key_value_record in ldb_file.GetKeyValueRecords():
       if plugin_class:
@@ -118,12 +132,12 @@ def LdbCommand(args):
         _Output(key_value_record, output=args.output)
 
   else:
-    print(f'{args.structure_type} is not supported for ldb files.')
+    print(f"{args.structure_type} is not supported for ldb files.")
 
 
-def LogCommand(args):
+def LogCommand(args: argparse.Namespace) -> None:
   """The CLI for processing log files."""
-  if args.plugin and args.plugin == 'list':
+  if args.plugin and args.plugin == "list":
     for plugin, _ in manager.LeveldbPluginManager.GetPlugins():
       print(plugin)
     return
@@ -135,23 +149,25 @@ def LogCommand(args):
 
   log_file = log.FileReader(args.source)
 
-  if args.structure_type == 'blocks':
+  if args.structure_type == "blocks":
     # Prints block information.
     for block in log_file.GetBlocks():
       _Output(block, output=args.output)
 
-  elif args.structure_type == 'physical_records':
+  elif args.structure_type == "physical_records":
     # Prints log file physical record information.
     for log_file_record in log_file.GetPhysicalRecords():
       _Output(log_file_record, output=args.output)
 
-  elif args.structure_type == 'write_batches':
+  elif args.structure_type == "write_batches":
     # Prints log file batch information.
     for batch in log_file.GetWriteBatches():
       _Output(batch, output=args.output)
 
-  elif (args.structure_type in ('parsed_internal_key', 'records')
-        or not args.structure_type):
+  elif (
+      args.structure_type in ("parsed_internal_key", "records")
+      or not args.structure_type
+  ):
     # Prints key value record information.
     for internal_key_record in log_file.GetParsedInternalKeys():
       if plugin_class:
@@ -161,10 +177,10 @@ def LogCommand(args):
         _Output(internal_key_record, output=args.output)
 
   else:
-    print(f'{args.structure_type} is not supported for log files.')
+    print(f"{args.structure_type} is not supported for log files.")
 
 
-def DescriptorCommand(args):
+def DescriptorCommand(args: argparse.Namespace) -> None:
   """The CLI for processing descriptor (MANIFEST) files."""
   manifest_file = descriptor.FileReader(args.source)
 
@@ -172,157 +188,156 @@ def DescriptorCommand(args):
     for levels in manifest_file.GetVersions():
       _Output(levels, output=args.output)
 
-  elif args.structure_type == 'blocks':
+  elif args.structure_type == "blocks":
     # Prints block information.
     for block in manifest_file.GetBlocks():
       _Output(block, output=args.output)
 
-  elif args.structure_type == 'physical_records':
+  elif args.structure_type == "physical_records":
     # Prints log file physical record information.
     for log_file_record in manifest_file.GetPhysicalRecords():
       _Output(log_file_record, output=args.output)
 
-  elif (args.structure_type == 'versionedit'
-        or not args.structure_type):
+  elif args.structure_type == "versionedit" or not args.structure_type:
     for version_edit in manifest_file.GetVersionEdits():
       _Output(version_edit, output=args.output)
 
   else:
-    print(f'{args.structure_type} is not supported for descriptor files.')
+    print(f"{args.structure_type} is not supported for descriptor files.")
 
 
-def App():
+def App() -> None:
   """The CLI app entrypoint for parsing leveldb files."""
   parser = argparse.ArgumentParser(
-      prog='dfleveldb',
-      description='A cli tool for parsing leveldb files',
-      epilog=f'Version {version.GetVersion()}')
+      prog="dfleveldb",
+      description="A cli tool for parsing leveldb files",
+      epilog=f"Version {version.GetVersion()}",
+  )
 
   subparsers = parser.add_subparsers()
 
-  parser_db = subparsers.add_parser(
-      'db', help='Parse a directory as leveldb.')
+  parser_db = subparsers.add_parser("db", help="Parse a directory as leveldb.")
   parser_db.add_argument(
-      '-s', '--source',
+      "-s",
+      "--source",
       required=True,
       type=pathlib.Path,
-      help='The source leveldb directory')
+      help="The source leveldb directory.",
+  )
   recover_group = parser_db.add_mutually_exclusive_group()
   recover_group.add_argument(
-      '--use_manifest',
-      action='store_true',
-      help='Use manifest file to determine active/deleted records.')
+      "--use_manifest",
+      action="store_true",
+      help="Use manifest file to determine active/deleted records.",
+  )
   recover_group.add_argument(
-      '--use_sequence_number',
-      action='store_true',
+      "--use_sequence_number",
+      action="store_true",
       help=(
-          'Use sequence number and file offset to determine active/deleted '
-          'records.'))
+          "Use sequence number and file offset to determine active/deleted "
+          "records."
+      ),
+  )
   parser_db.add_argument(
-      '-o',
-      '--output',
-      choices=[
-          'json',
-          'jsonl',
-          'repr'],
-      default='json',
-      help='Output format.  Default is json')
-  parser_db.add_argument(
-      '--plugin',
-      help='Use plugin to parse records.')
+      "-o",
+      "--output",
+      choices=["csv", "json", "jsonl", "repr"],
+      default="json",
+      help="Output format.  Default is json.",
+  )
+  parser_db.add_argument("--plugin", help="Use plugin to parse records.")
   parser_db.set_defaults(func=DbCommand)
 
-  parser_log = subparsers.add_parser(
-      'log', help='Parse a leveldb log file.')
+  parser_log = subparsers.add_parser("log", help="Parse a leveldb log file.")
   parser_log.add_argument(
-      '-s', '--source',
+      "-s",
+      "--source",
       required=True,
       type=pathlib.Path,
-      help='The source leveldb file')
-  parser_log.add_argument(
-      '-o',
-      '--output',
-      choices=[
-          'json',
-          'jsonl',
-          'repr'],
-      default='json',
-      help='Output format.  Default is json')
+      help="The source leveldb file.",
+  )
   parser_log.add_argument(
-      '--plugin',
-      help='Use plugin to parse records.')
+      "-o",
+      "--output",
+      choices=["csv", "json", "jsonl", "repr"],
+      default="json",
+      help="Output format.  Default is json.",
+  )
+  parser_log.add_argument("--plugin", help="Use plugin to parse records.")
   parser_log.add_argument(
-      '-t',
-      '--structure_type',
+      "-t",
+      "--structure_type",
       choices=[
-          'blocks',
-          'physical_records',
-          'write_batches',
-          'parsed_internal_key'],
-      help='Parses the specified structure.  Default is parsed_internal_key.')
+          "blocks",
+          "physical_records",
+          "write_batches",
+          "parsed_internal_key",
+      ],
+      help="Parses the specified structure.  Default is parsed_internal_key.",
+  )
   parser_log.set_defaults(func=LogCommand)
 
   parser_ldb = subparsers.add_parser(
-      'ldb', help='Parse a leveldb table (.ldb) file.')
+      "ldb", help="Parse a leveldb table (.ldb) file."
+  )
   parser_ldb.add_argument(
-      '-s', '--source',
+      "-s",
+      "--source",
       required=True,
       type=pathlib.Path,
-      help='The source leveldb file')
-  parser_ldb.add_argument(
-      '-o',
-      '--output',
-      choices=[
-          'json',
-          'jsonl',
-          'repr'],
-      default='json',
-      help='Output format.  Default is json')
+      help="The source leveldb file",
+  )
   parser_ldb.add_argument(
-      '--plugin',
-      help='Use plugin to parse records.')
+      "-o",
+      "--output",
+      choices=["csv", "json", "jsonl", "repr"],
+      default="json",
+      help="Output format.  Default is json.",
+  )
+  parser_ldb.add_argument("--plugin", help="Use plugin to parse records.")
   parser_ldb.add_argument(
-      '-t',
-      '--structure_type',
-      choices=[
-          'blocks',
-          'records'],
-      help='Parses the specified structure.  Default is records.')
+      "-t",
+      "--structure_type",
+      choices=["blocks", "records"],
+      help="Parses the specified structure.  Default is records.",
+  )
   parser_ldb.set_defaults(func=LdbCommand)
 
   parser_descriptor = subparsers.add_parser(
-      'descriptor', help='Parse a leveldb descriptor (MANIFEST) file.')
+      "descriptor", help="Parse a leveldb descriptor (MANIFEST) file."
+  )
   parser_descriptor.add_argument(
-      '-s', '--source',
+      "-s",
+      "--source",
       required=True,
       type=pathlib.Path,
-      help='The source leveldb file')
+      help="The source leveldb file",
+  )
   parser_descriptor.add_argument(
-      '-o',
-      '--output',
-      choices=[
-          'json',
-          'jsonl',
-          'repr'],
-      default='json',
-      help='Output format.  Default is json')
+      "-o",
+      "--output",
+      choices=["csv", "json", "jsonl", "repr"],
+      default="json",
+      help="Output format.  Default is json.",
+  )
   db_group = parser_descriptor.add_mutually_exclusive_group()
   db_group.add_argument(
-      '-t',
-      '--structure_type',
-      choices=[
-          'blocks', 'physical_records', 'versionedit'],
-      help='Parses the specified structure.  Default is versionedit.')
+      "-t",
+      "--structure_type",
+      choices=["blocks", "physical_records", "versionedit"],
+      help="Parses the specified structure.  Default is versionedit.",
+  )
   db_group.add_argument(
-      '-v',
-      '--version_history',
-      action='store_true',
-      help='Parses the leveldb version history.')
+      "-v",
+      "--version_history",
+      action="store_true",
+      help="Parses the leveldb version history.",
+  )
   parser_descriptor.set_defaults(func=DescriptorCommand)
 
   args = parser.parse_args()
 
-  if not hasattr(args, 'func'):
+  if not hasattr(args, "func"):
     parser.print_usage()
   else:
     args.func(args)

dfindexeddb/leveldb/definitions.py

@@ -19,23 +19,25 @@ import enum
 BLOCK_RESTART_ENTRY_LENGTH = 4
 BLOCK_TRAILER_SIZE = 5
 TABLE_FOOTER_SIZE = 48
-TABLE_MAGIC = b'\x57\xfb\x80\x8b\x24\x75\x47\xdb'
+TABLE_MAGIC = b"\x57\xfb\x80\x8b\x24\x75\x47\xdb"
 
 PACKED_SEQUENCE_AND_TYPE_LENGTH = 8
 SEQUENCE_LENGTH = 7
 TYPE_LENGTH = 1
 
-MANIFEST_FILENAME_PATTERN = r'MANIFEST-[0-9]{6}'
+MANIFEST_FILENAME_PATTERN = r"MANIFEST-[0-9]{6}"
 
 
 class BlockCompressionType(enum.IntEnum):
   """Block compression types."""
+
   SNAPPY = 1
   ZSTD = 2
 
 
 class VersionEditTags(enum.IntEnum):
   """VersionEdit tags."""
+
   COMPARATOR = 1
   LOG_NUMBER = 2
   NEXT_FILE_NUMBER = 3
@@ -49,6 +51,7 @@ class VersionEditTags(enum.IntEnum):
 
 class LogFilePhysicalRecordType(enum.IntEnum):
   """Log file physical record types."""
+
   FULL = 1
   FIRST = 2
   MIDDLE = 3
@@ -57,5 +60,6 @@ class LogFilePhysicalRecordType(enum.IntEnum):
 
 class InternalRecordType(enum.IntEnum):
   """Internal record types."""
+
   DELETED = 0
   VALUE = 1