dfindexeddb 20240417__py3-none-any.whl → 20240501__py3-none-any.whl

dfindexeddb/leveldb/record.py

@@ -18,7 +18,7 @@ import dataclasses
 import pathlib
 import re
 import sys
-from typing import Any, Generator, Optional, Union
+from typing import Generator, Optional, Union
 
 from dfindexeddb import errors
 from dfindexeddb.leveldb import definitions
@@ -37,7 +37,8 @@ class LevelDBRecord:
   Attributes:
     path: the file path where the record was parsed from.
     record: the leveldb record.
-    level: the leveldb level, None indicates the record came from a log file.
+    level: the leveldb level, None indicates the record came from a log file or
+        a file not part of the active file set (determined by a MANIFEST file).
     recovered: True if the record is a recovered record.
   """
   path: str
@@ -51,7 +52,7 @@ class LevelDBRecord:
   def FromFile(
       cls,
       file_path: pathlib.Path
-  ) -> Generator[LevelDBRecord, Any, Any]:
+  ) -> Generator[LevelDBRecord, None, None]:
     """Yields leveldb records from the given path.
 
     Yields:
@@ -74,45 +75,47 @@ class LevelDBRecord:
     else:
       print(f'Unsupported file type {file_path.as_posix()}', file=sys.stderr)
 
-  @classmethod
-  def FromDir(
-      cls,
-      path: pathlib.Path
-  ) -> Generator[LevelDBRecord, Any, Any]:
-    """Yields LevelDBRecords from the given directory.
+
+class FolderReader:
+  """A LevelDB folder reader.
+
+  Attributes:
+    foldername (str): the source LevelDB folder.
+  """
+
+  def __init__(self, foldername: pathlib.Path):
+    """Initializes the FolderReader.
 
     Args:
-      path: the file path.
+      foldername: the source LevelDB folder.
 
-    Yields:
-      LevelDBRecords
+    Raises:
+      ValueError: if foldername is None or not a directory.
     """
-    if not path or not path.is_dir():
-      raise ValueError(f'{path} is not a directory')
-    for file_path in path.iterdir():
-      yield from cls.FromFile(file_path=file_path)
+    if not foldername or not foldername.is_dir():
+      raise ValueError(f'{foldername} is None or not a directory')
+    self.foldername = foldername
 
-  @classmethod
-  def FromManifest(
-      cls,
-      path: pathlib.Path
-  ) -> Generator[LevelDBRecord, Any, Any]:
-    """Yields LevelDBRecords from the given directory using the manifest.
+  def LogFiles(self) -> Generator[pathlib.Path, None, None]:
+    """Returns the log filenames."""
+    yield from self.foldername.glob('*.log')
 
-    Args:
-      path: the file path.
+  def LdbFiles(self) -> Generator[pathlib.Path, None, None]:
+    """Returns the ldb filenames."""
+    yield from self.foldername.glob('*.ldb')
 
-    Yields:
-      LevelDBRecords
+  def Manifest(self) -> Generator[pathlib.Path, None, None]:
+    """Returns the Manifest filenames."""
+    yield from self.foldername.glob('MANIFEST-*')
+
+  def GetCurrentManifestPath(self) -> pathlib.Path:
+    """Returns the path of the current manifest file.
 
     Raises:
-      ParserError: if the CURRENT or MANIFEST-* file does not exist.
-      ValueError: if path is not a directory.
+      ParserError: when the CURRENT file does not exist/contain the expected
+          content or when the expected MANIFEST file does not exist.
     """
-    if not path or not path.is_dir():
-      raise ValueError(f'{path} is not a directory')
-
-    current_path = path / 'CURRENT'
+    current_path = self.foldername / 'CURRENT'
     if not current_path.exists():
       raise errors.ParserError(f'{current_path!s} does not exist.')
 
@@ -122,34 +125,113 @@ class LevelDBRecord:
       raise errors.ParserError(
           f'{current_path!s} does not contain the expected content')
 
-    manifest_path = path / current_manifest
+    manifest_path = self.foldername / current_manifest
     if not manifest_path.exists():
       raise errors.ParserError(f'{manifest_path!s} does not exist.')
+    return manifest_path
 
+  def GetLatestVersion(self) -> descriptor.LevelDBVersion:
+    """Returns the latest LevelDBVersion.
+
+    Raises:
+      ParserError: when the leveldb version could not be parsed.
+    """
+    current_manifest_path = self.GetCurrentManifestPath()
     latest_version = descriptor.FileReader(
-        str(manifest_path)).GetLatestVersion()
+        str(current_manifest_path)).GetLatestVersion()
     if not latest_version:
       raise errors.ParserError(
-          f'Could not parse a leveldb version from {manifest_path!s}')
+          f'Could not parse a leveldb version from {current_manifest_path!s}')
+    return latest_version
+
+  def _GetRecordsByFile(
+      self, filename: pathlib.Path) -> Generator[LevelDBRecord, None, None]:
+    """Yields the LevelDBRecords from a file.
+
+    Non-log/ldb files are ignored.
+
+    Args:
+      filename: the source LevelDB file.
+
+    Yields:
+      LevelDBRecords
+    """
+    if filename.name.endswith('.log'):
+      yield from self._GetLogRecords(filename)
+    elif filename.name.endswith('.ldb'):
+      yield from self._GetLdbRecords(filename)
+    elif filename.name.startswith('MANIFEST'):
+      print(f'Ignoring descriptor file {filename.as_posix()}', file=sys.stderr)
+    elif filename.name in ('LOCK', 'CURRENT', 'LOG', 'LOG.old'):
+      print(f'Ignoring {filename.as_posix()}', file=sys.stderr)
+    else:
+      print(f'Unsupported file type {filename.as_posix()}', file=sys.stderr)
+
+  def _GetLogRecords(
+      self,
+      filename: pathlib.Path
+  ) -> Generator[LevelDBRecord, None, None]:
+    """Yields the LevelDBRecords from a log file.
 
-    # read log records
+    Args:
+      filename: the source LevelDB file.
+
+    Yields:
+      LevelDBRecords
+    """
+    for record in log.FileReader(filename.as_posix()).GetParsedInternalKeys():
+      yield LevelDBRecord(path=filename.as_posix(), record=record)
+
+  def _GetLdbRecords(
+      self,
+      filename: pathlib.Path
+  ) -> Generator[LevelDBRecord, None, None]:
+    """Yields the LevelDBRecords from a log file.
+
+    Args:
+      filename: the source LevelDB file.
+
+    Yields:
+      LevelDBRecords
+    """
+    for record in ldb.FileReader(filename.as_posix()).GetKeyValueRecords():
+      yield LevelDBRecord(path=filename.as_posix(), record=record)
+
+  def _RecordsByManifest(self) -> Generator[LevelDBRecord, None, None]:
+    """Yields LevelDBRecords using active files determined by the MANIFEST file.
+
+    Yields:
+      LevelDBRecords.
+    """
+    latest_version = self.GetLatestVersion()
+
+    processed_files = set()
+
+    # read and cache the log records
     log_records = []
     if latest_version.current_log:
-      current_log = path / latest_version.current_log
-      if current_log.exists():
-        for log_record in cls.FromFile(file_path=current_log):
-          log_records.append(log_record)
+      current_log_filename = self.foldername / latest_version.current_log
+      if current_log_filename.exists():
+        log_records = list(self._GetLogRecords(filename=current_log_filename))
+        processed_files.add(current_log_filename)
     else:
       print('No current log file.', file=sys.stderr)
 
-    # read records from the "young" or 0-level
+    # read and cache the records from the "young" or 0-level
     young_records = []
     for active_file in latest_version.active_files.get(0, {}).keys():
-      current_young = path / active_file
-      if current_young.exists():
-        for young_record in cls.FromFile(current_young):
-          young_records.append(young_record)
+      current_young_filename = self.foldername / active_file
+      if current_young_filename.exists():
+        young_records = list(self._GetLdbRecords(current_young_filename))
+        processed_files.add(current_young_filename)
+      else:
+        print(
+            f'Could not find {current_young_filename} for level 0.',
+            file=sys.stderr)
 
+    # sort the log records by the leveldb sequence number in reverse
+    # order and update the recovered attribute based on the highest sequence
+    # number for a key.
     active_records = {}
     for record in sorted(
         log_records,
@@ -161,6 +243,10 @@ class LevelDBRecord:
       else:
         record.recovered = True
 
+    # sort the young records by the leveldb sequence number in reverse
+    # order and update:
+    # * the recovered attribute based on the highest sequence number for a key
+    # * the level attribute to 0
     for record in sorted(
         young_records,
         key=lambda record: record.record.sequence_number,
@@ -177,14 +263,55 @@ class LevelDBRecord:
         key=lambda record: record.record.sequence_number,
         reverse=False)
 
+    # read records from the active files in each level (except the 0 level)
+    # and update the recovered and level attribute.
     if latest_version.active_files.keys():
       for level in range(1, max(latest_version.active_files.keys()) + 1):
         for filename in latest_version.active_files.get(level, []):
-          current_filename = path / filename
-          for record in cls.FromFile(file_path=current_filename):
-            if record.record.key in active_records:
-              record.recovered = True
-            else:
-              record.recovered = False
-            record.level = level
-            yield record
+          current_filename = self.foldername / filename
+          if current_filename.exists():
+            processed_files.add(current_filename)
+            for record in self._GetLdbRecords(filename=current_filename):
+              record.recovered = record.record.key in active_records
+              record.level = level
+              yield record
+          else:
+            print(
+                f'Could not find {current_filename} for level {level}.',
+                file=sys.stderr)
+
+    # as a final step, parse any other log/ldb files which we will consider
+    # any records as recovered since they are not listed in the the active file
+    # set.
+    for log_file in self.LogFiles():
+      if log_file in processed_files:
+        continue
+      for record in self._GetLogRecords(filename=log_file):
+        record.recovered = True
+        yield record
+
+    for ldb_file in self.LdbFiles():
+      if ldb_file in processed_files:
+        continue
+      for record in self._GetLdbRecords(filename=ldb_file):
+        record.recovered = True
+        yield record
+
+  def GetRecords(
+      self,
+      use_manifest: bool = False
+  ) -> Generator[LevelDBRecord, None, None]:
+    """Yield LevelDBRecords.
+
+    Args:
+      use_manifest: True to use the current manifest in the folder as a means to
+          find the active file set.
+
+    Yields:
+      LevelDBRecords.
+    """
+    if use_manifest:
+      yield from self._RecordsByManifest()
+    else:
+      for filename in self.foldername.iterdir():
+        yield from LevelDBRecord.FromFile(filename)

dfindexeddb/version.py

@@ -15,7 +15,7 @@
 """Version information for dfIndexeddb."""
 
 
-__version__ = "20240417"
+__version__ = "20240501"
 
 
 def GetVersion():

{dfindexeddb-20240417.dist-info → dfindexeddb-20240501.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dfindexeddb
-Version: 20240417
+Version: 20240501
 Summary: dfindexeddb is an experimental Python tool for performing digital forensic analysis of IndexedDB and leveldb files.
 Author-email: Syd Pleno <sydp@google.com>
 Maintainer-email: dfIndexeddb Developers <dfindexeddb-dev@googlegroups.com>
@@ -223,12 +223,12 @@ Requires-Dist: zstd ==1.5.5.1
 # dfIndexeddb
 
 dfindexeddb is an experimental Python tool for performing digital forensic
-analysis of IndexedDB and leveldb files.
+analysis of IndexedDB and LevelDB files.
 
-It parses leveldb, IndexedDB and javascript structures from these files without
+It parses LevelDB, IndexedDB and JavaScript structures from these files without
 requiring native libraries.  (Note: only a subset of IndexedDB key types and
-Javascript types for Chromium-based browsers are currently supported.  Safari
-and Firefox are under development).
+JavaScript types for Safari and Chromium-based browsers are currently supported.
+Firefox is under development).
 
 The content of IndexedDB files is dependent on what a web application stores
 locally/offline using the web browser's
@@ -275,7 +275,7 @@ include:
 
 ## Usage
 
-Two CLI tools for parsing IndexedDB/leveldb files are available after
+Two CLI tools for parsing IndexedDB/LevelDB files are available after
 installation:
 
 
@@ -297,49 +297,42 @@ options:
   -h, --help    show this help message and exit
 ```
 
-To parse Indexeddb records from a LevelDB folder, use the following command:
+#### Examples:
 
-```
-dfindexeddb db -h
-usage: dfindexeddb db [-h] -s SOURCE [--use_manifest] [-o {json,jsonl,repr}]
+To parse IndexedDB records from an sqlite file for Safari and output the
+results as JSON-L, use the following command:
 
-options:
-  -h, --help            show this help message and exit
-  -s SOURCE, --source SOURCE
-                        The source leveldb folder
-  --use_manifest        Use manifest file to determine active/recovered records.
-  -o {json,jsonl,repr}, --output {json,jsonl,repr}
-                        Output format. Default is json
+```
+dfindexeddb db -s SOURCE --format safari -o jsonl
 ```
 
-To parse Indexeddb records from a LevelDB ldb (.ldb) file, use the following 
-command:
+To parse IndexedDB records from a LevelDB folder for Chrome/Chromium, using the
+manifest file to determine recovered records and output as JSON, use the
+following command:
 
 ```
-dfindexeddb ldb -h
-usage: dfindexeddb ldb [-h] -s SOURCE [-o {json,jsonl,repr}]
+dfindexeddb db -s SOURCE --format chrome --use_manifest
+```
+
+To parse IndexedDB records from a LevelDB ldb (.ldb) file and output the
+results as JSON-L, use the following command:
 
-options:
-  -h, --help            show this help message and exit
-  -s SOURCE, --source SOURCE
-                        The source .ldb file.
-  -o {json,jsonl,repr}, --output {json,jsonl,repr}
-                        Output format. Default is json
+```
+dfindexeddb ldb -s SOURCE -o jsonl
 ```
 
-To parse Indexeddb records from a LevelDB log (.log) file, use the following 
-command:
+To parse IndexedDB records from a LevelDB log (.log) file and output the
+results as the Python printable representation, use the following command:
 
 ```
-dfindexeddb log -h
-usage: dfindexeddb log [-h] -s SOURCE [-o {json,jsonl,repr}]
+dfindexeddb log -s SOURCE -o repr
+```
+
+To parse a file as a Chrome/Chromium IndexedDB blink value and output the
+results as JSON:
 
-options:
-  -h, --help            show this help message and exit
-  -s SOURCE, --source SOURCE
-                        The source .log file.
-  -o {json,jsonl,repr}, --output {json,jsonl,repr}
-                        Output format. Default is json
+```
+dfindexeddb blink -s SOURCE
 ```
 
 ### LevelDB
@@ -361,52 +354,32 @@ options:
   -h, --help            show this help message and exit
 ```
 
+#### Examples
+
 To parse records from a LevelDB folder, use the following command:
 
 ```
-dfindexeddb db -h
-usage: dfindexeddb db [-h] -s SOURCE [--use_manifest] [-o {json,jsonl,repr}]
-
-options:
-  -h, --help            show this help message and exit
-  -s SOURCE, --source SOURCE
-                        The source leveldb folder
-  --use_manifest        Use manifest file to determine active/recovered records.
-  -o {json,jsonl,repr}, --output {json,jsonl,repr}
-                        Output format. Default is json
+dfindexeddb db -s SOURCE
 ```
 
-To parse records from a LevelDB log (.log) file, use the following command:
+To parse blocks / physical records/ write batches / internal key records from a
+LevelDB log (.log) file, use the following command, specifying the type (block,
+physical_records, etc) via the `-t` option.  By default, internal key records are parsed:
 
 ```
-$ dfleveldb log  -s SOURCE [-o {json,jsonl,repr}] [-t {blocks,physical_records,write_batches,parsed_internal_key}]
-
-options:
-  -h, --help            show this help message and exit
-  -s SOURCE, --source SOURCE
-                        The source leveldb file
-  -o {json,jsonl,repr}, --output {json,jsonl,repr}
-                        Output format. Default is json
-  -t {blocks,physical_records,write_batches,parsed_internal_key}, --structure_type {blocks,physical_records,write_batches,parsed_internal_key}
-                        Parses the specified structure. Default is parsed_internal_key.
+$ dfleveldb log  -s SOURCE [-t {blocks,physical_records,write_batches,parsed_internal_key}]
 ```
 
-To parse records from a LevelDB table (.ldb) file, use the following command:
+To parse blocks / records from a LevelDB table (.ldb) file, use the following
+command, specifying the type (blocks, records) via the `-t` option.  By
+default, records are parsed:
 
 ```
-$ dfleveldb ldb -s SOURCE [-o {json,jsonl,repr}] [-t {blocks,records}]
-
-options:
-  -h, --help            show this help message and exit
-  -s SOURCE, --source SOURCE
-                        The source leveldb file
-  -o {json,jsonl,repr}, --output {json,jsonl,repr}
-                        Output format. Default is json
-  -t {blocks,records}, --structure_type {blocks,records}
-                        Parses the specified structure. Default is records.
+$ dfleveldb ldb -s SOURCE [-t {blocks,records}]
 ```
 
-To parse version edit records from a Descriptor (MANIFEST) file:
+To parse version edit records from a Descriptor (MANIFEST) file, use the
+following command:
 
 ```
 $ dfleveldb descriptor -s SOURCE [-o {json,jsonl,repr}] [-t {blocks,physical_records,versionedit} | -v]

{dfindexeddb-20240417.dist-info → dfindexeddb-20240501.dist-info}/RECORD

@@ -1,29 +1,32 @@
 dfindexeddb/__init__.py,sha256=KPYL9__l8od6_OyDfGRTgaJ6iy_fqIgZ-dS2S-e3Rac,599
 dfindexeddb/errors.py,sha256=PNpwyf_lrPc4TE77oAakX3mu5D_YcP3f80wq8Y1LkvY,749
 dfindexeddb/utils.py,sha256=pV2blFnMxDwk3kBRK6UVji66ctkYpm6wfH9p0jCC7Nk,8797
-dfindexeddb/version.py,sha256=9l1f_ZrRYqrqSAqNJB60Bvu4BzyNlWXvowSOCQQtpKw,751
+dfindexeddb/version.py,sha256=w3jTfQPKTCKVn_YxB-eYjw2WM5lN7n0625noN-CZErc,751
 dfindexeddb/indexeddb/__init__.py,sha256=kExXSVBCTKCD5BZJkdMfUMqGksH-DMJxP2_lI0gq-BE,575
-dfindexeddb/indexeddb/cli.py,sha256=apS5gfiJdkBJs5rLCAol5LzJ_d9lrHyWWfSqGupqp_w,6851
+dfindexeddb/indexeddb/cli.py,sha256=MsVqxIhZWXoiBEDbysO9gWztsUkZ8cwFMCYmgSZViLw,5980
 dfindexeddb/indexeddb/utils.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dfindexeddb/indexeddb/chromium/__init__.py,sha256=kExXSVBCTKCD5BZJkdMfUMqGksH-DMJxP2_lI0gq-BE,575
 dfindexeddb/indexeddb/chromium/blink.py,sha256=Sa5BvDkuwXg2IPc4iUDkNA9cbQKpMnXd3bPpNWSPG0I,32122
 dfindexeddb/indexeddb/chromium/definitions.py,sha256=1a-AmHVZ95uDB6se_fdarwJR8q0tFMQNh2xrZ2-VxN8,8739
-dfindexeddb/indexeddb/chromium/record.py,sha256=VJAXBgqWirS8Xjatf8_lNsepvZ3ix00JSpg8UveK9jE,45079
+dfindexeddb/indexeddb/chromium/record.py,sha256=kdItbJC5Mo6xWW71QylWvvKxtS4yYJ4BGc_ABHaHIiM,47366
 dfindexeddb/indexeddb/chromium/v8.py,sha256=NsbMgA6nRcAfdLg6CFwWadwsDS6TJ95-4MrgphaTuLw,22102
 dfindexeddb/indexeddb/firefox/__init__.py,sha256=kExXSVBCTKCD5BZJkdMfUMqGksH-DMJxP2_lI0gq-BE,575
 dfindexeddb/indexeddb/safari/__init__.py,sha256=kExXSVBCTKCD5BZJkdMfUMqGksH-DMJxP2_lI0gq-BE,575
+dfindexeddb/indexeddb/safari/definitions.py,sha256=nW8MmYx9Ob86W4pxm4QD4Xvr5QjoV34-U7wDhm2GIr0,2779
+dfindexeddb/indexeddb/safari/record.py,sha256=bzoMSgpXs2SsEOKHjVh9tkJDZtzGkQByq3G5dK_Yd7Q,8010
+dfindexeddb/indexeddb/safari/webkit.py,sha256=eQeKXOcVdCaZKM4xekw1d3iLTynXc_e-GCT_x9iSJNs,21595
 dfindexeddb/leveldb/__init__.py,sha256=KPYL9__l8od6_OyDfGRTgaJ6iy_fqIgZ-dS2S-e3Rac,599
-dfindexeddb/leveldb/cli.py,sha256=K_AnRyNteYbyd8Lz-4bPHJpyeFEYFqSPNE_wpp1-FrY,8060
+dfindexeddb/leveldb/cli.py,sha256=H1QmbZ2jQ75LqsxPaiOjflljDtYGCSxtsDI3mEf7BNU,7982
 dfindexeddb/leveldb/definitions.py,sha256=lPW_kjc47vyoGOoEWfgWvKcpGbN-0h7XXwCeMoFmYKk,1486
 dfindexeddb/leveldb/descriptor.py,sha256=WR3irG16oIE6VbaP9UPnzOD3KlHR8GYFnoeG6ySJUzU,12211
 dfindexeddb/leveldb/ldb.py,sha256=mN-M7PLtE_VLZCbCbzRgjkSezbMUhgDjgWgPgIxJ1jM,8087
 dfindexeddb/leveldb/log.py,sha256=QeH8oESOPEZUjANGiDRSmXZa2SuoKlPFBJY7SxTV1lg,9209
-dfindexeddb/leveldb/record.py,sha256=0wXVibHC8QX8NQRsG9jcTlEb2rvaBFta20pOXdKFRMU,6170
+dfindexeddb/leveldb/record.py,sha256=_xmeEBXz5nimqteOMAf0zYoi8wDLgG27F4BhLk6N21Q,10747
 dfindexeddb/leveldb/utils.py,sha256=RgEEZ7Z35m3CcOUypAiViQSzKjBgSXZ3aeJhQjY3H9w,3748
-dfindexeddb-20240417.dist-info/AUTHORS,sha256=QbvjbAom57fpEkekkCVFUj0B9KUMGraR510aUMBC-PE,286
-dfindexeddb-20240417.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
-dfindexeddb-20240417.dist-info/METADATA,sha256=Mw_W56yj0csKALgtIpn1gS4Po8sfSn6bPMnpp5Hm_1o,19937
-dfindexeddb-20240417.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-dfindexeddb-20240417.dist-info/entry_points.txt,sha256=WG9YNLZ9lBx4Q9QF6wS4dZdZfADT3Zs4_-MV5TcA0ls,102
-dfindexeddb-20240417.dist-info/top_level.txt,sha256=X9OTaub1c8S_JJ7g-f8JdkhhdiZ4x1j4eni1hdUCwE4,12
-dfindexeddb-20240417.dist-info/RECORD,,
+dfindexeddb-20240501.dist-info/AUTHORS,sha256=QbvjbAom57fpEkekkCVFUj0B9KUMGraR510aUMBC-PE,286
+dfindexeddb-20240501.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+dfindexeddb-20240501.dist-info/METADATA,sha256=Ejikzq8iS6BHrSroiBS4A_xR7O4GJnwVsmyoMF25jUM,18498
+dfindexeddb-20240501.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+dfindexeddb-20240501.dist-info/entry_points.txt,sha256=WG9YNLZ9lBx4Q9QF6wS4dZdZfADT3Zs4_-MV5TcA0ls,102
+dfindexeddb-20240501.dist-info/top_level.txt,sha256=X9OTaub1c8S_JJ7g-f8JdkhhdiZ4x1j4eni1hdUCwE4,12
+dfindexeddb-20240501.dist-info/RECORD,,