dissect.database 0.1.dev3__tar.gz → 0.1.dev5__tar.gz

{dissect_database-0.1.dev3 → dissect_database-0.1.dev5}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: dissect.database
-Version: 0.1.dev3
-Summary: A Dissect module implementing parsers for various database formats
+Version: 0.1.dev5
+Summary: A Dissect module implementing parsers for various database formats, including Berkeley DB, Microsofts Extensible Storage Engine (ESE) and SQLite3
 Author-email: Dissect Team <dissect@fox-it.com>
 License-Expression: Apache-2.0
 Project-URL: homepage, https://dissect.tools
@@ -22,6 +22,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 License-File: COPYRIGHT
 Requires-Dist: dissect.cstruct<5,>=4
+Requires-Dist: dissect.util<4,>=3.5
 Provides-Extra: dev
 Requires-Dist: dissect.cstruct<5.0.dev,>=4.0.dev; extra == "dev"
 Requires-Dist: dissect.util<4.0.dev,>=3.5.dev; extra == "dev"
@@ -29,8 +30,13 @@ Dynamic: license-file
 
 # dissect.database
 
-A Dissect module implementing parsers for various database formats. For more information,
-please see [the documentation](https://docs.dissect.tools/en/latest/projects/dissect.database/index.html).
+A Dissect module implementing parsers for various database formats, including:
+
+- Berkeley DB, used for example in older RPM databases
+- Microsofts Extensible Storage Engine (ESE), used for example in Active Directory, Exchange and Windows Update
+- SQLite3, commonly used by applications to store configuration data
+
+For more information, please see [the documentation](https://docs.dissect.tools/en/latest/projects/dissect.database/index.html).
 
 ## Installation
 
@@ -42,6 +48,20 @@ pip install dissect.database
 
 This module is also automatically installed if you install the `dissect` package.
 
+## Tools
+
+### Impacket compatibility shim for secretsdump.py
+
+Impacket does not ([yet](https://github.com/fortra/impacket/pull/1452)) have native support for `dissect.database`,
+so in the meantime a compatibility shim is provided. To use this shim, simply install `dissect.database` using the
+instructions above, and execute `secretsdump.py` like so:
+
+```bash
+python -m dissect.database.ese.tools.impacket /path/to/impacket/examples/secretsdump.py -h
+```
+
+Impacket `secretsdump.py` will now use `dissect.database` for parsing the `NTDS.dit` file, resulting in a significant performance improvement!
+
 ## Build and test instructions
 
 This project uses `tox` to build source and wheel distributions. Run the following command from the root folder to build

{dissect_database-0.1.dev3 → dissect_database-0.1.dev5}/README.md

@@ -1,7 +1,12 @@
 # dissect.database
 
-A Dissect module implementing parsers for various database formats. For more information,
-please see [the documentation](https://docs.dissect.tools/en/latest/projects/dissect.database/index.html).
+A Dissect module implementing parsers for various database formats, including:
+
+- Berkeley DB, used for example in older RPM databases
+- Microsofts Extensible Storage Engine (ESE), used for example in Active Directory, Exchange and Windows Update
+- SQLite3, commonly used by applications to store configuration data
+
+For more information, please see [the documentation](https://docs.dissect.tools/en/latest/projects/dissect.database/index.html).
 
 ## Installation
 
@@ -13,6 +18,20 @@ pip install dissect.database
 
 This module is also automatically installed if you install the `dissect` package.
 
+## Tools
+
+### Impacket compatibility shim for secretsdump.py
+
+Impacket does not ([yet](https://github.com/fortra/impacket/pull/1452)) have native support for `dissect.database`,
+so in the meantime a compatibility shim is provided. To use this shim, simply install `dissect.database` using the
+instructions above, and execute `secretsdump.py` like so:
+
+```bash
+python -m dissect.database.ese.tools.impacket /path/to/impacket/examples/secretsdump.py -h
+```
+
+Impacket `secretsdump.py` will now use `dissect.database` for parsing the `NTDS.dit` file, resulting in a significant performance improvement!
+
 ## Build and test instructions
 
 This project uses `tox` to build source and wheel distributions. Run the following command from the root folder to build

dissect_database-0.1.dev5/dissect/database/__init__.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+from dissect.database.bsd.db import DB
+from dissect.database.ese.ese import ESE
+from dissect.database.exception import Error
+from dissect.database.sqlite3.sqlite3 import SQLite3
+
+__all__ = [
+    "DB",
+    "ESE",
+    "Error",
+    "SQLite3",
+]

dissect_database-0.1.dev5/dissect/database/ese/__init__.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from dissect.database.ese.ese import ESE
+from dissect.database.ese.exception import (
+    InvalidDatabase,
+    KeyNotFoundError,
+    NoNeighbourPageError,
+)
+from dissect.database.ese.index import Index
+from dissect.database.ese.page import Page
+from dissect.database.ese.record import Record
+from dissect.database.ese.table import Table
+
+__all__ = [
+    "ESE",
+    "CompressedTaggedDataError",
+    "Index",
+    "InvalidDatabase",
+    "KeyNotFoundError",
+    "NoNeighbourPageError",
+    "Page",
+    "Record",
+    "Table",
+]

dissect_database-0.1.dev5/dissect/database/ese/btree.py

@@ -0,0 +1,166 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from dissect.database.ese.exception import KeyNotFoundError, NoNeighbourPageError
+
+if TYPE_CHECKING:
+    from dissect.database.ese.ese import ESE
+    from dissect.database.ese.page import Node, Page
+
+
+class BTree:
+    """A simple implementation for searching the ESE B+Trees.
+
+    This is a stateful interactive class that moves an internal cursor to a position within the BTree.
+
+    Args:
+        db: An instance of :class:`~dissect.database.ese.ese.ESE`.
+        page: The page to open the :class:`BTree` on.
+    """
+
+    def __init__(self, db: ESE, root: int | Page):
+        self.db = db
+
+        if isinstance(root, int):
+            page_num = root
+            root = db.page(page_num)
+        else:
+            page_num = root.num
+
+        self.root = root
+
+        self._page = root
+        self._page_num = page_num
+        self._node_num = 0
+
+    def reset(self) -> None:
+        """Reset the internal state to the root of the BTree."""
+        self._page = self.root
+        self._page_num = self._page.num
+        self._node_num = 0
+
+    def node(self) -> Node:
+        """Return the node the BTree is currently on.
+
+        Returns:
+            A :class:`~dissect.database.ese.page.Node` object of the current node.
+        """
+        return self._page.node(self._node_num)
+
+    def next(self) -> Node:
+        """Move the BTree to the next node and return it.
+
+        Can move the BTree to the next page as a side effect.
+
+        Returns:
+            A :class:`~dissect.database.ese.page.Node` object of the next node.
+        """
+        if self._node_num + 1 > self._page.node_count - 1:
+            self.next_page()
+        else:
+            self._node_num += 1
+
+        return self.node()
+
+    def next_page(self) -> None:
+        """Move the BTree to the next page in the tree.
+
+        Raises:
+            NoNeighbourPageError: If the current page has no next page.
+        """
+        if self._page.next_page:
+            self._page = self.db.page(self._page.next_page)
+            self._node_num = 0
+        else:
+            raise NoNeighbourPageError(f"{self._page} has no next page")
+
+    def prev(self) -> Node:
+        """Move the BTree to the previous node and return it.
+
+        Can move the BTree to the previous page as a side effect.
+
+        Returns:
+            A :class:`~dissect.database.ese.page.Node` object of the previous node.
+        """
+        if self._node_num - 1 < 0:
+            self.prev_page()
+        else:
+            self._node_num -= 1
+
+        return self.node()
+
+    def prev_page(self) -> None:
+        """Move the BTree to the previous page in the tree.
+
+        Raises:
+            NoNeighbourPageError: If the current page has no previous page.
+        """
+        if self._page.previous_page:
+            self._page = self.db.page(self._page.previous_page)
+            self._node_num = self._page.node_count - 1
+        else:
+            raise NoNeighbourPageError(f"{self._page} has no previous page")
+
+    def search(self, key: bytes, exact: bool = True) -> Node:
+        """Search the tree for the given ``key``.
+
+        Moves the BTree to the matching node, or on the last node that is less than the requested key.
+
+        Args:
+            key: The key to search for.
+            exact: Whether to only return successfully on an exact match.
+
+        Raises:
+            KeyNotFoundError: If an ``exact`` match was requested but not found.
+        """
+        page = self._page
+        while True:
+            node = find_node(page, key)
+
+            if page.is_branch:
+                page = self.db.page(node.child)
+            else:
+                self._page = page
+                self._page_num = page.num
+                self._node_num = node.num
+                break
+
+        if exact and key != node.key:
+            raise KeyNotFoundError(f"Can't find key: {key}")
+
+        return self.node()
+
+
+def find_node(page: Page, key: bytes) -> Node:
+    """Search a page for a node matching ``key``.
+
+    Args:
+        page: The page to search.
+        key: The key to search.
+    """
+    first_node_idx = 0
+    last_node_idx = page.node_count - 1
+
+    node = None
+    while first_node_idx < last_node_idx:
+        node_idx = (first_node_idx + last_node_idx) // 2
+        node = page.node(node_idx)
+
+        # It turns out that the way BTree keys are compared matches 1:1 with how Python compares bytes
+        # First compare data, then length
+        if key < node.key:
+            last_node_idx = node_idx
+        elif key == node.key:
+            if page.is_branch:
+                # If there's an exact match on a key on a branch page, the actual leaf nodes are in the next branch
+                # Page keys for branch pages appear to be non-inclusive upper bounds
+                node_idx = min(node_idx + 1, page.node_count - 1)
+                node = page.node(node_idx)
+
+            return node
+        else:
+            first_node_idx = node_idx + 1
+
+    # We're at the last node
+    return page.node(first_node_idx)