dissect.database 1.1.dev2__tar.gz → 1.1.dev4__tar.gz

dissect_database-1.1.dev4/MANIFEST.in

@@ -0,0 +1,4 @@
+exclude .gitattributes
+exclude .gitignore
+recursive-exclude .github/ *
+recursive-exclude tests/_data/ *

{dissect_database-1.1.dev2 → dissect_database-1.1.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dissect.database
-Version: 1.1.dev2
+Version: 1.1.dev4
 Summary: A Dissect module implementing parsers for various database formats, including Berkeley DB, Microsofts Extensible Storage Engine (ESE) and SQLite3
 Author-email: Dissect Team <dissect@fox-it.com>
 License-Expression: Apache-2.0
@@ -22,10 +22,13 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 License-File: COPYRIGHT
 Requires-Dist: dissect.cstruct<5,>=4
-Requires-Dist: dissect.util<4,>=3.5
+Requires-Dist: dissect.util<4,>=3.24.dev1
+Provides-Extra: full
+Requires-Dist: pycryptodome; extra == "full"
 Provides-Extra: dev
+Requires-Dist: dissect.database[full]; extra == "dev"
 Requires-Dist: dissect.cstruct<5.0.dev,>=4.0.dev; extra == "dev"
-Requires-Dist: dissect.util<4.0.dev,>=3.5.dev; extra == "dev"
+Requires-Dist: dissect.util<4.0.dev,>=3.24.dev; extra == "dev"
 Dynamic: license-file
 
 # dissect.database

{dissect_database-1.1.dev2 → dissect_database-1.1.dev4}/dissect/database/ese/btree.py

@@ -116,7 +116,8 @@ class BTree:
         """
         page = self._page
         while True:
-            node = find_node(page, key)
+            num = find_node(page, key)
+            node = page.node(num)
 
             if page.is_branch:
                 page = self.db.page(node.child)
@@ -132,35 +133,45 @@ class BTree:
         return self.node()
 
 
-def find_node(page: Page, key: bytes) -> Node:
+def find_node(page: Page, key: bytes) -> int:
     """Search a page for a node matching ``key``.
 
+    Referencing Extensible-Storage-Engine source, they bail out early if they find an exact match.
+    However, we prefer to always find the _first_ node that is greater than or equal to the key,
+    so we can handle cases where there are duplicate index keys. This is important for "range" searches
+    where we want to find all keys matching a certain prefix, and not end up somewhere in the middle of the range.
+
     Args:
         page: The page to search.
         key: The key to search.
+
+    Returns:
+        The node number of the first node that's greater than or equal to the key.
     """
-    first_node_idx = 0
-    last_node_idx = page.node_count - 1
+    lo, hi = 0, page.node_count - 1
+    res = 0
 
     node = None
-    while first_node_idx < last_node_idx:
-        node_idx = (first_node_idx + last_node_idx) // 2
-        node = page.node(node_idx)
+    while lo < hi:
+        mid = (lo + hi) // 2
+        node = page.node(mid)
 
         # It turns out that the way BTree keys are compared matches 1:1 with how Python compares bytes
         # First compare data, then length
-        if key < node.key:
-            last_node_idx = node_idx
-        elif key == node.key:
-            if page.is_branch:
-                # If there's an exact match on a key on a branch page, the actual leaf nodes are in the next branch
-                # Page keys for branch pages appear to be non-inclusive upper bounds
-                node_idx = min(node_idx + 1, page.node_count - 1)
-                node = page.node(node_idx)
+        res = (key < node.key) - (key > node.key)
 
-            return node
+        if res < 0:
+            lo = mid + 1
         else:
-            first_node_idx = node_idx + 1
+            hi = mid
+
+    # Final comparison on the last node
+    node = page.node(lo)
+    res = (key < node.key) - (key > node.key)
+
+    if page.is_branch and res == 0:
+        # If there's an exact match on a key on a branch page, the actual leaf nodes are in the next branch
+        # Page keys for branch pages appear to be non-inclusive upper bounds
+        lo = min(lo + 1, page.node_count - 1)
 
-    # We're at the last node
-    return page.node(first_node_idx)
+    return lo

{dissect_database-1.1.dev2 → dissect_database-1.1.dev4}/dissect/database/ese/c_ese.py

@@ -231,7 +231,7 @@ struct DBFILEHDR {
     BKINFOTYPE  bkinfoTypeCopyPrev;                 //  Type of Last successful Incremental backup
     BKINFOTYPE  bkinfoTypeDiffPrev;                 //  Type of Last successful Differential backup
 
-// 476 bytes
+//  476 bytes
     ULONG       ulIncrementalReseedCount;           //  number of times incremental reseed has been initiated on this database
     LOGTIME     logtimeIncrementalReseed;           //  the date of the last time that incremental reseed was initiated on this database
     ULONG       ulIncrementalReseedCountOld;        //  number of times incremental reseed was initiated on this database before the last defrag
@@ -240,7 +240,7 @@ struct DBFILEHDR {
     LOGTIME     logtimePagePatch;                   //  the date of the last time that a page was patched as a part of incremental reseed
     ULONG       ulPagePatchCountOld;                //  number of pages patched in the database as a part of incremental reseed before the last defrag
 
-// 508 bytes
+//  508 bytes
     QWORD       qwSortVersion;                      // DEPRECATED: In old versions had "default" (?English?) LCID version, in new versions has 0xFFFFFFFFFFFF.
 
 //  516 bytes                                       // checksum during recovery state
@@ -399,6 +399,22 @@ flag TAGFLD_HEADER : uint8 {
     Encrypted               = 0x40,                 // fEncrypted
 };
 
+flag FIELDFLAG : uint16 {
+    NotNull                 = 0x0001,               // NULL values not allowed
+    Version                 = 0x0002,               // Version field
+    Autoincrement           = 0x0004,               // Autoincrement field
+    Multivalued             = 0x0008,               // Multi-valued column
+    Default                 = 0x0010,               // Column has ISAM default value
+    EscrowUpdate            = 0x0020,               // Escrow updated column
+    Finalize                = 0x0040,               // Finalizable column
+    UserDefinedDefault      = 0x0080,               // The default value is generated through a callback
+    TemplateColumnESE98     = 0x0100,               // Template table column created in ESE98 (ie. fDerived bit will be set in TAGFLD of records of derived tables)
+    DeleteOnZero            = 0x0200,               // DeleteOnZero column
+    PrimaryIndexPlaceholder = 0x0800,               // Field is no longer in primary index, but must be retained as a placeholder
+    Compressed              = 0x1000,               // Data stored in the column should be compressed
+    Encrypted               = 0x2000,               // Data stored in the column is encrypted
+};
+
 flag JET_bitIndex : uint32 {
     Unique                  = 0x00000001,
     Primary                 = 0x00000002,
@@ -467,5 +483,6 @@ TAG_FLAG = c_ese.TAG_FLAG
 TAGFLD_HEADER = c_ese.TAGFLD_HEADER
 CODEPAGE = c_ese.CODEPAGE
 COMPRESSION_SCHEME = c_ese.COMPRESSION_SCHEME
+FIELDFLAG = c_ese.FIELDFLAG
 IDBFLAG = c_ese.IDBFLAG
 IDXFLAG = c_ese.IDXFLAG

{dissect_database-1.1.dev2 → dissect_database-1.1.dev4}/dissect/database/ese/c_ese.pyi

@@ -426,6 +426,21 @@ class _c_ese(__cs__.cstruct):
         Null = ...
         Encrypted = ...
 
+    class FIELDFLAG(__cs__.Flag):
+        NotNull = ...
+        Version = ...
+        Autoincrement = ...
+        Multivalued = ...
+        Default = ...
+        EscrowUpdate = ...
+        Finalize = ...
+        UserDefinedDefault = ...
+        TemplateColumnESE98 = ...
+        DeleteOnZero = ...
+        PrimaryIndexPlaceholder = ...
+        Compressed = ...
+        Encrypted = ...
+
     class JET_bitIndex(__cs__.Flag):
         Unique = ...
         Primary = ...
@@ -487,5 +502,6 @@ TAG_FLAG: TypeAlias = c_ese.TAG_FLAG
 TAGFLD_HEADER: TypeAlias = c_ese.TAGFLD_HEADER
 CODEPAGE: TypeAlias = c_ese.CODEPAGE
 COMPRESSION_SCHEME: TypeAlias = c_ese.COMPRESSION_SCHEME
+FIELDFLAG: TypeAlias = c_ese.FIELDFLAG
 IDBFLAG: TypeAlias = c_ese.IDBFLAG
 IDXFLAG: TypeAlias = c_ese.IDXFLAG

{dissect_database-1.1.dev2 → dissect_database-1.1.dev4}/dissect/database/ese/cursor.py

@@ -3,12 +3,14 @@ from __future__ import annotations
 from typing import TYPE_CHECKING
 
 from dissect.database.ese.btree import BTree
-from dissect.database.ese.exception import NoNeighbourPageError
+from dissect.database.ese.exception import KeyNotFoundError, NoNeighbourPageError
 from dissect.database.ese.record import Record
 
 if TYPE_CHECKING:
     from collections.abc import Iterator
 
+    from typing_extensions import Self
+
     from dissect.database.ese.index import Index
     from dissect.database.ese.page import Node
     from dissect.database.ese.util import RecordValue
@@ -30,13 +32,10 @@ class Cursor:
         self._secondary = None if index.is_primary else BTree(self.db, self.table.root)
 
     def __iter__(self) -> Iterator[Record]:
-        while True:
-            yield self._record()
-
-            try:
-                self._primary.next()
-            except NoNeighbourPageError:
-                break
+        record = self.record()
+        while record is not None:
+            yield record
+            record = self.next()
 
     def _node(self) -> Node:
         """Return the node the cursor is currently on. Resolves the secondary index if needed.
@@ -50,7 +49,7 @@ class Cursor:
             node = self._secondary.search(node.data.tobytes(), exact=True)
         return node
 
-    def _record(self) -> Record:
+    def record(self) -> Record:
         """Return the record the cursor is currently on.
 
         Returns:
@@ -58,25 +57,76 @@ class Cursor:
         """
         return Record(self.table, self._node())
 
-    def reset(self) -> None:
+    def reset(self) -> Self:
         """Reset the internal state."""
         self._primary.reset()
         if self._secondary:
             self._secondary.reset()
+        return self
+
+    def next(self) -> Record | None:
+        """Move the cursor to the next record and return it.
+
+        Can move the cursor to the next page as a side effect.
 
-    def search(self, **kwargs: RecordValue) -> Record:
+        Returns:
+            A :class:`~dissect.database.ese.record.Record` object of the next record.
+        """
+        try:
+            self._primary.next()
+        except NoNeighbourPageError:
+            return None
+        return self.record()
+
+    def prev(self) -> Record | None:
+        """Move the cursor to the previous node and return it.
+
+        Can move the cursor to the previous page as a side effect.
+
+        Returns:
+            A :class:`~dissect.database.ese.record.Record` object of the previous record.
+        """
+        try:
+            self._primary.prev()
+        except NoNeighbourPageError:
+            return None
+        return self.record()
+
+    def make_key(self, *args: RecordValue, **kwargs: RecordValue) -> bytes:
+        """Generate a key for this index from the given values.
+
+        Args:
+            *args: The values to generate a key for.
+            **kwargs: The columns and values to generate a key for.
+
+        Returns:
+            The generated key as bytes.
+        """
+        if not args and not kwargs:
+            raise ValueError("At least one value must be provided")
+
+        if args and kwargs:
+            raise ValueError("Cannot mix positional and keyword arguments in make_key")
+
+        if args and not len(args) == 1 and not isinstance(args[0], list):
+            raise ValueError("When using positional arguments, provide a single list of values")
+
+        return self.index.make_key(args[0] if args else kwargs)
+
+    def search(self, *args: RecordValue, **kwargs: RecordValue) -> Record:
         """Search the index for the requested values.
 
         Searching modifies the cursor state. Searching again will search from the current position.
         Reset the cursor with :meth:`reset` to start from the beginning.
 
         Args:
+            *args: The values to search for.
             **kwargs: The columns and values to search for.
 
         Returns:
             A :class:`~dissect.database.ese.record.Record` object of the found record.
         """
-        key = self.index.make_key(kwargs)
+        key = self.make_key(*args, **kwargs)
         return self.search_key(key, exact=True)
 
     def search_key(self, key: bytes, exact: bool = True) -> Record:
@@ -88,24 +138,27 @@ class Cursor:
                    next record that is greater than or equal to the key.
         """
         self._primary.search(key, exact)
-        return self._record()
+        return self.record()
 
-    def seek(self, **kwargs: RecordValue) -> None:
+    def seek(self, *args: RecordValue, **kwargs: RecordValue) -> Self:
         """Seek to the record with the given values.
 
         Args:
+            *args: The values to seek to.
             **kwargs: The columns and values to seek to.
         """
-        key = self.index.make_key(kwargs)
+        key = self.make_key(*args, **kwargs)
         self.search_key(key, exact=False)
+        return self
 
-    def seek_key(self, key: bytes) -> None:
+    def seek_key(self, key: bytes) -> Self:
         """Seek to the record with the given ``key``.
 
         Args:
             key: The key to seek to.
         """
         self._primary.search(key, exact=False)
+        return self
 
     def find(self, **kwargs: RecordValue) -> Record | None:
         """Find a record in the index.
@@ -130,7 +183,10 @@ class Cursor:
         other_columns = kwargs
 
         # We need at least an exact match on the indexed columns
-        self.search(**indexed_columns)
+        try:
+            self.search(**indexed_columns)
+        except KeyNotFoundError:
+            return
 
         current_key = self._primary.node().key
 
@@ -150,7 +206,7 @@ class Cursor:
             if current_key != self._primary.node().key:
                 break
 
-            record = self._record()
+            record = self.record()
             for k, v in other_columns.items():
                 value = record.get(k)
                 # If the record value is a list, we do a check based on the queried value
@@ -172,39 +228,3 @@ class Cursor:
                 self._primary.next()
             except NoNeighbourPageError:
                 break
-
-    def record(self) -> Record:
-        """Return the record the cursor is currently on.
-
-        Returns:
-            A :class:`~dissect.database.ese.record.Record` object of the current record.
-        """
-        return self._record()
-
-    def next(self) -> Record:
-        """Move the cursor to the next record and return it.
-
-        Can move the cursor to the next page as a side effect.
-
-        Returns:
-            A :class:`~dissect.database.ese.record.Record` object of the next record.
-        """
-        try:
-            self._primary.next()
-        except NoNeighbourPageError:
-            raise IndexError("No next record")
-        return self._record()
-
-    def prev(self) -> Record:
-        """Move the cursor to the previous node and return it.
-
-        Can move the cursor to the previous page as a side effect.
-
-        Returns:
-            A :class:`~dissect.database.ese.record.Record` object of the previous record.
-        """
-        try:
-            self._primary.prev()
-        except NoNeighbourPageError:
-            raise IndexError("No previous record")
-        return self._record()

{dissect_database-1.1.dev2 → dissect_database-1.1.dev4}/dissect/database/ese/ese.py

@@ -36,6 +36,7 @@ class ESE:
         self.fh = fh
         self.impacket_compat = impacket_compat
 
+        self.fh.seek(0)
         self.header = c_ese.DBFILEHDR(fh)
         if self.header.ulMagic != ulDAEMagic:
             raise InvalidDatabase("invalid file header signature")

{dissect_database-1.1.dev2 → dissect_database-1.1.dev4}/dissect/database/ese/index.py

@@ -74,16 +74,17 @@ class Index:
         """Create a new cursor for this index."""
         return Cursor(self)
 
-    def search(self, **kwargs) -> Record:
+    def search(self, *args, **kwargs) -> Record:
         """Search the index for the requested values.
 
         Args:
+            *args: The values to search for.
             **kwargs: The columns and values to search for.
 
         Returns:
             A :class:`~dissect.database.ese.record.Record` object of the found record.
         """
-        return self.cursor().search(**kwargs)
+        return self.cursor().search(*args, **kwargs)
 
     def search_key(self, key: bytes) -> Node:
         """Search the index for a specific ``key``.
@@ -105,20 +106,27 @@ class Index:
         values = {c.name: record[c.name] for c in self.columns}
         return self.make_key(values)
 
-    def make_key(self, values: dict[str, RecordValue]) -> bytes:
+    def make_key(self, values: list[RecordValue] | dict[str, RecordValue]) -> bytes:
         """Generate a key out of the given values.
 
         Args:
-            values: A map of the column names and values to generate a key for.
+            values: A map of the column names and values to generate a key for, or a list of values in the order of the
+                    index columns.
         """
         key_buf = []
         key_remaining = self._key_most
 
-        for column in self.columns:
-            if column.name not in values:
-                break
+        if isinstance(values, dict):
+            tmp = []
+            for column in self.columns:
+                if column.name not in values:
+                    break
+                tmp.append(values[column.name])
+
+            values = tmp
 
-            key_part = encode_key(self, column, values[column.name], self._var_seg_mac)
+        for column, value in zip(self.columns, values, strict=False):
+            key_part = encode_key(self, column, value, self._var_seg_mac)
             key_buf.append(key_part)
             key_remaining -= len(key_part)
 

{dissect_database-1.1.dev2 → dissect_database-1.1.dev4}/dissect/database/ese/lcmapstring.py

@@ -1,5 +1,6 @@
 # Based on Wine source
 # https://github.com/wine-mirror/wine/blob/master/dlls/kernelbase/locale.c
+# http://www.flounder.com/localeexplorer.htm
 
 from enum import IntEnum, IntFlag
 

dissect_database-1.1.dev4/dissect/database/ese/ntds/__init__.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+from dissect.database.ese.ntds.ntds import NTDS
+from dissect.database.ese.ntds.objects import Computer, Group, Object, Server, User
+
+__all__ = [
+    "NTDS",
+    "Computer",
+    "Group",
+    "Object",
+    "Server",
+    "User",
+]

dissect_database-1.1.dev4/dissect/database/ese/ntds/c_ds.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+from dissect.cstruct import cstruct
+
+ds_def = """
+typedef struct _USER_PROPERTY {
+    WORD            NameLength;
+    WORD            ValueLength;
+    WORD            PropertyFlag;
+    WCHAR           PropertyName[NameLength / 2];
+    CHAR            PropertyValue[ValueLength];
+} USER_PROPERTY;
+
+typedef struct _USER_PROPERTIES_HEADER {
+    DWORD           Reserved1;
+    DWORD           Length;
+    WORD            Reserved2;
+    WORD            Reserved3;
+    CHAR            Reserved4[96];
+    WORD            PropertySignature;
+    WORD            PropertyCount;
+} USER_PROPERTIES_HEADER;
+
+typedef struct _ADAM_PROPERTIES_HEADER {    // For lack of a better name
+    DWORD           Reserved1;
+    DWORD           Reserved2;
+    DWORD           Reserved3;
+    DWORD           Reserved4;
+    DWORD           Reserved5;
+    DWORD           Reserved6;
+} ADAM_PROPERTIES_HEADER;
+
+typedef struct _KERB_KEY_DATA {
+    WORD            Reserved1;
+    WORD            Reserved2;
+    DWORD           Reserved3;
+    DWORD           KeyType;
+    DWORD           KeyLength;
+    DWORD           KeyOffset;
+} KERB_KEY_DATA;
+
+typedef struct _KERB_STORED_CREDENTIAL {
+    WORD            Revision;
+    WORD            Flags;
+    WORD            CredentialCount;
+    WORD            OldCredentialCount;
+    WORD            DefaultSaltLength;
+    WORD            DefaultSaltMaximumLength;
+    DWORD           DefaultSaltOffset;
+    KERB_KEY_DATA   Credentials[CredentialCount];
+    KERB_KEY_DATA   OldCredentials[OldCredentialCount];
+    // CHAR         DefaultSalt[DefaultSaltLength];
+    // CHAR         KeyValues[...];
+} KERB_STORED_CREDENTIAL;
+
+typedef struct _KERB_KEY_DATA_NEW {
+    WORD            Reserved1;
+    WORD            Reserved2;
+    DWORD           Reserved3;
+    DWORD           IterationCount;
+    DWORD           KeyType;
+    DWORD           KeyLength;
+    DWORD           KeyOffset;
+} KERB_KEY_DATA_NEW;
+
+typedef struct _KERB_STORED_CREDENTIAL_NEW {
+    WORD            Revision;
+    WORD            Flags;
+    WORD            CredentialCount;
+    WORD            ServiceCredentialCount;
+    WORD            OldCredentialCount;
+    WORD            OlderCredentialCount;
+    WORD            DefaultSaltLength;
+    WORD            DefaultSaltMaximumLength;
+    DWORD           DefaultSaltOffset;
+    DWORD           DefaultIterationCount;
+    KERB_KEY_DATA_NEW   Credentials[CredentialCount];
+    KERB_KEY_DATA_NEW   ServiceCredentials[ServiceCredentialCount];
+    KERB_KEY_DATA_NEW   OldCredentials[OldCredentialCount];
+    KERB_KEY_DATA_NEW   OlderCredentials[OlderCredentialCount];
+    // CHAR         DefaultSalt[DefaultSaltLength];
+    // CHAR         KeyValues[...];
+} KERB_STORED_CREDENTIAL_NEW;
+
+typedef struct _WDIGEST_CREDENTIALS {
+    BYTE    Reserved1;
+    BYTE    Reserved2;
+    BYTE    Version;
+    BYTE    NumberOfHashes;
+    CHAR    Reserved3[12];
+    CHAR    Hash[29][16];                   // The formal definition has Hash1, Hash2, ..., Hash29
+} WDIGEST_CREDENTIALS;
+"""
+c_ds = cstruct(ds_def)