dissect.database 1.1.dev10__tar.gz → 1.1.dev11__tar.gz

{dissect_database-1.1.dev10/dissect.database.egg-info → dissect_database-1.1.dev11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dissect.database
-Version: 1.1.dev10
+Version: 1.1.dev11
 Summary: A Dissect module implementing parsers for various database formats, including Berkeley DB, Microsofts Extensible Storage Engine (ESE) and SQLite3
 Author-email: Dissect Team <dissect@fox-it.com>
 License-Expression: Apache-2.0

{dissect_database-1.1.dev10 → dissect_database-1.1.dev11}/dissect/database/ese/index.py

@@ -282,7 +282,7 @@ def _encode_text(index: Index, column: Column, value: str, max_size: int) -> byt
     else:
         # Unicode strings == LCMapStringW
         flags = index.record.get("LCMapFlags")
-        locale = index.record.get("LocaleName").decode("utf-16-le")
+        locale = local_name.decode("utf-16-le") if (local_name := index.record.get("LocaleName")) is not None else ""
         segment = map_string(value, flags, locale)
         key += segment[:max_size]
 

{dissect_database-1.1.dev10 → dissect_database-1.1.dev11}/dissect/database/ese/ntds/database.py

@@ -17,7 +17,7 @@ if TYPE_CHECKING:
     from collections.abc import Iterator
 
     from dissect.database.ese.index import Index
-    from dissect.database.ese.ntds.objects import Top
+    from dissect.database.ese.ntds.objects import DMD, NTDSDSA, Top
 
 
 class Database:
@@ -43,6 +43,8 @@ class DataTable:
     def __init__(self, db: Database):
         self.db = db
         self.table = self.db.ese.table("datatable")
+        self.hiddentable = self.db.ese.table("hiddentable")
+        self.hiddeninfo = next(self.hiddentable.records(), None)
 
         self.schema = Schema()
 
@@ -50,6 +52,18 @@ class DataTable:
         self.get = lru_cache(4096)(self.get)
         self._make_dn = lru_cache(4096)(self._make_dn)
 
+    def dsa(self) -> NTDSDSA:
+        """Return the Directory System Agent (DSA) object."""
+        if not self.hiddeninfo:
+            raise ValueError("No hiddentable information available")
+        return self.get(self.hiddeninfo.get("dsa_col"))
+
+    def dmd(self) -> DMD:
+        """Return the Directory Management Domain (DMD) object, a.k.a. the schema container."""
+        if not self.hiddeninfo:
+            raise ValueError("No hiddentable information available")
+        return self.get(self.dsa().get("dMDLocation", raw=True))
+
     def root(self) -> Top:
         """Return the top-level object in the NTDS database."""
         if (root := next(self.children_of(0), None)) is None:
@@ -129,7 +143,7 @@ class DataTable:
             raise ValueError(f"Attribute {key!r} is not found in the schema")
 
         index = self.table.find_index(schema.column)
-        record = index.search([encode_value(self.db, key, value)])
+        record = index.search([encode_value(self.db, schema, value)])
         return Object.from_record(self.db, record)
 
     def query(self, query: str, *, optimize: bool = True) -> Iterator[Object]:

{dissect_database-1.1.dev10 → dissect_database-1.1.dev11}/dissect/database/ese/ntds/objects/object.py

@@ -64,10 +64,15 @@ class Object:
             db: The database instance associated with this object.
             record: The :class:`Record` instance representing this object.
         """
-        if (object_classes := _get_attribute(db, record, "objectClass")) and (
-            known_cls := cls.__known_classes__.get(object_classes[0])
-        ) is not None:
-            return known_cls(db, record)
+        try:
+            if (object_classes := _get_attribute(db, record, "objectClass")) and (
+                known_cls := cls.__known_classes__.get(object_classes[0])
+            ) is not None:
+                return known_cls(db, record)
+        except ValueError:
+            # Resolving the objectClass values can fail if the schema is not loaded yet (or is malformed)
+            # Fallback to a generic Object in that case
+            pass
 
         return cls(db, record)
 
@@ -268,4 +273,4 @@ def _get_attribute(db: Database, record: Record, name: str, *, raw: bool = False
     if raw:
         return value
 
-    return decode_value(db, name, value)
+    return decode_value(db, schema, value)

{dissect_database-1.1.dev10 → dissect_database-1.1.dev11}/dissect/database/ese/ntds/query.py

@@ -83,7 +83,7 @@ class Query:
                 raise NotImplementedError("Wildcards in the middle or start of the value are not yet supported")
         else:
             # Exact match query
-            encoded_value = encode_value(self.db, filter.attribute, filter.value)
+            encoded_value = encode_value(self.db, schema, filter.value)
             yield from index.cursor().find_all(**{schema.column: encoded_value})
 
     def _process_and_operation(self, filter: SearchFilter, records: list[Record] | None) -> Iterator[Record]:
@@ -132,12 +132,11 @@ class Query:
         Yields:
             Records that match the filter criteria.
         """
-        encoded_value = encode_value(self.db, filter.attribute, filter.value)
-        schema = self.db.data.schema.lookup_attribute(name=filter.attribute)
-
-        if schema is None:
+        if (schema := self.db.data.schema.lookup_attribute(name=filter.attribute)) is None:
             return
 
+        encoded_value = encode_value(self.db, schema, filter.value)
+
         has_wildcard = "*" in filter.value
         wildcard_prefix = filter.value.replace("*", "").lower() if has_wildcard else None
 

{dissect_database-1.1.dev10 → dissect_database-1.1.dev11}/dissect/database/ese/ntds/schema.py

@@ -1,13 +1,11 @@
 from __future__ import annotations
 
+from io import BytesIO
 from typing import TYPE_CHECKING, NamedTuple
 
-from dissect.database.ese.ntds.objects.object import Object
-from dissect.database.ese.ntds.util import OID_TO_TYPE, attrtyp_to_oid
+from dissect.database.ese.ntds.c_ds import c_ds
 
 if TYPE_CHECKING:
-    from collections.abc import Iterator
-
     from dissect.database.ese.ntds.database import Database
     from dissect.database.ese.ntds.util import SearchFlags
 
@@ -50,6 +48,8 @@ BOOTSTRAP_ATTRIBUTES = [
     ("searchFlags", 131406, 0x00080009, True),  # ATTj131406
     # Class schema
     ("governsID", 131094, 0x00080002, True),  # ATTc131094
+    # DSA attributes
+    ("dMDLocation", 131108, 0x00080001, True),  # ATTb131108
 ]
 
 # For convenience, bootstrap some common object classes
@@ -60,21 +60,63 @@ BOOTSTRAP_OBJECT_CLASSES = {
     "attributeSchema": 0x0003000E,
 }
 
+# These are fixed OID prefixes used in the schema
+# Reference: MSPrefixTable
+BOOTSTRAP_OID_PREFIXES = {
+    0: b"\x55\x04",
+    1: b"\x55\x06",
+    2: b"\x2a\x86\x48\x86\xf7\x14\x01\x02",
+    3: b"\x2a\x86\x48\x86\xf7\x14\x01\x03",
+    4: b"\x60\x86\x48\x01\x65\x02\x02\x01",
+    5: b"\x60\x86\x48\x01\x65\x02\x02\x03",
+    6: b"\x60\x86\x48\x01\x65\x02\x01\x05",
+    7: b"\x60\x86\x48\x01\x65\x02\x01\x04",
+    8: b"\x55\x05",
+    9: b"\x2a\x86\x48\x86\xf7\x14\x01\x04",
+    10: b"\x2a\x86\x48\x86\xf7\x14\x01\x05",
+    11: b"\x2a\x86\x48\x86\xf7\x14\x01\x04\x82\x04",
+    12: b"\x2a\x86\x48\x86\xf7\x14\x01\x05\x38",
+    13: b"\x2a\x86\x48\x86\xf7\x14\x01\x04\x82\x06",
+    14: b"\x2a\x86\x48\x86\xf7\x14\x01\x05\x39",
+    15: b"\x2a\x86\x48\x86\xf7\x14\x01\x04\x82\x07",
+    16: b"\x2a\x86\x48\x86\xf7\x14\x01\x05\x3a",
+    17: b"\x2a\x86\x48\x86\xf7\x14\x01\x05\x49",
+    18: b"\x2a\x86\x48\x86\xf7\x14\x01\x04\x82\x31",
+    19: b"\x09\x92\x26\x89\x93\xf2\x2c\x64",
+    20: b"\x60\x86\x48\x01\x86\xf8\x42\x03",
+    21: b"\x09\x92\x26\x89\x93\xf2\x2c\x64\x01",
+    22: b"\x60\x86\x48\x01\x86\xf8\x42\x03\x01",
+    23: b"\x2a\x86\x48\x86\xf7\x14\x01\x05\xb6\x58",
+    24: b"\x55\x15",
+    25: b"\x55\x12",
+    26: b"\x55\x14",
+    27: b"\x2b\x06\x01\x04\x01\x8b\x3a\x65\x77",
+    28: b"\x60\x86\x48\x01\x86\xf8\x42\x03\x02",
+    29: b"\x2b\x06\x01\x04\x01\x81\x7a\x01",
+    30: b"\x2a\x86\x48\x86\xf7\x0d\x01\x09",
+    31: b"\x09\x92\x26\x89\x93\xf2\x2c\x64\x04",
+    32: b"\x2a\x86\x48\x86\xf7\x14\x01\x06\x17",
+    33: b"\x2a\x86\x48\x86\xf7\x14\x01\x06\x12\x01",
+    34: b"\x2a\x86\x48\x86\xf7\x14\x01\x06\x12\x02",
+    35: b"\x2a\x86\x48\x86\xf7\x14\x01\x06\x0d\x03",
+    36: b"\x2a\x86\x48\x86\xf7\x14\x01\x06\x0d\x04",
+    37: b"\x2b\x06\x01\x01\x01\x01",
+    38: b"\x2b\x06\x01\x01\x01\x02",
+}
+
 
 class ClassEntry(NamedTuple):
     dnt: int
-    oid: str
     id: int
     name: str
 
 
 class AttributeEntry(NamedTuple):
     dnt: int
-    oid: str
     id: int
     name: str
     column: str
-    type: str
+    syntax: int
     om_syntax: int | None
     om_object_class: bytes | None
     is_single_valued: bool
@@ -91,7 +133,6 @@ class Schema:
     def __init__(self):
         # Combined indices
         self._dnt_index: dict[int, ClassEntry | AttributeEntry] = {}
-        self._oid_index: dict[str, ClassEntry | AttributeEntry] = {}
         self._attrtyp_index: dict[int, ClassEntry | AttributeEntry] = {}
 
         # Attribute specific indices
@@ -104,16 +145,23 @@ class Schema:
         self._class_id_index: dict[int, ClassEntry] = {}
         self._class_name_index: dict[str, ClassEntry] = {}
 
+        # OID prefixes
+        self._oid_idx_to_prefix_index: dict[int, str] = {
+            idx: decode_oid(prefix) for idx, prefix in BOOTSTRAP_OID_PREFIXES.items()
+        }
+        self._oid_prefix_to_idx_index: dict[str, int] = {
+            prefix: idx for idx, prefix in self._oid_idx_to_prefix_index.items()
+        }
+
         # Bootstrap fixed database columns (these do not exist in the schema)
-        for ldap_name, column_name, syntax in BOOTSTRAP_COLUMNS:
+        for ldap_name, column_name, attrtyp in BOOTSTRAP_COLUMNS:
             self._add(
                 AttributeEntry(
                     dnt=-1,
-                    oid="",
                     id=-1,
                     name=ldap_name,
                     column=column_name,
-                    type=attrtyp_to_oid(syntax),
+                    syntax=attrtyp & 0xFF,
                     om_syntax=None,
                     om_object_class=None,
                     is_single_valued=True,
@@ -123,12 +171,12 @@ class Schema:
             )
 
         # Bootstrap initial attributes
-        for name, id, attribute_syntax, is_single_valued in BOOTSTRAP_ATTRIBUTES:
+        for name, id, attrtyp, is_single_valued in BOOTSTRAP_ATTRIBUTES:
             self._add_attribute(
                 dnt=-1,
                 id=id,
                 name=name,
-                syntax=attribute_syntax,
+                syntax=attrtyp,
                 om_syntax=None,
                 om_object_class=None,
                 is_single_valued=is_single_valued,
@@ -151,48 +199,48 @@ class Schema:
             db: The database instance to load the schema from.
         """
 
-        def _iter(id: int) -> Iterator[Object]:
-            # Use the ATTc0 (objectClass) index to iterate over all objects of the given objectClass
-            # TODO: In the future, maybe use `DataTable._get_index`, but that's not fully implemented yet
-            cursor = db.data.table.index("INDEX_00000000").cursor()
-            end = cursor.seek([id + 1]).record()
-
-            cursor.reset()
-            cursor.seek([id])
-
-            record = cursor.record()
-            while record is not None and record != end:
-                yield Object.from_record(db, record)
-                record = cursor.next()
+        # Load the schema entries from the DMD object
+        # This _should_ have all the attribute and class schema entries
+        # We used to perform an index search on objectClass (ATTc0, INDEX_00000000), but apparently
+        # not all databases have this index
+        dmd = db.data.dmd()
 
         # We bootstrapped these earlier
         attribute_schema = self.lookup_class(name="attributeSchema")
         class_schema = self.lookup_class(name="classSchema")
 
-        for obj in _iter(attribute_schema.id):
-            self._add_attribute(
-                dnt=obj.dnt,
-                id=obj.get("attributeID", raw=True),
-                name=obj.get("lDAPDisplayName"),
-                syntax=obj.get("attributeSyntax", raw=True),
-                om_syntax=obj.get("oMSyntax"),
-                om_object_class=obj.get("oMObjectClass"),
-                is_single_valued=obj.get("isSingleValued"),
-                link_id=obj.get("linkId"),
-                search_flags=obj.get("searchFlags"),
-            )
+        for obj in dmd.children():
+            # Get as raw to avoid decoding the attribute and class schema entries before we know which is which
+            classes = obj.get("objectClass", raw=True)
+            if attribute_schema.id in classes:
+                self._add_attribute(
+                    dnt=obj.dnt,
+                    id=obj.get("attributeID", raw=True),
+                    name=obj.get("lDAPDisplayName"),
+                    syntax=obj.get("attributeSyntax", raw=True),
+                    om_syntax=obj.get("oMSyntax"),
+                    om_object_class=obj.get("oMObjectClass"),
+                    is_single_valued=obj.get("isSingleValued"),
+                    link_id=obj.get("linkId"),
+                    search_flags=obj.get("searchFlags"),
+                )
 
-        for obj in _iter(class_schema.id):
-            self._add_class(
-                dnt=obj.dnt,
-                id=obj.get("governsID", raw=True),
-                name=obj.get("lDAPDisplayName"),
-            )
+            elif class_schema.id in classes:
+                self._add_class(
+                    dnt=obj.dnt,
+                    id=obj.get("governsID", raw=True),
+                    name=obj.get("lDAPDisplayName"),
+                )
+
+        # Load user-defined OID prefixes
+        if (prefix_map := db.data.dmd().get("prefixMap")) is not None:
+            self._oid_idx_to_prefix_index.update(parse_prefix_map(prefix_map))
+            # Rebuild the reverse index
+            self._oid_prefix_to_idx_index = {prefix: idx for idx, prefix in self._oid_idx_to_prefix_index.items()}
 
     def _add_class(self, dnt: int, id: int, name: str) -> None:
         entry = ClassEntry(
             dnt=dnt,
-            oid=attrtyp_to_oid(id),
             id=id,
             name=name,
         )
@@ -210,14 +258,12 @@ class Schema:
         link_id: int | None,
         search_flags: SearchFlags | None,
     ) -> None:
-        type_oid = attrtyp_to_oid(syntax)
         entry = AttributeEntry(
             dnt=dnt,
-            oid=attrtyp_to_oid(id),
             id=id,
             name=name,
-            column=f"ATT{OID_TO_TYPE[type_oid]}{id}",
-            type=type_oid,
+            column=f"ATT{chr(ord('a') + (syntax & 0xFFFF))}{id}",
+            syntax=syntax & 0xFF,
             om_syntax=om_syntax,
             om_object_class=om_object_class,
             is_single_valued=is_single_valued,
@@ -229,8 +275,6 @@ class Schema:
     def _add(self, entry: ClassEntry | AttributeEntry) -> None:
         if entry.dnt != -1:
             self._dnt_index[entry.dnt] = entry
-        if entry.oid != "":
-            self._oid_index[entry.oid] = entry
         if entry.id != -1:
             self._attrtyp_index[entry.id] = entry
 
@@ -311,6 +355,62 @@ class Schema:
 
         return None
 
+    def lookup_oid(self, oid: str) -> ClassEntry | AttributeEntry | None:
+        """Lookup a schema entry by OID.
+
+        Args:
+            oid: The OID to look up.
+
+        Returns:
+            The matching schema entry or ``None`` if not found.
+        """
+        parts = oid.split(".")
+        if len(parts) < 2:
+            return None
+
+        long_id = 0
+        prefix_length = 0
+        if len(parts) > 2 and int(parts[-2]) & 0x80:
+            prefix_length = len(parts) - 2
+            long_id = int(parts[-3]) >> 7
+        else:
+            prefix_length = len(parts) - 1
+
+        prefix = ".".join(parts[:prefix_length])
+        if (idx := self._oid_prefix_to_idx_index.get(prefix)) is None:
+            return None
+
+        attrtyp = idx << 16
+        if len(parts) == prefix_length + 2:
+            attrtyp += (int(parts[-2]) & 0x7F) << 7
+            if long_id:
+                attrtyp |= 0x8000
+
+        attrtyp += int(parts[-1])
+        return self._attrtyp_index.get(attrtyp)
+
+    def attrtyp_to_oid(self, attrtyp: int) -> str | None:
+        """Convert an ATTRTYP integer value to an OID string.
+
+        Args:
+            attrtyp: The ATTRTYP integer value to convert.
+
+        Returns:
+            The corresponding OID string or ``None`` if the ATTRTYP does not correspond to a valid OID.
+        """
+        if (prefix := self._oid_idx_to_prefix_index.get(attrtyp >> 16)) is None:
+            return None
+
+        parts = [prefix]
+
+        if attrtyp & 0xFFFF < 0x80:
+            parts.append(str(attrtyp & 0xFF))
+        else:
+            parts.append(str(((attrtyp & 0xFF80) >> 7) | 0x80))
+            parts.append(str(attrtyp & 0x7F))
+
+        return ".".join(parts)
+
     def lookup(
         self,
         *,
@@ -338,7 +438,7 @@ class Schema:
             return self._dnt_index.get(dnt)
 
         if oid is not None:
-            return self._oid_index.get(oid)
+            return self.lookup_oid(oid)
 
         if attrtyp is not None:
             return self._attrtyp_index.get(attrtyp)
@@ -348,3 +448,51 @@ class Schema:
             return self._class_name_index.get(name) or self._attribute_name_index.get(name)
 
         return None
+
+
+def parse_prefix_map(buf: bytes) -> dict[int, str]:
+    """Parse a prefix map.
+
+    Args:
+        buf: The buffer containing the prefix map data.
+    """
+    result = {}
+
+    fh = BytesIO(buf)
+    c_ds.uint32(fh)  # Number of prefixes
+    total_size = c_ds.uint32(fh)  # Total size
+
+    while fh.tell() < total_size:
+        index = c_ds.uint16(fh)
+        prefix_length = c_ds.uint16(fh)
+        prefix = fh.read(prefix_length)
+
+        result[index] = decode_oid(prefix)
+
+    return result
+
+
+def decode_oid(buf: bytes) -> str:
+    """Decode a BER encoded OID.
+
+    Args:
+        buf: The buffer containing the BER encoded OID.
+    """
+    values = [*divmod(buf[0], 40)]
+
+    idx = 1
+    while idx < len(buf):
+        value = buf[idx] & 0x7F
+        while buf[idx] & 0x80:
+            value <<= 7
+            idx += 1
+
+            if idx >= len(buf):
+                break
+
+            value |= buf[idx] & 0x7F
+
+        values.append(value)
+        idx += 1
+
+    return ".".join(map(str, values))