pyfield-esm 0.3.0__py3-none-any.whl

pyfield/__init__.py

@@ -0,0 +1,58 @@
+"""PyField: Bethesda game worlds as structured, persistent, validated content.
+
+Stable public API (covered by tests/test_public_api.py; see docs/api.md):
+
+    open_plugin, create_plugin, open_load_order
+    Plugin, LoadOrder, Query, Transaction, FormKey, Unresolved
+    RecordNode, GroupNode, Subrecord, TypedRecord
+    validate_plugin, Report, diff_plugins, Diff
+    games (adapter registry), Capability/Status
+    errors: PyFieldError and subclasses
+
+Everything else (including attributes starting with "_") may change between minor versions.
+"""
+from .capabilities import Capability, CapabilitySet, Status
+from .core.formids import FormKey, Unresolved
+from .core.loadorder import LoadOrder, Query
+from .core.nodes import GroupNode, RecordNode
+from .core.plugin import Plugin
+from .core.subrecords import Subrecord
+from .diff import Diff, diff_plugins
+from .edit import Transaction
+from .errors import (FormatError, PyFieldError, ResolutionError, SchemaError, TransactionError,
+                     UnsupportedOperation, ValidationError)
+from .schema.records import TypedRecord
+from .validate import Report, validate_plugin
+from . import games
+
+__version__ = '0.3.0'
+
+__all__ = [
+    'open_plugin', 'create_plugin', 'open_load_order',
+    'Plugin', 'LoadOrder', 'Query', 'Transaction', 'FormKey', 'Unresolved',
+    'RecordNode', 'GroupNode', 'Subrecord', 'TypedRecord',
+    'validate_plugin', 'Report', 'diff_plugins', 'Diff',
+    'games', 'Capability', 'CapabilitySet', 'Status',
+    'PyFieldError', 'FormatError', 'ResolutionError', 'SchemaError', 'TransactionError',
+    'UnsupportedOperation', 'ValidationError', '__version__',
+]
+
+
+def open_plugin(path, game=None, **kwargs) -> Plugin:
+    """Open one plugin file; the game is detected from its header unless given."""
+    return Plugin.open(path, game, **kwargs)
+
+
+def create_plugin(name: str, game: str, masters=(), *, flags: int = 0, author: str = '',
+                  description: str = '', master_kinds: dict | None = None) -> Plugin:
+    """A new empty plugin. Pass master_kinds (name -> 'full'|'medium'|'small') when the
+    masters are not on disk next to where it will be saved."""
+    plugin = Plugin.create(name, game, masters, flags=flags, author=author, description=description)
+    if master_kinds:
+        plugin.set_master_kinds(master_kinds)
+    return plugin
+
+
+def open_load_order(data_dir, plugins, game=None, with_masters: bool = True) -> LoadOrder:
+    """Open plugins from a data folder (masters added first, recursively)."""
+    return LoadOrder.open(data_dir, plugins, game, with_masters)

pyfield/assets/ba2.py

@@ -0,0 +1,183 @@
+"""BA2 archives (Fallout 4 / Starfield "BTDX"): read GNRL and DX10, write uncompressed GNRL.
+
+Reader adapted from the workspace's proven bolero/ScreenSFSE `ba2.py` (GNRL v1-3, DX10 with
+LZ4 block or zlib chunks, DDS header reconstruction for textures).
+
+The writer is for tests and an experimental fallback only: production archives are built
+with the game's Archive2.exe (see pyfield.assets.pack). Member names are stored as given
+(normalized to backslashes); name/directory hashes are zero, which PyField's reader does not
+need but the game does — never ship archives from this writer.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+import struct
+import zlib
+
+from ..errors import FormatError
+
+
+def normalize(name: str) -> str:
+    """Archive member key: forward slashes, lower case, no leading 'data/'."""
+    name = name.replace('\\', '/').lstrip('/').lower()
+    return name[5:] if name.startswith('data/') else name
+
+
+def lz4_block(src: bytes, size: int) -> bytes:
+    """Decode one LZ4 block (no frame header) to `size` bytes."""
+    out = bytearray()
+    i, n = 0, len(src)
+    while i < n:
+        token = src[i]
+        i += 1
+        lit = token >> 4
+        if lit == 15:
+            while True:
+                b = src[i]
+                i += 1
+                lit += b
+                if b != 255:
+                    break
+        out += src[i:i + lit]
+        i += lit
+        if i >= n:
+            break
+        offset = src[i] | (src[i + 1] << 8)
+        i += 2
+        ml = token & 15
+        if ml == 15:
+            while True:
+                b = src[i]
+                i += 1
+                ml += b
+                if b != 255:
+                    break
+        ml += 4
+        start = len(out) - offset
+        if offset >= ml:
+            out += out[start:start + ml]
+        else:
+            for k in range(ml):
+                out.append(out[start + k])
+    if len(out) != size:
+        raise FormatError(f'LZ4 block decoded to {len(out)} bytes, expected {size}')
+    return bytes(out)
+
+
+BLOCK_BYTES = {70: 8, 71: 8, 72: 8, 79: 8, 80: 8, 81: 8, 73: 16, 74: 16, 75: 16, 76: 16, 77: 16, 78: 16,
+               82: 16, 83: 16, 84: 16, 94: 16, 95: 16, 96: 16, 97: 16, 98: 16, 99: 16}
+PIXEL_BYTES = {28: 4, 29: 4, 87: 4, 91: 4, 61: 1, 49: 2, 10: 8, 2: 16, 24: 4, 56: 2}
+
+
+def dds_header(width, height, mips, dxgi, cubemap) -> bytes:
+    if dxgi in BLOCK_BYTES:
+        pitch = max(1, (width + 3) // 4) * BLOCK_BYTES[dxgi] * max(1, (height + 3) // 4)
+        flags = 0x1 | 0x2 | 0x4 | 0x1000 | 0x20000 | 0x80000
+    else:
+        pitch = width * PIXEL_BYTES.get(dxgi, 4)
+        flags = 0x1 | 0x2 | 0x4 | 0x1000 | 0x20000 | 0x8
+    caps = 0x1000 | (0x400008 if mips > 1 else 0) | (0x8 if cubemap else 0)
+    caps2 = 0xFE00 if cubemap else 0
+    head = struct.pack('<4sIIIIIII44x', b'DDS ', 124, flags, height, width, pitch, 0, mips)
+    pf = struct.pack('<II4s20x', 32, 0x4, b'DX10')
+    head += pf + struct.pack('<IIII4x', caps, caps2, 0, 0)
+    return head + struct.pack('<IIIII', dxgi, 3, 0x4 if cubemap else 0, 1, 0)
+
+
+@dataclass
+class Member:
+    name: str                 # normalized key
+    stored_name: str          # as written in the archive's name table
+    kind: str                 # 'GNRL' or 'DX10'
+    info: tuple
+
+
+class Archive:
+    """One BA2 file. `members` maps normalized names to Member."""
+
+    def __init__(self, path):
+        self.path = Path(path)
+        self.members: dict[str, Member] = {}
+        with open(self.path, 'rb') as f:
+            head = f.read(36)
+            if len(head) < 24 or head[:4] != b'BTDX':
+                raise FormatError(f'{self.path.name}: not a BA2 archive')
+            _, self.version, kind, count, table = struct.unpack_from('<4sI4sIQ', head)
+            self.kind = kind.decode('latin-1')
+            header = 24 + (8 if self.version >= 2 else 0) + (4 if self.version == 3 else 0)
+            self.compression = struct.unpack_from('<I', head, 32)[0] if self.version == 3 else 0
+            f.seek(header)
+            records = []
+            if self.kind == 'GNRL':
+                for _ in range(count):
+                    _, _, _, _, offset, packed, size, _ = struct.unpack('<I4sIIQIII', f.read(36))
+                    records.append(('GNRL', (offset, packed, size)))
+            elif self.kind == 'DX10':
+                for _ in range(count):
+                    _, _, _, _, nchunks, _, h, w, mips, fmt, cube, _ = struct.unpack('<I4sIBBHHHBBBB', f.read(24))
+                    chunks = [struct.unpack('<QIIHHI', f.read(24)) for _ in range(nchunks)]
+                    records.append(('DX10', ((w, h, mips, fmt, cube), chunks)))
+            else:
+                raise FormatError(f'{self.path.name}: unsupported BA2 type {self.kind}')
+            f.seek(table)
+            for kind_, info in records:
+                n, = struct.unpack('<H', f.read(2))
+                stored = f.read(n).decode('utf-8', 'replace')
+                key = normalize(stored)
+                self.members[key] = Member(key, stored, kind_, info)
+
+    def __contains__(self, name):
+        return normalize(name) in self.members
+
+    def names(self) -> list[str]:
+        return sorted(self.members)
+
+    def _unpack(self, f, offset, packed, size):
+        f.seek(offset)
+        if not packed:
+            return f.read(size)
+        data = f.read(packed)
+        if self.compression == 3:
+            return lz4_block(data, size)
+        return zlib.decompress(data)
+
+    def read(self, name: str) -> bytes:
+        m = self.members[normalize(name)]
+        with open(self.path, 'rb') as f:
+            if m.kind == 'GNRL':
+                return self._unpack(f, *m.info)
+            (w, h, mips, fmt, cube), chunks = m.info
+            body = b''.join(self._unpack(f, off, packed, size) for off, packed, size, _, _, _ in chunks)
+            return dds_header(w, h, mips, fmt, cube) + body
+
+    def __repr__(self):
+        return f'<Archive {self.path.name} {self.kind} {len(self.members)} members>'
+
+
+def write_gnrl(path, files: dict[str, bytes], version: int = 2):
+    """Experimental: uncompressed GNRL archive (tests/fallback only; hashes are zero)."""
+    names = sorted(files)
+    header_size = 24 + (8 if version >= 2 else 0) + (4 if version == 3 else 0)
+    data_offset = header_size + 36 * len(names)
+    entries, blobs, offset = [], [], data_offset
+    for name in names:
+        data = files[name]
+        ext = (name.rsplit('.', 1)[-1] if '.' in name else '').encode('latin-1')[:4].ljust(4, b'\0')
+        entries.append(struct.pack('<I4sIIQIII', 0, ext, 0, 0, offset, 0, len(data), 0xBAADF00D))
+        blobs.append(data)
+        offset += len(data)
+    table = offset
+    out = bytearray(struct.pack('<4sI4sIQ', b'BTDX', version, b'GNRL', len(names), table))
+    if version >= 2:
+        out += struct.pack('<II', 1, 0)
+    if version == 3:
+        out += struct.pack('<I', 0)
+    for e in entries:
+        out += e
+    for b in blobs:
+        out += b
+    for name in names:
+        stored = name.replace('/', '\\').encode('utf-8')
+        out += struct.pack('<H', len(stored)) + stored
+    Path(path).write_bytes(bytes(out))

pyfield/assets/deps.py

@@ -0,0 +1,179 @@
+"""Asset dependency discovery: records -> files -> files, every candidate verified.
+
+Strings are found by scanning (records: path-like text in any subrecord; NIF and other
+binaries: length-prefixed strings; JSON: every string value). A candidate becomes a
+dependency only if the AssetIndex resolves it to an existing file; everything else is
+reported as unresolved, never assumed. Adapter hooks add structured dependencies that are
+not plain paths (Starfield: Wwise events -> soundbank files).
+"""
+from __future__ import annotations
+
+from collections import deque
+from dataclasses import dataclass, field
+import json
+import re
+import struct
+
+from .index import AssetIndex
+
+PRINTABLE = re.compile(rb'[\x20-\x7e]{4,260}')
+PATHY = re.compile(r'[\\/]|\.[A-Za-z0-9]{2,7}$')
+SCANNED_EXTENSIONS = ('.nif', '.mat', '.json', '.txt', '.agx', '.rig', '.bgsm', '.bgem')
+
+
+@dataclass
+class AssetRef:
+    path: str                  # resolved, normalized asset path
+    source: str                # what referenced it ('STAT 01000800:MODL', 'meshes/x.nif', ...)
+    provider: str = ''         # 'archive:<file>' / 'loose:<file>'
+    category: str = ''         # owned | vanilla | foreign | loose
+
+
+@dataclass
+class AssetClosure:
+    files: dict = field(default_factory=dict)       # path -> AssetRef (first reference)
+    unresolved: list = field(default_factory=list)  # (source, text) that looked like paths but resolve nowhere
+    external: list = field(default_factory=list)    # (source, text) living in a compiled database (unverifiable)
+
+    def by_category(self) -> dict:
+        out = {}
+        for ref in self.files.values():
+            out.setdefault(ref.category, []).append(ref.path)
+        return {k: sorted(v) for k, v in out.items()}
+
+    def to_dict(self):
+        return {'files': {p: {'source': r.source, 'provider': r.provider, 'category': r.category}
+                          for p, r in sorted(self.files.items())},
+                'unresolved': self.unresolved, 'external_unverified': self.external,
+                'by_category': self.by_category()}
+
+
+def record_strings(fields) -> list[tuple[str, str]]:
+    """(subrecord signature, text) for path-like printable runs in a record's subrecords."""
+    out = []
+    for f in fields:
+        for m in PRINTABLE.finditer(f.data):
+            text = m.group(0).decode('latin-1').strip()
+            if PATHY.search(text):
+                out.append((f.signature, text))
+    return out
+
+
+def binary_strings(data: bytes) -> list[str]:
+    """Length-prefixed (u32 or u16) printable strings, as NIF and similar formats store them."""
+    out, n = set(), len(data)
+    for i in range(n - 6):
+        for width, fmt in ((4, '<I'), (2, '<H')):
+            length = struct.unpack_from(fmt, data, i)[0]
+            if 4 <= length <= 260 and i + width + length <= n:
+                chunk = data[i + width:i + width + length]
+                if chunk.isascii() and all(32 <= b < 127 for b in chunk.rstrip(b'\0')):
+                    text = chunk.rstrip(b'\0').decode('ascii')
+                    if PATHY.search(text) or '\\' in text:
+                        out.add(text)
+                    break
+    return sorted(out)
+
+
+def json_strings(data: bytes) -> list[str]:
+    try:
+        obj = json.loads(data)
+    except ValueError:
+        return [m.group(0).decode('latin-1') for m in PRINTABLE.finditer(data)]
+    out = []
+
+    def walk(o):
+        if isinstance(o, str):
+            out.append(o)
+        elif isinstance(o, dict):
+            for v in o.values():
+                walk(v)
+        elif isinstance(o, list):
+            for v in o:
+                walk(v)
+    walk(obj)
+    return out
+
+
+def file_strings(path: str, data: bytes) -> list[str]:
+    if path.endswith(('.mat', '.json')):
+        return json_strings(data)
+    if path.endswith('.txt'):
+        return [line.strip() for line in data.decode('utf-8', 'replace').splitlines() if line.strip()]
+    return binary_strings(data)
+
+
+class DependencyFinder:
+    def __init__(self, assets: AssetIndex, record_hooks: dict | None = None, source_plugin: str | None = None):
+        """record_hooks: {signature: fn(view, assets, cache) -> [paths]} for structured references."""
+        self.assets = assets
+        self.hooks = record_hooks or {}
+        self.source_plugin = source_plugin
+        self._cache = {}
+
+    def _ref(self, path, source):
+        prov = self.assets.provider(path)
+        return AssetRef(path, source, str(prov), self.assets.classify(prov, self.source_plugin))
+
+    def _unresolved(self, closure, source, text):
+        if not re.search(r'\.(nif|mat|dds|wem|bnk|mesh|af|afx|rig|pex|swf)$', text, re.I):
+            return
+        if self.assets.rules.is_unverifiable(text):
+            closure.external.append((source, text))
+        else:
+            closure.unresolved.append((source, text))
+
+    def for_records(self, views, follow: bool = True) -> AssetClosure:
+        closure = AssetClosure()
+        queue = deque()
+        for view in views:
+            label = f'{view.signature} {view.form_key}'
+            found = set()
+            for sig, text in record_strings(view.node.fields):
+                resolved = self.assets.resolve(text)
+                if resolved:
+                    found.add((resolved, f'{label}:{sig}'))
+                else:
+                    self._unresolved(closure, f'{label}:{sig}', text)
+            hook = self.hooks.get(view.signature)
+            if hook:
+                for path in hook(view, self.assets, self._cache):
+                    found.add((path, f'{label}:{view.signature}-hook'))
+            for path, source in sorted(found):
+                if path not in closure.files:
+                    closure.files[path] = self._ref(path, source)
+                    queue.append(path)
+        if follow:
+            self._follow(closure, queue)
+        return closure
+
+    def for_files(self, paths) -> AssetClosure:
+        closure = AssetClosure()
+        queue = deque()
+        for p in paths:
+            resolved = self.assets.resolve(p)
+            if resolved is None:
+                closure.unresolved.append(('request', p))
+                continue
+            closure.files[resolved] = self._ref(resolved, 'request')
+            queue.append(resolved)
+        self._follow(closure, queue)
+        return closure
+
+    def _follow(self, closure: AssetClosure, queue: deque):
+        while queue:
+            path = queue.popleft()
+            if not path.endswith(SCANNED_EXTENSIONS) or path.startswith(self.assets.rules.opaque_prefixes):
+                continue
+            try:
+                data = self.assets.read(path)
+            except FileNotFoundError:
+                continue
+            for text in file_strings(path, data):
+                resolved = self.assets.resolve(text)
+                if resolved is None:
+                    self._unresolved(closure, path, text)
+                    continue
+                if resolved != path and resolved not in closure.files:
+                    closure.files[resolved] = self._ref(resolved, path)
+                    queue.append(resolved)

pyfield/assets/index.py

@@ -0,0 +1,140 @@
+"""Asset index: which file (loose or in which archive) provides an asset path.
+
+Precedence follows the game: archives in load order (base archives first, then each
+plugin's `<plugin> - *.ba2`), later ones overriding earlier; loose files override archives.
+Resolution of a referenced string is *verified*: a candidate path counts only if a provider
+exists for it. Game knowledge (asset roots, default roots per extension, implicit
+extensions, official plugin names) comes from an `AssetRules` object supplied by the adapter.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+import fnmatch
+from pathlib import Path
+
+from .ba2 import Archive, normalize
+
+
+@dataclass(frozen=True)
+class AssetRules:
+    roots: tuple[str, ...]                          # top-level asset folders, lower case
+    extension_roots: dict = field(default_factory=dict)   # '.nif' -> 'meshes/'
+    implicit: tuple = ()                            # (root, extension) tried for extension-less names
+    base_archives: tuple[str, ...] = ()             # glob patterns of base-game archives
+    official_plugins: tuple[str, ...] = ()          # glob patterns (lower case) of first-party plugins
+    texture_extensions: tuple[str, ...] = ('.dds',)
+    unverifiable: tuple = ()                        # (root, extension): may live in a compiled database
+    opaque_prefixes: tuple[str, ...] = ()           # files not scanned generically (handled by hooks)
+
+    def is_unverifiable(self, text: str) -> bool:
+        return any(c.startswith(root) and c.endswith(ext) for c in self.candidates(text)
+                   for root, ext in self.unverifiable)
+
+    def candidates(self, text: str) -> list[str]:
+        path = normalize(text.strip().strip('\0'))
+        if not path:
+            return []
+        out = [path]
+        ext = Path(path).suffix.lower()
+        first = path.split('/', 1)[0]
+        if ext and first not in self.roots and ext in self.extension_roots:
+            out.append(self.extension_roots[ext] + path)
+        if not ext:
+            for root, implicit_ext in self.implicit:
+                out.append((path if first == root.rstrip('/') else root + path) + implicit_ext)
+        return list(dict.fromkeys(out))
+
+    def is_official(self, plugin_name: str) -> bool:
+        name = plugin_name.lower()
+        return any(fnmatch.fnmatchcase(name, pat) for pat in self.official_plugins)
+
+
+@dataclass(frozen=True)
+class Provider:
+    kind: str                  # 'loose' | 'archive'
+    path: Path                 # loose file or archive file
+    owner: str | None          # plugin stem owning the archive (None for base archives / loose)
+
+    def __str__(self):
+        return f'{self.kind}:{self.path.name}'
+
+
+class AssetIndex:
+    def __init__(self, data_dir, rules: AssetRules, plugins=(), include_loose: bool = True):
+        """plugins: plugin file names in load order (their `<stem> - *.ba2` archives are indexed)."""
+        self.data_dir = Path(data_dir)
+        self.rules = rules
+        self.include_loose = include_loose
+        self.archives: list[Archive] = []
+        self._owner: dict[Path, str | None] = {}
+        self._entries: dict[str, Archive] = {}
+        files = sorted(p for p in self.data_dir.iterdir() if p.suffix.lower() == '.ba2') \
+            if self.data_dir.is_dir() else []
+        base = [p for p in files if any(fnmatch.fnmatchcase(p.name.lower(), g.lower()) for g in rules.base_archives)]
+        for p in base:
+            self._add(p, None)
+        for plugin in plugins:
+            stem = Path(plugin).stem.lower()
+            for p in files:
+                if p not in base and p.name.lower().startswith(stem + ' - '):
+                    self._add(p, Path(plugin).stem)
+
+    def _add(self, path: Path, owner):
+        arc = Archive(path)
+        self.archives.append(arc)
+        self._owner[arc.path] = owner
+        for name in arc.members:
+            self._entries[name] = arc
+
+    # -- lookup -----------------------------------------------------------------------------
+    def _loose(self, name: str) -> Path | None:
+        if not self.include_loose:
+            return None
+        p = self.data_dir / name
+        return p if p.is_file() else None
+
+    def provider(self, name: str) -> Provider | None:
+        name = normalize(name)
+        loose = self._loose(name)
+        if loose is not None:
+            return Provider('loose', loose, None)
+        arc = self._entries.get(name)
+        if arc is not None:
+            return Provider('archive', arc.path, self._owner[arc.path])
+        return None
+
+    def exists(self, name: str) -> bool:
+        return self.provider(name) is not None
+
+    def resolve(self, text: str) -> str | None:
+        """The asset path a referenced string denotes, if a provider exists for it."""
+        for cand in self.rules.candidates(text):
+            if self.exists(cand):
+                return cand
+        return None
+
+    def read(self, name: str) -> bytes:
+        name = normalize(name)
+        loose = self._loose(name)
+        if loose is not None:
+            return loose.read_bytes()
+        arc = self._entries.get(name)
+        if arc is None:
+            raise FileNotFoundError(name)
+        return arc.read(name)
+
+    def names(self, prefix: str = '') -> list[str]:
+        prefix = normalize(prefix)
+        return sorted(n for n in self._entries if n.startswith(prefix))
+
+    def classify(self, provider: Provider, source_plugin: str | None = None) -> str:
+        """owned (by source_plugin) | vanilla | foreign | loose."""
+        if provider.kind == 'loose':
+            return 'loose'
+        if provider.owner is None:
+            return 'vanilla'
+        if source_plugin and provider.owner.lower() == Path(source_plugin).stem.lower():
+            return 'owned'
+        if self.rules.is_official(provider.owner + '.esm'):
+            return 'vanilla'
+        return 'foreign'