wscodec 0.1.0

package/structs.mjs

@@ -0,0 +1,125 @@
+/**
+ * StructValue + STRUCT_HANDLERS registry.
+ *
+ * Soulmask is UE 4.27 so "core" structs (Vector etc.) use 32-bit floats.
+ * Known struct names read directly as binary; unknown struct names fall
+ * through to a nested property stream (handled in properties.mjs, not here
+ * — StructValue.read is supplied a `streamReader` callback to avoid a
+ * load-order cycle).
+ */
+
+import { FGuid } from './primitives.mjs';
+
+// Binary struct handlers. Each entry has read(cursor) → plain object and
+// write(writer, plainObject). The plain object is what callers see as
+// `structValue.value` when the struct is one of these known shapes.
+export const STRUCT_HANDLERS = {
+  Vector:      { read: c => ({ x: c.readFloat32(), y: c.readFloat32(), z: c.readFloat32() }),
+                 write: (w, v) => { w.writeFloat32(v.x); w.writeFloat32(v.y); w.writeFloat32(v.z); } },
+  Vector2D:    { read: c => ({ x: c.readFloat32(), y: c.readFloat32() }),
+                 write: (w, v) => { w.writeFloat32(v.x); w.writeFloat32(v.y); } },
+  Vector4:     { read: c => ({ x: c.readFloat32(), y: c.readFloat32(), z: c.readFloat32(), w: c.readFloat32() }),
+                 write: (w, v) => { w.writeFloat32(v.x); w.writeFloat32(v.y); w.writeFloat32(v.z); w.writeFloat32(v.w); } },
+  Rotator:     { read: c => ({ pitch: c.readFloat32(), yaw: c.readFloat32(), roll: c.readFloat32() }),
+                 write: (w, v) => { w.writeFloat32(v.pitch); w.writeFloat32(v.yaw); w.writeFloat32(v.roll); } },
+  Quat:        { read: c => ({ x: c.readFloat32(), y: c.readFloat32(), z: c.readFloat32(), w: c.readFloat32() }),
+                 write: (w, v) => { w.writeFloat32(v.x); w.writeFloat32(v.y); w.writeFloat32(v.z); w.writeFloat32(v.w); } },
+  Color:       { read: c => ({ b: c.readUint8(), g: c.readUint8(), r: c.readUint8(), a: c.readUint8() }),
+                 write: (w, v) => { w.writeUint8(v.b); w.writeUint8(v.g); w.writeUint8(v.r); w.writeUint8(v.a); } },
+  LinearColor: { read: c => ({ r: c.readFloat32(), g: c.readFloat32(), b: c.readFloat32(), a: c.readFloat32() }),
+                 write: (w, v) => { w.writeFloat32(v.r); w.writeFloat32(v.g); w.writeFloat32(v.b); w.writeFloat32(v.a); } },
+  Guid:        { read: c => FGuid.read(c).value,
+                 write: (w, v) => new FGuid(v).write(w) },
+  DateTime:    { read: c => c.readInt64().toString(),
+                 write: (w, v) => w.writeInt64(v) },
+  Timespan:    { read: c => c.readInt64().toString(),
+                 write: (w, v) => w.writeInt64(v) },
+  IntPoint:    { read: c => ({ x: c.readInt32(), y: c.readInt32() }),
+                 write: (w, v) => { w.writeInt32(v.x); w.writeInt32(v.y); } },
+  IntVector:   { read: c => ({ x: c.readInt32(), y: c.readInt32(), z: c.readInt32() }),
+                 write: (w, v) => { w.writeInt32(v.x); w.writeInt32(v.y); w.writeInt32(v.z); } },
+  Box:         { read: c => ({ min: STRUCT_HANDLERS.Vector.read(c), max: STRUCT_HANDLERS.Vector.read(c), isValid: c.readUint8() }),
+                 write: (w, v) => { STRUCT_HANDLERS.Vector.write(w, v.min); STRUCT_HANDLERS.Vector.write(w, v.max); w.writeUint8(v.isValid); } },
+  Sphere:      { read: c => ({ center: STRUCT_HANDLERS.Vector.read(c), radius: c.readFloat32() }),
+                 write: (w, v) => { STRUCT_HANDLERS.Vector.write(w, v.center); w.writeFloat32(v.radius); } },
+  Plane:       { read: c => ({ x: c.readFloat32(), y: c.readFloat32(), z: c.readFloat32(), w: c.readFloat32() }),
+                 write: (w, v) => { w.writeFloat32(v.x); w.writeFloat32(v.y); w.writeFloat32(v.z); w.writeFloat32(v.w); } },
+  Transform:   { read: c => ({ rotation: STRUCT_HANDLERS.Quat.read(c), translation: STRUCT_HANDLERS.Vector.read(c), scale3D: STRUCT_HANDLERS.Vector.read(c) }),
+                 write: (w, v) => { STRUCT_HANDLERS.Quat.write(w, v.rotation); STRUCT_HANDLERS.Vector.write(w, v.translation); STRUCT_HANDLERS.Vector.write(w, v.scale3D); } },
+};
+
+export class StructValue {
+  constructor(structName, { value = null, terminated = false, decodeError = null, opaqueTail = null } = {}) {
+    this._structName = structName;
+    this.value = value;
+    this.terminated = terminated;
+    if (decodeError) this._structDecodeError = decodeError;
+    if (opaqueTail) this._opaqueTail = opaqueTail;
+  }
+
+  get structName() { return this._structName; }
+  get isKnownBinary() { return STRUCT_HANDLERS[this._structName] != null; }
+
+  /**
+   * `streamReader(cursor, endOffset)` is `readPropertyStream` from
+   * properties.mjs, passed in to avoid a module cycle. Returns
+   * { properties, terminated, endPos }.
+   *
+   * `peekFn(cursor)` (optional) peeks at the cursor without advancing it
+   * and returns true if the bytes look like the start of a tagged
+   * PropertyTag (FString name with identifier-character ASCII content).
+   * When supplied and the wire bytes look tagged, the read switches to
+   * the property-stream path even for structs that have a known binary
+   * handler — Soulmask encodes known-binary structs (Transform, Box, ...)
+   * as TAGGED property streams inside Map struct values, which would
+   * otherwise be misread as raw 40-byte Transforms / etc. The decision
+   * is recorded on the returned StructValue via `Array.isArray(value)`,
+   * so `write()` dispatches correctly.
+   */
+  static read(cursor, structName, sizeHint, streamReader, peekFn = null) {
+    const handler = STRUCT_HANDLERS[structName];
+    if (handler && (!peekFn || !peekFn(cursor))) {
+      return new StructValue(structName, { value: handler.read(cursor) });
+    }
+    const startOff = cursor.pos();
+    let nested;
+    try { nested = streamReader(cursor, startOff + sizeHint); }
+    catch (e) {
+      const consumed = cursor.pos() - startOff;
+      const tail = sizeHint - consumed;
+      const opaqueTail = tail > 0 ? cursor.readBytes(tail).slice() : null;
+      return new StructValue(structName, { value: [], decodeError: e.message, opaqueTail });
+    }
+    return new StructValue(structName, { value: nested.properties, terminated: nested.terminated });
+  }
+
+  /**
+   * `streamWriter(writer, propertiesArray)` is `writePropertyStream` (nested form).
+   *
+   * Dispatches on the value shape, NOT on whether a handler exists: when
+   * `value` is an array of Property objects, the struct was decoded via
+   * the property-stream path (regardless of handler registration) and
+   * must be written the same way for byte-identical round-trip. The
+   * handler path runs only when the value is a plain object.
+   */
+  write(writer, streamWriter) {
+    if (Array.isArray(this.value)) {
+      if (this._structDecodeError && this.value.length === 0) {
+        if (this._opaqueTail) writer.writeBytes(this._opaqueTail);
+        return;
+      }
+      if (this._structDecodeError && this.value.length > 0) {
+        throw new Error(`StructValue.write: struct '${this._structName}' had decode error and partial properties; cannot safely re-emit`);
+      }
+      streamWriter(writer, this.value);
+      return;
+    }
+    const handler = STRUCT_HANDLERS[this._structName];
+    if (handler) { handler.write(writer, this.value); return; }
+    if (this._structDecodeError) {
+      if (this._opaqueTail) writer.writeBytes(this._opaqueTail);
+      return;
+    }
+    throw new Error(`StructValue.write: no handler for '${this._structName}' and value is not a property array`);
+  }
+}

package/values.mjs

@@ -0,0 +1,214 @@
+/**
+ * Wrapper classes for non-trivial property-value shapes:
+ *   ObjectRef       — ObjectProperty / ClassProperty / Weak / Lazy
+ *   SoftObjectRef   — SoftObjectProperty / SoftClassProperty
+ *   FTextValue      — TextProperty (FText: localized / culture-invariant string)
+ *   OpaqueValue     — bytes we don't decode (fallback for unknown/unimplemented)
+ *
+ * Array/Set/Map values live in properties.mjs because they're tightly
+ * coupled to PropertyTag (struct arrays carry an inner tag).
+ */
+
+// Circular import: properties.mjs imports ObjectRef/SoftObjectRef/OpaqueValue
+// from this module, and ObjectRef.write needs writeNestedPropertyStream
+// at call time. ESM live bindings make this safe as long as the binding is
+// only USED inside function bodies (deferred), which it is.
+import { writeNestedPropertyStream } from './properties.mjs';
+
+/**
+ * Soulmask ObjectProperty value layout. Each field is optional — the wire
+ * shape is bounded by the property tag's size budget and the reader stops
+ * at whichever boundary it hits first:
+ *
+ *   u8       kind             always present (observed 0x03 at top level; arrays vary)
+ *   u32      kindOnePrefix    ONLY when kind === 0x01. Soulmask-specific
+ *                             4-byte field sitting between the kind byte
+ *                             and pathFS. Observed value is always 1; the
+ *                             semantic meaning is unclear (a flag, an
+ *                             FName.Number, or a count) — we capture and
+ *                             replay it verbatim for byte-identical
+ *                             round-trip. Seen on hard actor references
+ *                             like NPC `HBindBGCompActor` (the pawn's
+ *                             link to its inventory actor).
+ *   FString  path             present iff there's budget left after kind
+ *                             (and kindOnePrefix, when applicable)
+ *   FString  classPath        present iff sizeHint > kind + path length
+ *   nested   stream           present iff sizeHint > kind + path + classPath length;
+ *                             terminated by None, optionally followed by a 4-byte
+ *                             FName.Number trailer for certain Soulmask embeddeds
+ *
+ * `null` for `path` or `classPath` means the field was NOT on the wire
+ * (so the writer skips it). An empty string with the corresponding `isNull`
+ * flag preserves the wire distinction between FString null-form (SaveNum=0,
+ * 4 bytes) and empty-with-terminator (SaveNum=1, 5 bytes).
+ */
+export class ObjectRef {
+  constructor({
+    kind = 0x03,
+    kindOnePrefix = null,
+    path = null,
+    pathIsNull = false,
+    classPath = null,
+    classPathIsNull = false,
+    embedded = null,
+    terminated = false,
+    hasTerminatorTrailer = false,
+  } = {}) {
+    this._objectKind = kind;
+    // null = "not on the wire" (any kind other than 0x01, or a kind=0x01
+    // ObjectRef built programmatically without an explicit prefix).
+    // Numeric = capture from the wire; replayed verbatim on write.
+    this._kindOnePrefix = kindOnePrefix;
+    this.path = path;
+    this._pathIsNull = pathIsNull;
+    this.classPath = classPath;
+    this._classPathIsNull = classPathIsNull;
+    this.embedded = embedded;
+    this.terminated = terminated;
+    // When true, the embedded property stream was followed by a 4-byte
+    // FName.Number trailer (the outermost-stream None-trailer convention,
+    // applied here by Soulmask to some nested ObjectProperty embeddeds —
+    // e.g. JianZhuInstGLQComponent). The reader detects this when exactly
+    // 4 trailing bytes remain inside the tag's size budget; the writer
+    // replays them so round-trip stays byte-identical.
+    this.hasTerminatorTrailer = hasTerminatorTrailer;
+  }
+
+  /** When true, this ObjectRef carries an embedded nested property stream. */
+  get hasEmbedded() { return Array.isArray(this.embedded); }
+
+  write(writer, { requireClassPath = false } = {}) {
+    writer.writeUint8(this._objectKind ?? 0x03);
+    // Soulmask kind=0x01 actor reference: replay the captured 4-byte
+    // prefix between the kind byte and the path FString. Only emitted
+    // when it was on the wire at read time (null otherwise).
+    if (this._kindOnePrefix !== null && this._kindOnePrefix !== undefined) {
+      writer.writeUint32(this._kindOnePrefix);
+    }
+    // Kind-only on the wire: path was null AND nothing forces emission.
+    if (this.path === null && !requireClassPath && !this.hasEmbedded) return;
+    writer.writeFString(this.path ?? '', null, this._pathIsNull);
+    // classPath was either on the wire (non-null) or is forced by `requireClassPath`
+    // (the array-of-ObjectProperty caller, which always writes all three fields)
+    // or by the presence of an embedded stream (the stream can't be reached
+    // on the wire without a classPath FString in front of it).
+    if (requireClassPath || this.classPath !== null || this.hasEmbedded) {
+      writer.writeFString(this.classPath ?? '', null, this._classPathIsNull);
+    }
+    if (this.hasEmbedded) {
+      writeNestedPropertyStream(writer, this.embedded, this.hasTerminatorTrailer);
+    }
+  }
+}
+
+export class SoftObjectRef {
+  constructor({ assetPath = '', subPath = '' } = {}) {
+    this.assetPath = assetPath;
+    this.subPath = subPath;
+  }
+  write(writer) {
+    writer.writeFString(this.assetPath);
+    writer.writeFString(this.subPath);
+  }
+}
+
+/**
+ * Decoded FText value (TextProperty).
+ *
+ * UE4 FText wire format:
+ *   uint32  Flags
+ *   int8    HistoryType
+ *   — HistoryType -1 (None / culture-invariant):
+ *       int32   bHasCultureInvariantString
+ *       [FString displayString]   (only when bHasCultureInvariantString != 0)
+ *   — HistoryType 0 (Base / localized):
+ *       FString Namespace
+ *       FString Key
+ *       FString SourceString
+ *   — HistoryType 2 (OrderedFormat):
+ *       FText   SourceFmt        — the format pattern, e.g. "{0} < {1} >"
+ *       int32   NumArguments
+ *       for each: int8 ContentType + value
+ *         0=Int(int64)  1=UInt(uint64)  2=Float(f32)  3=Double(f64)
+ *         4=Text(FText) 5=Gender(int8)
+ *   — HistoryType 4 (AsNumber, FTextHistory_AsNumber):
+ *       FFormatArgumentValue SourceValue (int8 type + value-by-type)
+ *       uint32 bHasFormatOptions  ← legacy UE3-style 4-byte bool, NOT 1-byte
+ *       [FNumberFormattingOptions FormatOptions]
+ *       uint32 bHasCulture        ← also a uint32 bool
+ *       [FString TargetCulture]
+ *       FNumberFormattingOptions = AlwaysSign(uint32) + UseGrouping(uint32) +
+ *         RoundingMode(int8) + 4 × int32 digit-count fields.
+ *   — All other types: remaining bytes stored in _raw for verbatim round-trip.
+ */
+export class FTextValue {
+  constructor({
+    flags = 0, historyType = -1,
+    displayString, displayStringIsNull = false,
+    namespace, namespaceIsNull = false,
+    key, keyIsNull = false,
+    sourceString, sourceStringIsNull = false,
+    sourceFmt, arguments: args,
+    sourceValue, formatOptions, culture, cultureIsNull = false,
+    _raw,
+  } = {}) {
+    this.flags = flags;
+    this.historyType = historyType;
+    // Per-field isNull flags preserve the wire's choice between FString
+    // null-form (SaveNum=0, 4 B on wire) and empty-with-terminator
+    // (SaveNum=1, 5 B). The two forms decode to the same JS string ("")
+    // but round-trip-equal encoding requires picking the original form.
+    if (historyType === -1) {
+      this.displayString = displayString ?? null;
+      this._displayStringIsNull = displayStringIsNull;
+    } else if (historyType === 0) {
+      this.namespace = namespace ?? '';
+      this._namespaceIsNull = namespaceIsNull;
+      this.key = key ?? '';
+      this._keyIsNull = keyIsNull;
+      this.sourceString = sourceString ?? null;
+      this._sourceStringIsNull = sourceStringIsNull;
+    } else if (historyType === 2) {
+      this.sourceFmt = sourceFmt ?? null;   // FTextValue (the pattern)
+      this.arguments = args ?? [];          // [{type, value}]
+    } else if (historyType === 4) {
+      this.sourceValue = sourceValue ?? null;  // { type: int8, value: number|string|FTextValue }
+      this.formatOptions = formatOptions ?? null; // FNumberFormattingOptions or null
+      this.culture = culture ?? null;          // FString value or null
+      this._cultureIsNull = cultureIsNull;
+    } else {
+      this._raw = _raw ?? null;
+    }
+  }
+
+  /** Best displayable string for this FText, or null if none. */
+  get text() {
+    if (this.historyType === -1) return this.displayString;
+    if (this.historyType === 0) return this.sourceString ?? null;
+    if (this.historyType === 2) return this.sourceFmt?.text ?? null;
+    if (this.historyType === 4) {
+      const v = this.sourceValue?.value;
+      return v != null ? String(v) : null;
+    }
+    return null;
+  }
+}
+
+/**
+ * Holds raw bytes we couldn't (or wouldn't) decode. `reason` is for
+ * debugging only; encoding writes the bytes back verbatim.
+ *
+ * Two access paths intentionally co-exist:
+ *   - `.bytes` / `.reason` — the canonical getter API.
+ *   - `._opaque` / `._opaqueReason` — backing-store fields, also publicly
+ *     readable for consumers that pre-date the getter API.
+ */
+export class OpaqueValue {
+  constructor(bytes, reason = null) {
+    this._opaque = bytes;
+    if (reason) this._opaqueReason = reason;
+  }
+  get bytes()  { return this._opaque; }
+  get reason() { return this._opaqueReason ?? null; }
+  write(writer) { writer.writeBytes(this._opaque); }
+}

package/wscodec.mjs

@@ -0,0 +1,156 @@
+/**
+ * wscodec — pure-JS codec for Soulmask actor_data property streams.
+ *
+ * The library accepts uncompressed bytes (the payload that comes out of
+ * Soulmask's outer LZ4 wrapper) and returns a JavaScript object tree, and
+ * vice versa. It has zero runtime dependencies — LZ4 handling, SQLite
+ * access, etc. are the caller's responsibility.
+ *
+ * Wire layout (the bytes accepted by `UnrealBlob.decode` and produced by
+ * `UnrealBlob.serialize`):
+ *   [0..3]   u32 LE   version tag = 0x00000002
+ *   [4..]    FPropertyTag stream terminated by FString "None" + int32 0
+ *
+ * Soulmask actor_data envelope (handled OUTSIDE this library):
+ *   [0..3]   u32 LE       outer version tag = 0x00000002
+ *   [4..]    LZ4 block    size-prefixed; decompresses to the bytes above.
+ *
+ * The SQLite `actor_table.data_version` column stores the NEGATIVE of the
+ * wire-format DataVersion. A healthy blob with DataVersion=2 lives in a row
+ * whose `data_version` column reads -2. The wire bytes themselves are
+ * always the unsigned 0x00000002 — the negation is purely a column-side
+ * convention.
+ *
+ * Round-trip safety: when `_dirty` is false, `serialize` returns the
+ * original input bytes verbatim. When `_dirty` is true, it re-emits the
+ * property stream from scratch via `writePropertyStream`. Both paths are
+ * verified byte-identical against every row in a tested world.db
+ * (174.6 MB, 11,667 rows; `npm test`).
+ *
+ * Re-exports the most commonly used types so callers can do
+ *   import { UnrealBlob, FName, FGuid, ObjectRef, ... } from 'wscodec';
+ * instead of reaching into individual submodules.
+ */
+
+import { Cursor, Writer } from './io.mjs';
+import { readPropertyStream, writePropertyStream } from './properties.mjs';
+
+// Convenience re-exports for the public API surface.
+export { Cursor, Writer } from './io.mjs';
+export { FName, FGuid } from './primitives.mjs';
+export { StructValue, STRUCT_HANDLERS } from './structs.mjs';
+export { ObjectRef, SoftObjectRef, FTextValue, OpaqueValue } from './values.mjs';
+export {
+  PropertyTag, Property,
+  ArrayValue, SetValue, MapValue,
+  readPropertyStream, writePropertyStream, writeNestedPropertyStream,
+  readValue, writeValue,
+} from './properties.mjs';
+
+const NAME = 'unreal-properties';
+const VERSION_HEADER_SIZE = 4;
+export const VERSION_TAG = 0x00000002;
+
+export class UnrealBlob {
+  constructor({
+    versionTag = VERSION_TAG,
+    properties = [],
+    terminated = false,
+    bodyTrailing = null,
+    error = null,
+    raw = null,
+  } = {}) {
+    this.versionTag = versionTag;
+    this.properties = properties;
+    this.terminated = terminated;
+    this.bodyTrailing = bodyTrailing;
+    this.error = error;
+    this._raw = raw;
+    this._dirty = false;
+  }
+
+  get kind()      { return NAME; }
+  get totalSize() { return this._raw ? this._raw.length : 0; }
+
+  /** First top-level property with the given name, or null. */
+  findProperty(propName) {
+    for (const p of this.properties) {
+      if (p.tag && p.tag.name && p.tag.name.value === propName) return p;
+    }
+    return null;
+  }
+
+  static detect(u8) {
+    if (!u8 || u8.length < VERSION_HEADER_SIZE) return false;
+    const dv = new DataView(u8.buffer, u8.byteOffset, u8.byteLength);
+    return dv.getUint32(0, true) === VERSION_TAG;
+  }
+
+  /**
+   * Parse uncompressed property-stream bytes into an UnrealBlob.
+   *
+   * On unrecoverable structural failure the returned blob has `error` set
+   * and `properties` empty — callers that need a hard failure should check
+   * `blob.error` after decode.
+   */
+  static decode(u8) {
+    if (!UnrealBlob.detect(u8)) {
+      const head = u8 ? Array.from(u8.subarray(0, 4)).map(b => b.toString(16).padStart(2, '0')).join(' ') : '(empty)';
+      throw new Error(`UnrealBlob.decode: not an unreal-properties blob (header bytes: ${head})`);
+    }
+
+    const dv = new DataView(u8.buffer, u8.byteOffset, u8.byteLength);
+    const versionTag = dv.getUint32(0, true);
+    const cursor = new Cursor(u8, VERSION_HEADER_SIZE);
+
+    let properties = [];
+    let terminated = false;
+    let bodyTrailing = null;
+    let error = null;
+
+    try {
+      const stream = readPropertyStream(cursor, u8.length, /*consumeTerminatorTrailer=*/true);
+      properties = stream.properties;
+      terminated = stream.terminated;
+      if (cursor.pos() < u8.length) {
+        bodyTrailing = u8.slice(cursor.pos());
+      }
+    } catch (e) {
+      error = e.message;
+    }
+
+    return new UnrealBlob({ versionTag, properties, terminated, bodyTrailing, error, raw: u8 });
+  }
+
+  /**
+   * Return the uncompressed property-stream bytes for this blob.
+   *
+   * Pass-through when `_dirty` is false: returns the input bytes verbatim.
+   * Re-encodes from `properties` when `_dirty` is true. `bodyTrailing`, if
+   * present, is appended after the None terminator + 4-byte FName.Number
+   * trailer that `writePropertyStream` emits.
+   */
+  serialize() {
+    if (!this._dirty && this._raw instanceof Uint8Array) return this._raw;
+
+    const w = new Writer(this._raw?.length || 256);
+    w.writeUint32(this.versionTag);
+    writePropertyStream(w, this.properties, /*emitTerminatorTrailer=*/true);
+    if (this.bodyTrailing && this.bodyTrailing.length > 0) {
+      w.writeBytes(this.bodyTrailing);
+    }
+    return w.finalize();
+  }
+}
+
+// Generic codec-adapter shape (name + detect + decode + encode), suitable
+// for plugging into any registry that uses that quartet. Operates on the
+// uncompressed bytes that `UnrealBlob.decode` accepts; for callers reading
+// Soulmask's actor_data column directly, wrap this with the column's outer
+// LZ4 envelope (4-byte version tag + size-prefixed LZ4 block).
+export const codec = {
+  name: NAME,
+  detect: u8 => UnrealBlob.detect(u8),
+  decode: u8 => UnrealBlob.decode(u8),
+  encode: blob => blob.serialize(),
+};