wscodec 0.1.0 → 0.3.0

package/structs.mjs

@@ -3,16 +3,20 @@
  *
  * Soulmask is UE 4.27 so "core" structs (Vector etc.) use 32-bit floats.
  * Known struct names read directly as binary; unknown struct names fall
- * through to a nested property stream (handled in properties.mjs, not here
- * — StructValue.read is supplied a `streamReader` callback to avoid a
+ * through to a nested property stream (handled in properties.mjs, not here;
+ * StructValue.read is supplied a `streamReader` callback to avoid a
  * load-order cycle).
  */
 
 import { FGuid } from './primitives.mjs';
 
-// Binary struct handlers. Each entry has read(cursor) → plain object and
+// Binary struct handlers. Each entry has read(cursor) -> plain object and
 // write(writer, plainObject). The plain object is what callers see as
 // `structValue.value` when the struct is one of these known shapes.
+//
+// Consumers can extend this registry to teach the codec about additional
+// known-binary structs; prefer the `registerStructHandler` helper below over
+// mutating this object directly, since it validates the handler shape.
 export const STRUCT_HANDLERS = {
   Vector:      { read: c => ({ x: c.readFloat32(), y: c.readFloat32(), z: c.readFloat32() }),
                  write: (w, v) => { w.writeFloat32(v.x); w.writeFloat32(v.y); w.writeFloat32(v.z); } },
@@ -24,12 +28,20 @@ export const STRUCT_HANDLERS = {
                  write: (w, v) => { w.writeFloat32(v.pitch); w.writeFloat32(v.yaw); w.writeFloat32(v.roll); } },
   Quat:        { read: c => ({ x: c.readFloat32(), y: c.readFloat32(), z: c.readFloat32(), w: c.readFloat32() }),
                  write: (w, v) => { w.writeFloat32(v.x); w.writeFloat32(v.y); w.writeFloat32(v.z); w.writeFloat32(v.w); } },
+  // FColor wire order is B, G, R, A (not R, G, B, A). This matches UE4's
+  // FColor::Serialize, where the in-memory union exposes the bytes in BGRA
+  // order to match Windows DIB / DirectX texture layout. Don't "fix" the
+  // ordering; it's correct as-is.
   Color:       { read: c => ({ b: c.readUint8(), g: c.readUint8(), r: c.readUint8(), a: c.readUint8() }),
                  write: (w, v) => { w.writeUint8(v.b); w.writeUint8(v.g); w.writeUint8(v.r); w.writeUint8(v.a); } },
   LinearColor: { read: c => ({ r: c.readFloat32(), g: c.readFloat32(), b: c.readFloat32(), a: c.readFloat32() }),
                  write: (w, v) => { w.writeFloat32(v.r); w.writeFloat32(v.g); w.writeFloat32(v.b); w.writeFloat32(v.a); } },
-  Guid:        { read: c => FGuid.read(c).value,
-                 write: (w, v) => new FGuid(v).write(w) },
+  // Guid returns an FGuid INSTANCE (not a bare string). FGuid carries
+  // toJSON/equals/isZero helpers; the write path accepts FGuid or a bare
+  // 8-4-4-4-12 string for backward compatibility with code that built the
+  // struct value from a literal.
+  Guid:        { read: c => FGuid.read(c),
+                 write: (w, v) => FGuid.from(v).write(w) },
   DateTime:    { read: c => c.readInt64().toString(),
                  write: (w, v) => w.writeInt64(v) },
   Timespan:    { read: c => c.readInt64().toString(),
@@ -48,6 +60,27 @@ export const STRUCT_HANDLERS = {
                  write: (w, v) => { STRUCT_HANDLERS.Quat.write(w, v.rotation); STRUCT_HANDLERS.Vector.write(w, v.translation); STRUCT_HANDLERS.Vector.write(w, v.scale3D); } },
 };
 
+/**
+ * Register (or replace) a struct handler. Callers can use this to teach the
+ * codec about additional binary structs the game emits that aren't in the
+ * stock registry. Without a handler, an unknown struct name falls through to
+ * the nested-property-stream path; that's still correct when the struct is
+ * actually tagged, and is byte-identical on round-trip via OpaqueValue when
+ * it isn't.
+ *
+ * Validates that `handler` has both `read(cursor)` and `write(writer, value)`
+ * functions and that `name` is a non-empty string.
+ */
+export function registerStructHandler(name, handler) {
+  if (typeof name !== 'string' || name.length === 0) {
+    throw new TypeError('registerStructHandler: name must be a non-empty string');
+  }
+  if (!handler || typeof handler.read !== 'function' || typeof handler.write !== 'function') {
+    throw new TypeError('registerStructHandler: handler must expose read(cursor) and write(writer, value) functions');
+  }
+  STRUCT_HANDLERS[name] = handler;
+}
+
 export class StructValue {
   constructor(structName, { value = null, terminated = false, decodeError = null, opaqueTail = null } = {}) {
     this._structName = structName;
@@ -70,7 +103,7 @@ export class StructValue {
    * PropertyTag (FString name with identifier-character ASCII content).
    * When supplied and the wire bytes look tagged, the read switches to
    * the property-stream path even for structs that have a known binary
-   * handler — Soulmask encodes known-binary structs (Transform, Box, ...)
+   * handler: Soulmask encodes known-binary structs (Transform, Box, ...)
    * as TAGGED property streams inside Map struct values, which would
    * otherwise be misread as raw 40-byte Transforms / etc. The decision
    * is recorded on the returned StructValue via `Array.isArray(value)`,

package/values.mjs

@@ -1,9 +1,9 @@
 /**
  * Wrapper classes for non-trivial property-value shapes:
- *   ObjectRef       — ObjectProperty / ClassProperty / Weak / Lazy
- *   SoftObjectRef   — SoftObjectProperty / SoftClassProperty
- *   FTextValue      — TextProperty (FText: localized / culture-invariant string)
- *   OpaqueValue     — bytes we don't decode (fallback for unknown/unimplemented)
+ *   ObjectRef       : ObjectProperty / ClassProperty / Weak / Lazy
+ *   SoftObjectRef   : SoftObjectProperty / SoftClassProperty
+ *   FTextValue      : TextProperty (FText: localized / culture-invariant string)
+ *   OpaqueValue     : bytes we don't decode (fallback for unknown/unimplemented)
  *
  * Array/Set/Map values live in properties.mjs because they're tightly
  * coupled to PropertyTag (struct arrays carry an inner tag).
@@ -16,7 +16,7 @@
 import { writeNestedPropertyStream } from './properties.mjs';
 
 /**
- * Soulmask ObjectProperty value layout. Each field is optional — the wire
+ * Soulmask ObjectProperty value layout. Each field is optional; the wire
  * shape is bounded by the property tag's size budget and the reader stops
  * at whichever boundary it hits first:
  *
@@ -25,7 +25,7 @@ import { writeNestedPropertyStream } from './properties.mjs';
  *                             4-byte field sitting between the kind byte
  *                             and pathFS. Observed value is always 1; the
  *                             semantic meaning is unclear (a flag, an
- *                             FName.Number, or a count) — we capture and
+ *                             FName.Number, or a count); we capture and
  *                             replay it verbatim for byte-identical
  *                             round-trip. Seen on hard actor references
  *                             like NPC `HBindBGCompActor` (the pawn's
@@ -67,7 +67,7 @@ export class ObjectRef {
     this.terminated = terminated;
     // When true, the embedded property stream was followed by a 4-byte
     // FName.Number trailer (the outermost-stream None-trailer convention,
-    // applied here by Soulmask to some nested ObjectProperty embeddeds —
+    // applied here by Soulmask to some nested ObjectProperty embeddeds,
     // e.g. JianZhuInstGLQComponent). The reader detects this when exactly
     // 4 trailing bytes remain inside the tag's size budget; the writer
     // replays them so round-trip stays byte-identical.
@@ -118,28 +118,28 @@ export class SoftObjectRef {
  * UE4 FText wire format:
  *   uint32  Flags
  *   int8    HistoryType
- *   — HistoryType -1 (None / culture-invariant):
+ *   HistoryType -1 (None / culture-invariant):
  *       int32   bHasCultureInvariantString
  *       [FString displayString]   (only when bHasCultureInvariantString != 0)
- *   — HistoryType 0 (Base / localized):
+ *   HistoryType 0 (Base / localized):
  *       FString Namespace
  *       FString Key
  *       FString SourceString
- *   — HistoryType 2 (OrderedFormat):
- *       FText   SourceFmt        — the format pattern, e.g. "{0} < {1} >"
+ *   HistoryType 2 (OrderedFormat):
+ *       FText   SourceFmt        (the format pattern, e.g. "{0} < {1} >")
  *       int32   NumArguments
  *       for each: int8 ContentType + value
  *         0=Int(int64)  1=UInt(uint64)  2=Float(f32)  3=Double(f64)
  *         4=Text(FText) 5=Gender(int8)
- *   — HistoryType 4 (AsNumber, FTextHistory_AsNumber):
+ *   HistoryType 4 (AsNumber, FTextHistory_AsNumber):
  *       FFormatArgumentValue SourceValue (int8 type + value-by-type)
  *       uint32 bHasFormatOptions  ← legacy UE3-style 4-byte bool, NOT 1-byte
  *       [FNumberFormattingOptions FormatOptions]
  *       uint32 bHasCulture        ← also a uint32 bool
  *       [FString TargetCulture]
  *       FNumberFormattingOptions = AlwaysSign(uint32) + UseGrouping(uint32) +
- *         RoundingMode(int8) + 4 × int32 digit-count fields.
- *   — All other types: remaining bytes stored in _raw for verbatim round-trip.
+ *         RoundingMode(int8) + 4 x int32 digit-count fields.
+ *   All other types: remaining bytes stored in _raw for verbatim round-trip.
  */
 export class FTextValue {
   constructor({
@@ -168,6 +168,9 @@ export class FTextValue {
       this._keyIsNull = keyIsNull;
       this.sourceString = sourceString ?? null;
       this._sourceStringIsNull = sourceStringIsNull;
+    } else if (historyType === 1) {
+      this.sourceFmt = sourceFmt ?? null;   // FTextValue (the pattern)
+      this.arguments = args ?? [];          // [{key, keyIsNull, type, value}]
     } else if (historyType === 2) {
       this.sourceFmt = sourceFmt ?? null;   // FTextValue (the pattern)
       this.arguments = args ?? [];          // [{type, value}]
@@ -185,6 +188,7 @@ export class FTextValue {
   get text() {
     if (this.historyType === -1) return this.displayString;
     if (this.historyType === 0) return this.sourceString ?? null;
+    if (this.historyType === 1) return this.sourceFmt?.text ?? null;
     if (this.historyType === 2) return this.sourceFmt?.text ?? null;
     if (this.historyType === 4) {
       const v = this.sourceValue?.value;
@@ -195,20 +199,21 @@ export class FTextValue {
 }
 
 /**
- * Holds raw bytes we couldn't (or wouldn't) decode. `reason` is for
- * debugging only; encoding writes the bytes back verbatim.
+ * Holds raw bytes we couldn't (or wouldn't) decode. `reason` is a free-form
+ * string for debugging only; encoding writes the bytes back verbatim, so a
+ * value the codec couldn't parse still round-trips byte-identical.
  *
- * Two access paths intentionally co-exist:
- *   - `.bytes` / `.reason` — the canonical getter API.
- *   - `._opaque` / `._opaqueReason` — backing-store fields, also publicly
- *     readable for consumers that pre-date the getter API.
+ * OpaqueValue is the codec's universal fallback for everything from unknown
+ * property types to mid-decode recoveries (Struct/Array/Set/Map/Text whose
+ * inner shape didn't parse cleanly). The reader's contract is: on any
+ * structural failure inside a finite byte budget, rewind to the value's
+ * start and capture the budget verbatim into an OpaqueValue, so the outer
+ * stream stays byte-aligned regardless of what went wrong inside.
  */
 export class OpaqueValue {
   constructor(bytes, reason = null) {
-    this._opaque = bytes;
-    if (reason) this._opaqueReason = reason;
+    this.bytes = bytes;
+    this.reason = reason;
   }
-  get bytes()  { return this._opaque; }
-  get reason() { return this._opaqueReason ?? null; }
-  write(writer) { writer.writeBytes(this._opaque); }
+  write(writer) { writer.writeBytes(this.bytes); }
 }

package/wscodec.mjs

@@ -1,9 +1,9 @@
 /**
- * wscodec — pure-JS codec for Soulmask actor_data property streams.
+ * wscodec: pure-JS codec for Soulmask actor_data property streams.
  *
  * The library accepts uncompressed bytes (the payload that comes out of
  * Soulmask's outer LZ4 wrapper) and returns a JavaScript object tree, and
- * vice versa. It has zero runtime dependencies — LZ4 handling, SQLite
+ * vice versa. It has zero runtime dependencies; LZ4 handling, SQLite
  * access, etc. are the caller's responsibility.
  *
  * Wire layout (the bytes accepted by `UnrealBlob.decode` and produced by
@@ -18,14 +18,14 @@
  * The SQLite `actor_table.data_version` column stores the NEGATIVE of the
  * wire-format DataVersion. A healthy blob with DataVersion=2 lives in a row
  * whose `data_version` column reads -2. The wire bytes themselves are
- * always the unsigned 0x00000002 — the negation is purely a column-side
+ * always the unsigned 0x00000002; the negation is purely a column-side
  * convention.
  *
  * Round-trip safety: when `_dirty` is false, `serialize` returns the
  * original input bytes verbatim. When `_dirty` is true, it re-emits the
  * property stream from scratch via `writePropertyStream`. Both paths are
  * verified byte-identical against every row in a tested world.db
- * (174.6 MB, 11,667 rows; `npm test`).
+ * (`npm test`).
  *
  * Re-exports the most commonly used types so callers can do
  *   import { UnrealBlob, FName, FGuid, ObjectRef, ... } from 'wscodec';
@@ -38,7 +38,7 @@ import { readPropertyStream, writePropertyStream } from './properties.mjs';
 // Convenience re-exports for the public API surface.
 export { Cursor, Writer } from './io.mjs';
 export { FName, FGuid } from './primitives.mjs';
-export { StructValue, STRUCT_HANDLERS } from './structs.mjs';
+export { StructValue, STRUCT_HANDLERS, registerStructHandler } from './structs.mjs';
 export { ObjectRef, SoftObjectRef, FTextValue, OpaqueValue } from './values.mjs';
 export {
   PropertyTag, Property,
@@ -46,6 +46,14 @@ export {
   readPropertyStream, writePropertyStream, writeNestedPropertyStream,
   readValue, writeValue,
 } from './properties.mjs';
+// JSON converter: declared at the bottom so UnrealBlob (below) is already
+// defined when json.mjs's deferred references resolve. ESM live bindings
+// keep this load-order safe even with the json.mjs ↔ wscodec.mjs cycle.
+export {
+  blobToJSON, jsonToBlob,
+  blobToJSONString, jsonStringToBlob,
+  jsonReplacer, jsonReviver,
+} from './json.mjs';
 
 const NAME = 'unreal-properties';
 const VERSION_HEADER_SIZE = 4;
@@ -69,10 +77,25 @@ export class UnrealBlob {
     this._dirty = false;
   }
 
-  get kind()      { return NAME; }
+  /**
+   * Codec-adapter name (`'unreal-properties'`). Surfaced for registries that
+   * dispatch on a codec's `kind` field; matches the `name` on the bare
+   * `codec` adapter exported at the bottom of this module.
+   */
+  get kind() { return NAME; }
+
+  /**
+   * Number of bytes the blob was decoded from (`_raw.length`), or 0 if the
+   * blob was constructed without an input buffer. NOT the post-serialize
+   * size; for that, call `serialize().length`.
+   */
   get totalSize() { return this._raw ? this._raw.length : 0; }
 
-  /** First top-level property with the given name, or null. */
+  /**
+   * First TOP-LEVEL property with the given tag name, or null. Does NOT
+   * traverse into embedded streams, struct values, array elements, or map
+   * entries. Use `findPropertyDeep` to walk the full tree.
+   */
   findProperty(propName) {
     for (const p of this.properties) {
       if (p.tag && p.tag.name && p.tag.name.value === propName) return p;
@@ -80,6 +103,23 @@ export class UnrealBlob {
     return null;
   }
 
+  /**
+   * First property with the given tag name found anywhere in the property
+   * tree, or null. Performs a depth-first traversal across:
+   *
+   *   - top-level properties
+   *   - ObjectRef.embedded streams (nested ObjectProperty values)
+   *   - StructValue.value when it's a tagged property array
+   *   - ArrayProperty / SetProperty struct elements
+   *   - MapProperty entries: both key (if StructValue) and value
+   *
+   * Returns the first match in traversal order; later matches are not
+   * surfaced. For all matches, walk the tree manually.
+   */
+  findPropertyDeep(propName) {
+    return _findPropertyDeep(this.properties, propName);
+  }
+
   static detect(u8) {
     if (!u8 || u8.length < VERSION_HEADER_SIZE) return false;
     const dv = new DataView(u8.buffer, u8.byteOffset, u8.byteLength);
@@ -90,7 +130,7 @@ export class UnrealBlob {
    * Parse uncompressed property-stream bytes into an UnrealBlob.
    *
    * On unrecoverable structural failure the returned blob has `error` set
-   * and `properties` empty — callers that need a hard failure should check
+   * and `properties` empty. Callers that need a hard failure should check
    * `blob.error` after decode.
    */
   static decode(u8) {
@@ -125,15 +165,39 @@ export class UnrealBlob {
   /**
    * Return the uncompressed property-stream bytes for this blob.
    *
-   * Pass-through when `_dirty` is false: returns the input bytes verbatim.
-   * Re-encodes from `properties` when `_dirty` is true. `bodyTrailing`, if
-   * present, is appended after the None terminator + 4-byte FName.Number
+   * Pass-through when `_dirty` is false: returns the input bytes verbatim,
+   * even if `error` is set (the original bytes round-trip even when decode
+   * was incomplete). Re-encodes from `properties` when `_dirty` is true,
+   * appending `bodyTrailing` after the None terminator + 4-byte FName.Number
    * trailer that `writePropertyStream` emits.
+   *
+   * Options:
+   *   `recomputeSizes` — override `this._recomputeSizes`. When truthy, every
+   *     PropertyTag.size field (and ArrayValue innerTag.size) is rewritten
+   *     from the actual encoded value byte count. Required after edits that
+   *     change variable-length fields (FString contents, FText, MapValue
+   *     contents, etc.); without it, a stale size field leaves the Soulmask
+   *     reader misaligned and the blob is rejected on load. The default is
+   *     to honor `this._recomputeSizes`; jsonToBlob sets that to true so
+   *     the JSON pipeline always recomputes.
+   *
+   * Throws if `_dirty` is true AND `error` is set: re-emitting would produce
+   * a malformed stream (the property tree is empty after a structural
+   * failure). Clear `.error` first if you intentionally want to emit from
+   * an externally-constructed properties array.
    */
-  serialize() {
+  serialize({ recomputeSizes } = {}) {
     if (!this._dirty && this._raw instanceof Uint8Array) return this._raw;
 
+    if (this.error != null) {
+      throw new Error(
+        `UnrealBlob.serialize: cannot re-emit a blob with decode error (${this.error}). ` +
+        `Leave _dirty=false to pass through _raw verbatim, or clear .error if you've replaced .properties manually.`
+      );
+    }
+
     const w = new Writer(this._raw?.length || 256);
+    if (recomputeSizes ?? this._recomputeSizes) w._wsRecomputeSizes = true;
     w.writeUint32(this.versionTag);
     writePropertyStream(w, this.properties, /*emitTerminatorTrailer=*/true);
     if (this.bodyTrailing && this.bodyTrailing.length > 0) {
@@ -143,6 +207,59 @@ export class UnrealBlob {
   }
 }
 
+// Deep-search helper. Walks the property tree in depth-first order and
+// returns the first Property whose tag.name matches. Kept out of the class
+// body so the recursion can reach into nested shapes uniformly without
+// having to thread `this` around.
+function _findPropertyDeep(properties, propName) {
+  if (!Array.isArray(properties)) return null;
+  for (const p of properties) {
+    if (p.tag && p.tag.name && p.tag.name.value === propName) return p;
+    const v = p.value;
+    if (v == null) continue;
+    // ObjectRef with embedded property stream.
+    if (v.embedded) {
+      const hit = _findPropertyDeep(v.embedded, propName);
+      if (hit) return hit;
+    }
+    // StructValue: .value is either a property array (unknown struct) or a
+    // plain binary record (known struct). Only the array form is searchable.
+    if (v._structName && Array.isArray(v.value)) {
+      const hit = _findPropertyDeep(v.value, propName);
+      if (hit) return hit;
+    }
+    // ArrayProperty / SetProperty struct elements + ObjectRef embeddeds.
+    if (Array.isArray(v.elements)) {
+      for (const e of v.elements) {
+        if (e && e._structName && Array.isArray(e.value)) {
+          const hit = _findPropertyDeep(e.value, propName);
+          if (hit) return hit;
+        }
+        if (e && e.embedded) {
+          const hit = _findPropertyDeep(e.embedded, propName);
+          if (hit) return hit;
+        }
+      }
+    }
+    // MapProperty entries: both key (if StructValue) and value can hold a
+    // nested property stream.
+    if (Array.isArray(v.entries)) {
+      for (const ent of v.entries) {
+        if (ent.key && ent.key._structName && Array.isArray(ent.key.value)) {
+          const hit = _findPropertyDeep(ent.key.value, propName);
+          if (hit) return hit;
+        }
+        const ev = ent.value;
+        if (ev && ev._structName && Array.isArray(ev.value)) {
+          const hit = _findPropertyDeep(ev.value, propName);
+          if (hit) return hit;
+        }
+      }
+    }
+  }
+  return null;
+}
+
 // Generic codec-adapter shape (name + detect + decode + encode), suitable
 // for plugging into any registry that uses that quartet. Operates on the
 // uncompressed bytes that `UnrealBlob.decode` accepts; for callers reading