threenative-asset-mcp 0.9.2 → 0.9.4

package/dist/unreal/importer.js

@@ -8,7 +8,7 @@ import { Document, NodeIO, VertexLayout } from "@gltf-transform/core";
 import { EXTMeshGPUInstancing, KHRLightsPunctual, KHRMaterialsUnlit } from "@gltf-transform/extensions";
 import { attachPsaAnimations, parsePsa } from "./psa.js";
 import { resolveMaterial, } from "./materials.js";
-import { readPackageCooking } from "./cooking.js";
+import { readPackageCooking, readPackageObjectNames } from "./cooking.js";
 import { extractUnrealFonts } from "./fonts.js";
 import { parseOfflineFontDescriptor, writeOfflineFont } from "./bitmap-fonts.js";
 import { paperObjectPathToPackage, parsePaperFlipbookDescriptor, parsePaperSpriteDescriptor, writePaperSpriteGlb, } from "./paper2d.js";
@@ -18,7 +18,21 @@ import { assembleSceneGlb, parseUnrealSceneSource, } from "./scenes.js";
 import { ToolchainError, assertSupportedHost, runBounded } from "./toolchain.js";
 const statfsAsync = promisify(statfs);
 /** Bumped whenever the conversion contract changes; it participates in the reuse cache key. */
-export const IMPORTER_VERSION = 44;
+export const IMPORTER_VERSION = 46;
+/** First and last UE4 object versions whose uncooked StaticMesh source models are FMeshDescription
+ * bulk data (UE4.25–4.27), which only the engine-free converter reads. Below that window UE Viewer
+ * reads uncooked source geometry itself, static and skeletal alike: verified on FAB packs saved at
+ * object versions 401–516 (UE4.0–4.20), where it matched or beat every hand-written decoder. */
+const MESH_DESCRIPTION_FIRST_VERSION = 517;
+const MESH_DESCRIPTION_LAST_VERSION = 522;
+/** Which tool decodes an uncooked mesh package, or undefined when it must be refused. */
+export function uncookedMeshRoute(meshKind, fileVersionUE4) {
+    if (fileVersionUE4 !== undefined && fileVersionUE4 < MESH_DESCRIPTION_FIRST_VERSION)
+        return "umodel";
+    if (meshKind === "skeletal")
+        return "modern";
+    return fileVersionUE4 !== undefined && fileVersionUE4 <= MESH_DESCRIPTION_LAST_VERSION ? "mesh-description" : undefined;
+}
 export class ImportError extends Error {
     code;
     retryable;
@@ -57,6 +71,23 @@ export function summarizeClasses(classes, keep = 4) {
         return unique.join(", ");
     return `${unique.slice(0, keep).join(", ")} and ${unique.length - keep} more`;
 }
+/** Staging older than this belongs to an import that was killed before its `finally` ran. */
+const STALE_STAGING_MS = 24 * 60 * 60 * 1000;
+/** Removes `run-*` staging left by killed imports. One such run can hold gigabytes, and they
+ * accumulate until the cache volume fills and every later import fails its free-space check. */
+export async function sweepStaleStaging(root, now = Date.now()) {
+    for (const key of await readdir(root).catch(() => [])) {
+        for (const run of await readdir(join(root, key)).catch(() => [])) {
+            if (!run.startsWith("run-"))
+                continue;
+            const path = join(root, key, run);
+            const modified = (await stat(path).catch(() => undefined))?.mtimeMs;
+            if (modified !== undefined && now - modified > STALE_STAGING_MS) {
+                await rm(path, { recursive: true, force: true }).catch(() => { });
+            }
+        }
+    }
+}
 function cacheRoot(environment) {
     return (environment.THREENATIVE_UNREAL_CACHE_DIR?.trim() ||
         join(environment.XDG_CACHE_HOME?.trim() || join(homedir(), ".cache"), "threenative-asset-mcp", "unreal-import"));
@@ -815,7 +846,9 @@ export function inspectWebAudio(data) {
     }
     return undefined;
 }
-async function freeBytes(path) {
+async function freeBytes(path, override) {
+    if (override !== undefined)
+        return override;
     try {
         const info = await statfsAsync(path);
         return Number(info.bavail) * Number(info.bsize);
@@ -824,6 +857,66 @@ async function freeBytes(path) {
         return Number.POSITIVE_INFINITY;
     }
 }
+/** `Foo.uasset` is accompanied by `Foo.uexp`, `Foo.ubulk`, and `Foo.uptnl`; a partial import
+ * reads every file of the stem it converts, and only files of that stem. */
+function packageStem(file) {
+    return join(dirname(file), basename(file, extname(file)));
+}
+/** Source bytes a filtered import will actually read: the requested packages, their sidecars, and
+ * the packages they name in their import tables — a mesh's materials and textures are separate
+ * packages and hold most of a pack's bytes. Counting the whole tree instead refuses every
+ * partial import on a full volume, and counting only the requested `.uasset` would let an import
+ * start that cannot finish. */
+async function importedPackageBytes(files, wanted) {
+    const groups = new Map();
+    for (const file of files) {
+        const stem = packageStem(file.path);
+        const group = groups.get(stem) ?? { primary: file.path, bytes: 0 };
+        group.bytes += file.size;
+        if ([".uasset", ".umap"].includes(extname(file.path).toLowerCase()))
+            group.primary = file.path;
+        groups.set(stem, group);
+    }
+    const stemsByName = new Map();
+    for (const stem of groups.keys()) {
+        const name = basename(stem);
+        stemsByName.set(name, [...(stemsByName.get(name) ?? []), stem]);
+    }
+    let total = 0;
+    const counted = new Set();
+    let frontier = [...groups.keys()].filter((stem) => wanted.has(basename(stem)));
+    while (frontier.length > 0) {
+        const next = [];
+        for (const stem of frontier) {
+            counted.add(stem);
+            total += groups.get(stem).bytes;
+            // ponytail: a bounded byte scan for names, not a parsed import table. A name that merely
+            // shares a string with the package counts its bytes; a name in another encoding does not.
+            for (const name of await readPackageObjectNames(groups.get(stem).primary)) {
+                for (const referenced of stemsByName.get(name) ?? []) {
+                    if (counted.has(referenced))
+                        continue;
+                    next.push(referenced);
+                }
+            }
+        }
+        frontier = next;
+    }
+    return total;
+}
+/**
+ * Why the modern converter produced nothing, in the terms that decide the next move: what it
+ * printed (its last lines, not only the one it exits on), which engine wrote the package, and
+ * whether the mesh is Nanite — the commonest reason a UE5 editor mesh has no render data.
+ */
+function describeModernFailure(stderr, packages) {
+    const lines = stderr.trim().split(/\r?\n/).filter((line) => line.trim()).slice(-20);
+    const versions = packages.map(({ entry, legacyFileVersion, fileVersionUE4, fileVersionUE5, naniteHint }) => `${entry.package} is ${legacyFileVersion !== undefined && legacyFileVersion <= -8 ? "UE5" : "UE4"} (legacy file version ${legacyFileVersion ?? "unknown"}), UE4 object version ${fileVersionUE4 ?? "unknown"}, UE5 object version ${fileVersionUE5 ?? "unknown"}; Nanite: ${naniteHint ? "likely (NaniteSettings present)" : "not detected"}`);
+    return [
+        ...versions,
+        ...(lines.length > 0 ? [`Converter output (last ${lines.length} lines):`, ...lines.map((line) => `  ${line.trim()}`)] : []),
+    ].join(" ");
+}
 /**
  * Converts a local Unreal asset directory into self-contained source GLBs plus a provenance
  * report. Provider-independent: a pack downloaded by FabCLI and one unzipped by hand take the
@@ -854,6 +947,11 @@ export async function importUnrealDirectory(request) {
         throw new ImportError("UNREAL_SOURCE_EMPTY", `"${sourceDir}" contains no .uasset or .umap files.`);
     }
     const totalBytes = files.reduce((sum, file) => sum + file.size, 0);
+    // Only a single-package request is actually filtered: both converters take `--filter` for one
+    // package alone and convert the whole tree otherwise, so two or more still need the whole tree.
+    const neededBytes = request.onlyPackages?.length === 1 && request.onlyPackages[0]
+        ? await importedPackageBytes(files, new Set(request.onlyPackages))
+        : totalBytes;
     const sourceHash = await hashSourceTree(sourceDir, files);
     const umodel = request.umodel ?? (await ensureUmodel(environment, log));
     const cacheKey = createHash("sha256")
@@ -886,6 +984,7 @@ export async function importUnrealDirectory(request) {
             throw new ImportError("UNREAL_OUTPUT_COLLISION", `${outputDir} is not empty and holds no import report. Choose a new output directory.`);
         }
     }
+    await sweepStaleStaging(cacheRoot(environment));
     const stagingRoot = join(cacheRoot(environment), cacheKey);
     await mkdir(stagingRoot, { recursive: true });
     // The cache key identifies reusable results, not ownership of mutable scratch space. Give every
@@ -899,17 +998,17 @@ export async function importUnrealDirectory(request) {
     // Unbounded scene assembly can temporarily hold model GLBs plus a source-sized aggregate GLB.
     // Staging and output may be different volumes, so checking only the cache volume is insufficient.
     const minimum = 2 * 1024 ** 3;
-    const stagingRequired = totalBytes;
-    const outputRequired = totalBytes * (request.maxTextureSize === undefined ? 2 : 1);
+    const stagingRequired = neededBytes;
+    const outputRequired = neededBytes * (request.maxTextureSize === undefined ? 2 : 1);
     const stagingDevice = (await stat(staging)).dev;
     const outputDevice = (await stat(promotionParent)).dev;
     const required = Math.max(stagingDevice === outputDevice ? stagingRequired + outputRequired : outputRequired, minimum);
-    const available = await freeBytes(promotionParent);
+    const available = await freeBytes(promotionParent, request.freeSpaceBytes);
     if (available < required) {
         throw new ImportError("UNREAL_DISK_SPACE", `The import needs about ${Math.ceil(required / 1024 ** 3)} GiB on the output volume; ${Math.floor(available / 1024 ** 3)} GiB is free at ${promotionParent}. Use --max-texture-size or choose an output on a larger volume.`);
     }
     if (stagingDevice !== outputDevice) {
-        const stagingAvailable = await freeBytes(staging);
+        const stagingAvailable = await freeBytes(staging, request.freeSpaceBytes);
         const separateStagingRequired = Math.max(stagingRequired, minimum);
         if (stagingAvailable < separateStagingRequired) {
             throw new ImportError("UNREAL_DISK_SPACE", `The import needs about ${Math.ceil(separateStagingRequired / 1024 ** 3)} GiB of staging space; ${Math.floor(stagingAvailable / 1024 ** 3)} GiB is free at ${staging}.`);
@@ -964,12 +1063,16 @@ export async function importUnrealDirectory(request) {
                     ? "static"
                     : cooking?.meshKindHint;
             const isMapBuildData = classes.includes("MapBuildDataRegistry");
-            const hasTexture = !isMapBuildData && (classes.includes("Texture2D") || cooking?.textureHint === true);
+            // The name-table hints stand in for a listing UE Viewer could not produce. When it did list
+            // the package, its classes win: a material instance names Texture2D and carries
+            // AssetImportData too, and reading that as a texture drops the material entirely.
+            const listed = run.code === 0 && classes.length > 0;
+            const hasTexture = !isMapBuildData && (classes.includes("Texture2D") || (!listed && cooking?.textureHint === true));
             // MapBuildDataRegistry owns transient reflection-capture cubes that are tied to baked level
             // lighting. They are not standalone environment assets and cannot be meaningfully reused.
-            const hasCubemap = !isMapBuildData && (classes.includes("TextureCube") || cooking?.cubemapHint === true);
+            const hasCubemap = !isMapBuildData && (classes.includes("TextureCube") || (!listed && cooking?.cubemapHint === true));
             const hasMaterial = classes.some((className) => ["Material", "Material3", "MaterialInstance", "MaterialInstanceConstant"].includes(className));
-            const hasSound = classes.includes("SoundWave") || cooking?.soundHint === true;
+            const hasSound = classes.includes("SoundWave") || (!listed && cooking?.soundHint === true);
             const dataClass = classes.find((className) => DATA_ASSET_CLASSES.has(className)) ?? cooking?.dataClassHint;
             const textureStackClass = classes.find((className) => TEXTURE_STACK_CLASSES.has(className)) ?? cooking?.textureStackClassHint;
             const hasFont = classes.some((className) => className === "Font" || className === "FontFace") || cooking?.fontHint === true;
@@ -1132,64 +1235,43 @@ export async function importUnrealDirectory(request) {
         package: entry.package,
         ...(await readPackageCooking(entry.file)),
     }));
-    const modernPackages = cooking.filter(({ entry, state }) => entry.needsModernConverter || (entry.meshKind === "skeletal" && state === "uncooked"));
+    const modernPackages = cooking.filter(({ entry, state, fileVersionUE4 }) => entry.needsModernConverter ||
+        (entry.meshKind === "skeletal" && state === "uncooked" && uncookedMeshRoute("skeletal", fileVersionUE4) === "modern"));
     const modernTexturePackages = texturePackages.filter((entry) => entry.needsModernConverter);
     const modernMaterialPackages = materialPackages.filter((entry) => entry.needsModernConverter);
     const modernSoundPackages = soundPackages.filter((entry) => entry.needsModernConverter);
     const modernFiles = new Set(modernPackages.map(({ entry }) => entry.file));
     const exportableMeshPackages = assetMeshPackages.filter((entry) => !modernFiles.has(entry.file));
     const uncooked = cooking.filter(({ entry, state }) => entry.meshKind === "static" && state === "uncooked" && !entry.needsModernConverter);
-    const uncookedMeshDescription = uncooked.filter((entry) => entry.fileVersionUE4 !== undefined &&
-        entry.fileVersionUE4 >= 517 &&
-        entry.fileVersionUE4 <= 522);
-    const legacyRawMesh = uncooked.filter((entry) => entry.fileVersionUE4 === 516);
-    const unsupportedUncooked = uncooked.filter((entry) => !legacyRawMesh.includes(entry) &&
-        (entry.fileVersionUE4 === undefined ||
-            entry.fileVersionUE4 < 517 ||
-            entry.fileVersionUE4 > 522));
+    const uncookedMeshDescription = uncooked.filter((entry) => uncookedMeshRoute("static", entry.fileVersionUE4) === "mesh-description");
+    const unsupportedUncooked = uncooked.filter((entry) => uncookedMeshRoute("static", entry.fileVersionUE4) === undefined);
     if (unsupportedUncooked.length > 0) {
         const versions = [...new Set(unsupportedUncooked.map((entry) => entry.fileVersionUE4 ?? "unknown"))];
-        throw new ImportError("UNREAL_SOURCE_UNSUPPORTED", `${unsupportedUncooked.length} uncooked static-mesh package${unsupportedUncooked.length === 1 ? " uses" : "s use"} UE4 object version ${versions.join(", ")}. The engine-free MeshDescription path is verified for versions 517–522 (UE4.26/4.27-era source assets); refusing to guess at a different binary layout.`);
+        throw new ImportError("UNREAL_SOURCE_UNSUPPORTED", `${unsupportedUncooked.length} uncooked static-mesh package${unsupportedUncooked.length === 1 ? " uses" : "s use"} UE4 object version ${versions.join(", ")}. UE Viewer reads versions below 517 and the engine-free MeshDescription path reads 517–522 (UE4.25–4.27-era source assets); refusing to guess at a different binary layout.`);
     }
-    // Packages share textures, so two exporters can write the same PNG at once and `-nooverwrite`
-    // would see a half-written file as done. Listing is read-only and parallel; exporting is not.
-    log(`Exporting ${exportableMeshPackages.length} static/skeletal mesh packages with UE Viewer…`);
-    const exportFailures = await mapWithConcurrency(exportableMeshPackages, 1, async (entry) => {
+    /** UE Viewer's mesh export, in one place: the primary route for a mesh package, and the retry
+     * for one the modern converter could not read. Returns the reason it failed, or undefined. */
+    const exportMeshWithUmodel = async (entry, out) => {
+        const run = () => runBounded(umodel.path, [`-path=${sourceDir}`, "-export", "-gltf", "-png", "-nooverwrite", `-out=${out}`, entry.selector], { timeoutMs: 1_800_000, maxOutputBytes: 32 * 1024 * 1024 });
         try {
-            let run = await runBounded(umodel.path, [
-                `-path=${sourceDir}`,
-                "-export",
-                "-gltf",
-                "-png",
-                "-nooverwrite",
-                `-out=${raw}`,
-                entry.selector,
-            ], { timeoutMs: 1_800_000, maxOutputBytes: 32 * 1024 * 1024 });
             // UE Viewer occasionally exits non-zero for one package under filesystem pressure even
             // though the same read-only export succeeds immediately afterward. One bounded retry keeps
             // a transient process failure from silently removing a mesh from an otherwise valid pack.
-            if (run.code !== 0) {
-                run = await runBounded(umodel.path, [
-                    `-path=${sourceDir}`,
-                    "-export",
-                    "-gltf",
-                    "-png",
-                    "-nooverwrite",
-                    `-out=${raw}`,
-                    entry.selector,
-                ], { timeoutMs: 1_800_000, maxOutputBytes: 32 * 1024 * 1024 });
-            }
-            if (run.code !== 0) {
-                return { package: entry.package, reason: `UE Viewer export exited ${run.code}.` };
-            }
-            return undefined;
+            let outcome = await run();
+            if (outcome.code !== 0)
+                outcome = await run();
+            return outcome.code === 0 ? undefined : `UE Viewer export exited ${outcome.code}.`;
         }
         catch (error) {
-            return {
-                package: entry.package,
-                reason: error instanceof ToolchainError ? error.message : "UE Viewer export failed.",
-            };
+            return error instanceof ToolchainError ? error.message : "UE Viewer export failed.";
         }
+    };
+    // Packages share textures, so two exporters can write the same PNG at once and `-nooverwrite`
+    // would see a half-written file as done. Listing is read-only and parallel; exporting is not.
+    log(`Exporting ${exportableMeshPackages.length} static/skeletal mesh packages with UE Viewer…`);
+    const exportFailures = await mapWithConcurrency(exportableMeshPackages, 1, async (entry) => {
+        const reason = await exportMeshWithUmodel(entry, raw);
+        return reason ? { package: entry.package, reason } : undefined;
     });
     for (const failure of exportFailures)
         if (failure)
@@ -1366,9 +1448,14 @@ export async function importUnrealDirectory(request) {
                     `-out=${animationRaw}`,
                     entry.selector,
                 ], { timeoutMs: 1_800_000, maxOutputBytes: 32 * 1024 * 1024 });
-                return run.code === 0
-                    ? undefined
-                    : { package: entry.package, reason: `UE Viewer animation export exited ${run.code}.` };
+                if (run.code === 0)
+                    return undefined;
+                // UE Viewer prints its reason last; without it every failure reads the same.
+                const diagnostic = `${run.stdout}\n${run.stderr}`.trim().split(/\r?\n/).filter((line) => line.trim()).at(-1)?.trim();
+                return {
+                    package: entry.package,
+                    reason: `UE Viewer animation export exited ${run.code}.${diagnostic ? ` ${diagnostic.slice(0, 300)}` : ""}`,
+                };
             }
             catch (error) {
                 return {
@@ -1409,6 +1496,8 @@ export async function importUnrealDirectory(request) {
     let uncookedGlbs = new Map();
     let modernConverter;
     let modernGlbs = new Map();
+    /** Meshes UE Viewer exported after the modern converter failed on them. */
+    const recoveredByUmodel = new Set();
     const modernSceneModelSources = [];
     const sceneSources = join(staging, "scene-sources");
     const sceneSourcePaths = new Map();
@@ -1463,26 +1552,41 @@ export async function importUnrealDirectory(request) {
             maxOutputBytes: 64 * 1024 * 1024,
         });
         if (converted.code !== 0) {
-            const diagnosticLines = converted.stderr.trim().split(/\r?\n/);
-            const diagnostic = diagnosticLines.find((line) => line.includes(".usmap")) ??
-                diagnosticLines.find((line) => /exception:/i.test(line)) ??
-                diagnosticLines.at(-1);
-            throw new ToolchainError("UNREAL_TOOL_FAILED", `The modern UE5 asset converter exited ${converted.code}; no partial output was promoted.${diagnostic ? ` ${diagnostic.trim()}` : ""}`);
-        }
-        modernGlbs = await indexGlbs(modernRaw);
-        assets = mergeExported(assets, await indexExported(modernRaw));
-        for (const entry of modernTexturePackages) {
-            const name = basename(entry.package, extname(entry.package));
-            const source = assets.png.get(name);
-            if (source)
-                textureSources.set(entry.file, source);
-            else
-                failed.push({
-                    package: entry.package,
-                    reason: "The modern UE5 texture converter produced no PNG for this package.",
-                });
+            // A static mesh old enough for UE Viewer never needed the modern converter, so one
+            // converter crash must not take a mesh UE Viewer can read down with it. That fallback is only
+            // honest when it covers everything: recovering part of the request would drop the rest
+            // silently, so anything UE Viewer cannot supply reports the converter's own diagnostic.
+            const retryable = modernPackages.filter(({ entry, fileVersionUE4 }) => entry.meshKind === "static" && uncookedMeshRoute("static", fileVersionUE4) === "umodel");
+            const retried = await mapWithConcurrency(retryable, 1, async ({ entry }) => ({
+                entry,
+                reason: await exportMeshWithUmodel(entry, raw),
+            }));
+            const incomplete = retried.find((result) => result.reason !== undefined);
+            if (retryable.length !== modernPackages.length || incomplete) {
+                throw new ToolchainError("UNREAL_TOOL_FAILED", `The modern UE5 asset converter exited ${converted.code}; no partial output was promoted. ${describeModernFailure(converted.stderr, modernPackages)}`);
+            }
+            for (const result of retried) {
+                recoveredByUmodel.add(basename(result.entry.package, extname(result.entry.package)));
+            }
+            assets = mergeExported(assets, await indexExported(raw));
+            warnings.push(`The modern UE5 asset converter exited ${converted.code}; UE Viewer decoded ${recoveredByUmodel.size} static mesh package${recoveredByUmodel.size === 1 ? "" : "s"} it could read itself.`);
+        }
+        else {
+            modernGlbs = await indexGlbs(modernRaw);
+            assets = mergeExported(assets, await indexExported(modernRaw));
+            for (const entry of modernTexturePackages) {
+                const name = basename(entry.package, extname(entry.package));
+                const source = assets.png.get(name);
+                if (source)
+                    textureSources.set(entry.file, source);
+                else
+                    failed.push({
+                        package: entry.package,
+                        reason: "The modern UE5 texture converter produced no PNG for this package.",
+                    });
+            }
+            warnings.push(`Decoded ${modernAssetCount} requested modern UE5 asset package${modernAssetCount === 1 ? "" : "s"} without Unreal Engine.`);
         }
-        warnings.push(`Decoded ${modernAssetCount} requested modern UE5 asset package${modernAssetCount === 1 ? "" : "s"} without Unreal Engine.`);
     }
     if (modernMapPackages.length > 0) {
         modernConverter ??= request.modernConverter ?? (await ensureModernConverter(environment, log));
@@ -1755,7 +1859,9 @@ export async function importUnrealDirectory(request) {
         }
         warnings.push(`Decoded ${paperPackages.length} requested Paper2D package${paperPackages.length === 1 ? "" : "s"} without Unreal Engine.`);
     }
-    const modernNames = new Set(modernPackages.map(({ entry }) => basename(entry.package, extname(entry.package))));
+    const modernNames = new Set(modernPackages
+        .map(({ entry }) => basename(entry.package, extname(entry.package)))
+        .filter((name) => !recoveredByUmodel.has(name)));
     const promotion = await mkdtemp(join(promotionParent, ".threenative-import-"));
     // Only a verified entitlement earns a copyright line. A local pack whose licence nobody
     // checked stays blank so the game's asset health check keeps saying "unknown".