topotopo 0.1.0 → 0.2.0

package/dist/host/export_scene.py

@@ -0,0 +1,200 @@
+"""Headless .blend -> static GLB for TopoTopo. Never saves or overwrites the .blend."""
+import json
+import os
+import copy
+import struct
+import sys
+import traceback
+from collections import Counter
+
+import bpy
+
+
+def static_copies():
+    """Bake evaluated geometry/transforms into copies; keep linked, unmodified meshes shared."""
+    scenes = list(bpy.data.scenes)
+    original_scene = bpy.context.window.scene
+    objects = list(dict.fromkeys(obj for scene in scenes for obj in scene.objects))
+    print("TopoTopo objects before export: " + json.dumps(dict(sorted(Counter(o.type for o in objects).items()))), flush=True)
+
+    # Excluded collections must be evaluated too, even though visibility is not an export filter.
+    def enable(layer):
+        layer.exclude = False
+        layer.hide_viewport = False
+        for child in layer.children:
+            enable(child)
+
+    for collection in bpy.data.collections:
+        collection.hide_viewport = False
+    for scene in scenes:
+        for layer in scene.view_layers:
+            enable(layer.layer_collection)
+    for obj in objects:
+        obj.hide_viewport = False
+        obj.hide_render = False
+        obj.hide_set(False)
+
+    copies, worlds, names, data_copies = {}, {}, {}, {}
+    for scene in scenes:
+        bpy.context.window.scene = scene
+        bpy.context.view_layer.update()
+        depsgraph = bpy.context.evaluated_depsgraph_get()
+        for obj in list(scene.objects):
+            if obj in copies:
+                continue
+            evaluated = obj.evaluated_get(depsgraph)
+            worlds[obj] = evaluated.matrix_world.copy()
+            names[obj] = obj.name
+            if obj.type in {'MESH', 'CURVE', 'SURFACE', 'FONT'}:
+                if obj.type == 'MESH' and not obj.modifiers and not obj.data.shape_keys:
+                    if obj.data not in data_copies:
+                        data_copies[obj.data] = obj.data.copy()
+                    mesh = data_copies[obj.data]
+                else:
+                    # Unlike the exporter’s curve path, this also retains unbevelled curve edges.
+                    mesh = bpy.data.meshes.new_from_object(evaluated, preserve_all_data_layers=True, depsgraph=depsgraph)
+                    if mesh is None:
+                        raise RuntimeError("Could not evaluate " + obj.name + " as a mesh")
+                    for key in obj.data.keys():
+                        if key not in mesh:
+                            mesh[key] = obj.data[key]
+                copy = bpy.data.objects.new(obj.name, mesh)
+                for key in obj.keys():
+                    copy[key] = obj[key]
+                for index, slot in enumerate(obj.material_slots):
+                    if index < len(copy.material_slots) and slot.link == 'OBJECT':
+                        copy.material_slots[index].link = 'OBJECT'
+                        copy.material_slots[index].material = slot.material
+            else:
+                copy = obj.copy()
+                copy.animation_data_clear()
+                copy.constraints.clear()
+            copies[obj] = copy
+
+    # Relink copies in the existing collection hierarchy; originals are removed only from this process's memory.
+    parents = {obj: obj.parent for obj in objects}
+    for obj, copy in copies.items():
+        for collection in obj.users_collection:
+            collection.objects.link(copy)
+        copy.parent = copies.get(parents[obj])
+        copy.parent_type = 'OBJECT'
+        copy.matrix_parent_inverse.identity()
+    def depth(obj):
+        return 0 if parents[obj] not in parents else 1 + depth(parents[obj])
+    for obj in sorted(objects, key=depth):
+        copies[obj].matrix_world = worlds[obj]
+    desired_names = [(copies[obj], names[obj]) for obj in objects]
+    for obj in objects:
+        bpy.data.objects.remove(obj, do_unlink=True)
+    for copy, name in desired_names:
+        copy.name = name
+    for scene in scenes:
+        scene.unit_settings.scale_length = 1.0  # apply units once to the exported roots, including collection instances
+    bpy.context.window.scene = original_scene
+    bpy.context.view_layer.update()
+    print("TopoTopo objects after conversion: " + json.dumps(dict(sorted(Counter(o.type for o in copies.values()).items()))), flush=True)
+
+
+def apply_scene_units(output, units):
+    """Scale GLB roots, never prototypes: a collection instance must not apply the unit factor twice."""
+    if all(value == 1.0 for value in units.values()):
+        return
+    with open(output, 'rb') as stream:
+        data = stream.read()
+    magic, version, length = struct.unpack_from('<4sII', data)
+    if magic != b'glTF' or version != 2 or length != len(data):
+        raise RuntimeError('Exporter did not produce a valid GLB container')
+    chunks, offset = [], 12
+    while offset < length:
+        size, kind = struct.unpack_from('<II', data, offset)
+        chunks.append((kind, data[offset + 8:offset + 8 + size]))
+        offset += 8 + size
+    document = json.loads(chunks[0][1])
+    nodes = document.get('nodes', [])
+    originals = copy.deepcopy(nodes)
+    requests = {}
+    for scene in document.get('scenes', []):
+        factor = units.get(scene.get('name'), 1.0)
+        for index, root in enumerate(scene.get('nodes', [])):
+            requests.setdefault(root, {}).setdefault(factor, []).append((scene, index))
+
+    def clone_tree(index):
+        node = copy.deepcopy(originals[index])
+        if 'children' in node:
+            node['children'] = [clone_tree(child) for child in node['children']]
+        nodes.append(node)
+        return len(nodes) - 1
+
+    for root, factors in requests.items():
+        for variant, (factor, references) in enumerate(factors.items()):
+            # Usually roots are unique per scene. If scenes share a root but use different units, share meshes
+            # and copy only the node hierarchy so one scene's conversion cannot alter the other.
+            target = root if variant == 0 else clone_tree(root)
+            for scene, index in references:
+                scene['nodes'][index] = target
+            node = nodes[target]
+            if 'matrix' in node:
+                node['matrix'] = [value * factor if i % 4 != 3 else value for i, value in enumerate(node['matrix'])]
+            else:
+                node['translation'] = [value * factor for value in node.get('translation', [0, 0, 0])]
+                node['scale'] = [value * factor for value in node.get('scale', [1, 1, 1])]
+    payload = json.dumps(document, separators=(',', ':'), ensure_ascii=False).encode('utf8')
+    chunks[0] = (chunks[0][0], payload + b' ' * (-len(payload) % 4))
+    body = b''.join(struct.pack('<II', len(payload), kind) + payload for kind, payload in chunks)
+    with open(output, 'wb') as stream:
+        stream.write(struct.pack('<4sII', b'glTF', 2, 12 + len(body)) + body)
+    print('TopoTopo scene unit scales: ' + json.dumps(units, sort_keys=True), flush=True)
+
+
+def main():
+    args = sys.argv[sys.argv.index('--') + 1:] if '--' in sys.argv else []
+    if len(args) != 1:
+        raise RuntimeError("Expected one output .glb path after --")
+    output = os.path.abspath(args[0])
+    units = {scene.name: scene.unit_settings.scale_length for scene in bpy.data.scenes}
+    static_copies()
+    properties = bpy.ops.export_scene.gltf.get_rna_type().properties
+    kwargs, missing = {}, []
+
+    def option(identifiers, value):
+        # Option identifiers come from RNA, not a version assumption. Aliases cover older exporters.
+        for identifier in identifiers:
+            if identifier in properties:
+                kwargs[identifier] = value
+                return
+        missing.append('/'.join(identifiers))
+
+    for identifiers, value in [
+        (('filepath',), output), (('export_format',), 'GLB'), (('check_existing',), False),
+        (('export_yup',), True), (('export_apply',), True),
+        (('use_mesh_edges', 'export_loose_edges'), True),
+        (('use_mesh_vertices', 'export_loose_points'), True),
+        (('export_extras',), True), (('export_materials',), 'EXPORT'),
+        (('export_image_format',), 'AUTO'), (('export_keep_originals',), False),
+        (('export_cameras',), False), (('export_lights',), False),
+        (('export_animations',), False), (('export_morph',), False), (('export_skins',), False),
+        (('use_selection',), False), (('use_visible',), False), (('use_renderable',), False),
+        (('use_active_collection',), False), (('use_active_collection_with_nested',), False),
+        (('use_active_scene',), False), (('collection',), ''),
+        (('export_hierarchy_full_collections',), True), (('export_hierarchy_flatten_objs',), False),
+        (('export_gpu_instances',), False), (('will_save_settings',), False),
+    ]:
+        option(identifiers, value)
+    print("TopoTopo exporter options set: " + json.dumps(kwargs, sort_keys=True), flush=True)
+    print("TopoTopo exporter options missing: " + json.dumps(missing), flush=True)
+    result = bpy.ops.export_scene.gltf(**kwargs)
+    if 'FINISHED' not in result or not os.path.isfile(output):
+        raise RuntimeError("glTF export did not finish: " + str(result))
+    apply_scene_units(output, units)
+
+
+if __name__ == '__main__':
+    try:
+        main()
+    except Exception:
+        traceback.print_exc()
+        print("TopoTopo .blend conversion failed", file=sys.stderr, flush=True)
+        # Blender otherwise reports many Python errors with exit code zero.
+        sys.stdout.flush()
+        sys.stderr.flush()
+        os._exit(1)

package/dist/host/node.d.ts

@@ -5,6 +5,8 @@ import type { NodePoolOptions } from './pool.ts';
  * sources (src/host) and for the built package (dist/host): both sit two levels below the package root.
  */
 export declare const SHELF_DIR: string;
+/** Executable lookup shared by conversion and integration tests. An unavailable installation returns null. */
+export declare function findBlender(): Promise<string | null>;
 export interface NodeHostOptions {
     baseDir?: string;
     /** shelf directory (default SHELF_DIR), or false for an engine without the shelf */
@@ -16,5 +18,11 @@ export interface NodeHostOptions {
     workers?: number;
     /** further pool settings for a host with its own pool (idleMs, minElements) */
     pool?: Partial<NodePoolOptions>;
+    /** Optional diagnostics for actual Blender conversions (not version probes or cache hits). */
+    onBlenderRun?: (result: {
+        executable: string;
+        durationMs: number;
+        output: string;
+    }) => void;
 }
 export declare function createNodeHost(opts?: NodeHostOptions): Host;

package/dist/host/node.js

@@ -3,6 +3,8 @@ import vm from 'node:vm';
 import fs from 'node:fs';
 import path from 'node:path';
 import os from 'node:os';
+import { execFile } from 'node:child_process';
+import { createHash } from 'node:crypto';
 import { fileURLToPath } from 'node:url';
 import { createNodePool, poolOptionsFromEnv } from './pool.js';
 import { REALM_CONTROL, SCRIPT_REALM_SETUP } from '../core/snippet-lib.js';
@@ -33,11 +35,55 @@ function filterRealmRejections() {
  * sources (src/host) and for the built package (dist/host): both sit two levels below the package root.
  */
 export const SHELF_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../shelf');
+/** No shell: executable paths and file names are passed as separate arguments on every platform. */
+function runBlender(executable, args, timeout) {
+    return new Promise((resolve, reject) => {
+        execFile(executable, args, { timeout, windowsHide: true, maxBuffer: 16 * 1024 * 1024, killSignal: 'SIGKILL' }, (error, stdout, stderr) => {
+            const output = `${stdout}\n${stderr}`.trim();
+            if (error) {
+                const tail = output.split(/\r?\n/).slice(-40).join('\n');
+                const reason = error.message.split(/\r?\n/)[0]; // execFile's message may itself contain all of stderr
+                reject(new Error(`Blender failed (exit code ${error.code ?? 'none'}${error.signal ? `, signal ${error.signal}` : ''}${error.killed ? ', timed out or killed' : ''}): ${reason}\n${tail}`));
+            }
+            else
+                resolve(output);
+        });
+    });
+}
+/** Executable lookup shared by conversion and integration tests. An unavailable installation returns null. */
+export async function findBlender() {
+    const candidates = [process.env.TOPOTOPO_BLENDER, 'blender'].filter((p) => !!p);
+    const foundation = path.join(process.env.ProgramFiles ?? 'C:\\Program Files', 'Blender Foundation');
+    try {
+        const versions = fs.readdirSync(foundation, { withFileTypes: true }).filter((e) => e.isDirectory() && /^Blender /i.test(e.name));
+        versions.sort((a, b) => b.name.localeCompare(a.name, 'en', { numeric: true }));
+        candidates.push(...versions.map((e) => path.join(foundation, e.name, 'blender.exe')));
+    }
+    catch { /* normal on non-Windows hosts, or without a Foundation install */ }
+    candidates.push('C:\\Program Files (x86)\\Steam\\steamapps\\common\\Blender\\blender.exe', '/Applications/Blender.app/Contents/MacOS/Blender', '/usr/bin/blender', '/snap/bin/blender');
+    for (const candidate of new Set(candidates)) {
+        try {
+            await runBlender(candidate, ['--version'], 10_000);
+            return candidate;
+        }
+        catch { /* try the next installation */ }
+    }
+    return null;
+}
+function blenderScript() {
+    for (const relative of ['./export_scene.py', '../../integrations/blender/export_scene.py']) {
+        const file = fileURLToPath(new URL(relative, import.meta.url));
+        if (fs.existsSync(file))
+            return file;
+    }
+    throw new Error('Blender conversion script export_scene.py is missing from this TopoTopo installation');
+}
 /** opts.shelf: shelf directory (default SHELF_DIR), or false for an engine without the shelf. */
 /** The process-wide pool of hosts created with the environment's settings (created lazily: no thread until used). */
 let envPool = null;
 export function createNodeHost(opts = {}) {
     const baseDir = opts.baseDir ?? process.cwd();
+    const conversions = new Map();
     let pool;
     if (opts.workers === undefined && !opts.pool) {
         if (envPool === null)
@@ -202,6 +248,74 @@ export function createNodeHost(opts = {}) {
                 return 'missing';
             }
         },
+        isDirectory(p) {
+            try {
+                return fs.statSync(this.resolvePath(p)).isDirectory();
+            }
+            catch {
+                return false;
+            }
+        },
+        listDir(p) {
+            const dir = this.resolvePath(p);
+            return fs.readdirSync(dir, { withFileTypes: true }).filter((e) => e.isFile() || (e.isSymbolicLink() && fs.statSync(path.join(dir, e.name)).isFile()))
+                .map((e) => e.name).sort((a, b) => a.localeCompare(b, 'en'));
+        },
+        async convertFile(p, target) {
+            if (target !== 'glb')
+                throw new Error(`unsupported conversion target "${target}"`);
+            const absolute = this.resolvePath(p);
+            if (/^https?:\/\//.test(absolute))
+                throw new Error('.blend conversion needs a local file path');
+            const stamp = this.fileStamp(absolute);
+            if (stamp === 'missing')
+                throw new Error(`cannot read "${absolute}": file is missing`);
+            const key = createHash('sha256').update(JSON.stringify([absolute, stamp])).digest('hex');
+            const cacheDir = path.resolve(baseDir, '.cache', 'blend');
+            const cached = path.join(cacheDir, `${key}.glb`);
+            // In-flight conversions of the same source share a process, including concurrent file nodes.
+            let job = conversions.get(key);
+            if (!job) {
+                job = (async () => {
+                    try {
+                        return new Uint8Array(await fs.promises.readFile(cached));
+                    }
+                    catch (e) {
+                        if (e.code !== 'ENOENT')
+                            throw e;
+                    }
+                    const executable = await findBlender();
+                    if (!executable)
+                        throw new Error('Blender was not found. Set TOPOTOPO_BLENDER to its executable, or install Blender from https://www.blender.org/download/');
+                    const seconds = Number(process.env.TOPOTOPO_BLENDER_TIMEOUT ?? 300);
+                    if (!Number.isFinite(seconds) || seconds <= 0 || seconds * 1000 > 2_147_483_647)
+                        throw new Error('TOPOTOPO_BLENDER_TIMEOUT must be a positive number of seconds (default 300)');
+                    await fs.promises.mkdir(cacheDir, { recursive: true });
+                    const tempDir = await fs.promises.mkdtemp(path.join(cacheDir, 'convert-'));
+                    try {
+                        const outputFile = path.join(tempDir, 'scene.glb');
+                        const start = performance.now();
+                        // Factory settings keep user add-ons/startup handlers out of deterministic batch conversions.
+                        const output = await runBlender(executable, ['--factory-startup', '--disable-autoexec', '-b', absolute, '--python', blenderScript(), '--', outputFile], Math.ceil(seconds * 1000));
+                        opts.onBlenderRun?.({ executable, durationMs: performance.now() - start, output });
+                        const bytes = new Uint8Array(await fs.promises.readFile(outputFile));
+                        await fs.promises.rename(outputFile, cached);
+                        return bytes;
+                    }
+                    finally {
+                        await fs.promises.rm(tempDir, { recursive: true, force: true });
+                    }
+                })();
+                conversions.set(key, job);
+            }
+            try {
+                return await job;
+            }
+            finally {
+                if (conversions.get(key) === job)
+                    conversions.delete(key);
+            }
+        },
         compile(source) {
             if (source === 'globalThis')
                 return realmGlobal;

package/dist/host/types.d.ts

@@ -95,6 +95,11 @@ export interface Host {
     /** A string that changes when the file changes (mtime+size), for cache keys. */
     fileStamp(path: string): string;
     resolvePath(path: string): string;
+    /** Optional local-directory support. listDir returns immediate file names only (no subdirectories). */
+    isDirectory?(path: string): boolean;
+    listDir?(path: string): string[];
+    /** Convert a local source file to GLB (Node: .blend through headless Blender). Absent in browser/kernel hosts. */
+    convertFile?(path: string, target: 'glb'): Promise<Uint8Array>;
     /**
      * Compile JS source inside the scripting realm (which has the snippet library
      * installed) and return the resulting value (usually a function). The realm is set up with

package/dist/inspect.js

@@ -130,8 +130,11 @@ export function summarize(g) {
         if (!t.map.size)
             continue;
         const items = [...t.map].map(([k, a]) => {
-            if (cls === 'detail')
-                return `${k} ${typeCode(a)} = ${fmtAttr(a, 0)}`;
+            if (cls === 'detail') {
+                // the summary stays readable: long values (e.g. the materials JSON) are cut; detail "attributes" shows them whole
+                const v = fmtAttr(a, 0);
+                return `${k} ${typeCode(a)} = ${v.length > 120 ? `${v.slice(0, 100)}… (${v.length} chars)` : v}`;
+            }
             if (a.isArray)
                 return `${k} ${a.type} (${arrayLens(a, t.count)})`;
             if (a.type === 'string') {

package/dist/io/common.d.ts

@@ -22,6 +22,12 @@ export interface ExportOptions {
     quantize?: boolean;
     /** glTF: write materials from the prim "material" attribute and the detail table (default true) */
     materials?: boolean;
+    /**
+     * glTF: a named material's base colour is the factor even where Cd varies (COLOR_0 is still written). For engines
+     * whose default materials ignore vertex colours (Unreal Interchange), where glTF's white factor × COLOR_0 imports
+     * white; spec viewers then show factor × COLOR_0. The unreal_landscape bundle sets it. Default false.
+     */
+    materialColorWins?: boolean;
     /** glTF: polylines as one LINE_STRIP per curve ('strip', default) or one LINES primitive ('lines') */
     lineMode?: 'strip' | 'lines';
     /** csv / points JSON: point attributes to write (names or a comma/space separated list; default all) */

package/dist/io/export.js

@@ -168,6 +168,9 @@ export function exportPLY(g, o = {}) {
 const usdName = (s) => (s.replace(/[^A-Za-z0-9_]/g, '_').replace(/^(\d)/, '_$1') || 'mesh');
 const fmt = (n) => (Number.isInteger(n) ? String(n) : n.toFixed(6).replace(/0+$/, '').replace(/\.$/, ''));
 const tup = (a, k, n) => '(' + Array.from({ length: n }, (_, c) => fmt(a[k * n + c])).join(', ') + ')';
+/** A colour tuple for USD: TopoTopo colours are display values, USD colour inputs and primvars are linear. */
+const linear = (c) => (c <= 0.04045 ? c / 12.92 : ((c + 0.055) / 1.055) ** 2.4);
+const tupColor = (a, k, n) => '(' + Array.from({ length: n }, (_, c) => fmt(linear(a[k * n + c]))).join(', ') + ')';
 /**
  * USDA stage: meshes, curves and points under /root, split by prim name. Packed instances (instancing 'packed', the
  * default) become one PointInstancer /root/instances whose prototypes (one per packed source, under its
@@ -226,10 +229,10 @@ function usdaMaterials(table, o, out) {
         const I = '            ';
         out.push(`        def Material "${id}"`, '        {', `            token outputs:surface.connect = <${mp}/surface.outputs:surface>`, '');
         out.push(`${I}def Shader "surface"`, `${I}{`, `${I}    uniform token info:id = "UsdPreviewSurface"`);
-        out.push(`${I}    color3f inputs:diffuseColor = ${tup(m.baseColor, 0, 3)}`);
+        out.push(`${I}    color3f inputs:diffuseColor = ${tupColor(m.baseColor, 0, 3)}`);
         if (tex.baseColor)
             out.push(`${I}    color3f inputs:diffuseColor.connect = <${mp}/base_color_map.outputs:rgb>`);
-        out.push(`${I}    color3f inputs:emissiveColor = ${tup(m.emissive, 0, 3)}`);
+        out.push(`${I}    color3f inputs:emissiveColor = ${tupColor(m.emissive, 0, 3)}`);
         out.push(`${I}    float inputs:metallic = ${fmt(m.metallic)}`);
         if (tex.orm)
             out.push(`${I}    float inputs:metallic.connect = <${mp}/orm_map.outputs:b>`);
@@ -343,7 +346,7 @@ function usdaInstancer(g, o, uniq, out, mats = null) {
         if ((a.name === 'Cd' || a.name === 'Alpha') && t === 'float' && num.every((v) => v.every((x) => x === 1)))
             continue;
         if (a.name === 'Cd' && t === 'float' && sz >= 3) {
-            out.push(`        color3f[] primvars:displayColor = [${num.map((v) => tup(v, 0, 3)).join(', ')}] (`, ...vertex);
+            out.push(`        color3f[] primvars:displayColor = [${num.map((v) => tupColor(v, 0, 3)).join(', ')}] (`, ...vertex);
             continue;
         }
         if (a.name === 'Alpha' && t === 'float' && sz === 1) {
@@ -419,7 +422,7 @@ function usdaPrims(g, o, uniq, out, mats = null) {
                     const c = cd.attr.get(i);
                     return Array.isArray(c) ? c.slice(0, 3) : [c, c, c];
                 });
-                out.push(`        color3f[] primvars:displayColor = [${Array.from({ length: cols.length / 3 }, (_, k) => tup(cols, k, 3)).join(', ')}] (`, '            interpolation = "faceVarying"', '        )');
+                out.push(`        color3f[] primvars:displayColor = [${Array.from({ length: cols.length / 3 }, (_, k) => tupColor(cols, k, 3)).join(', ')}] (`, '            interpolation = "faceVarying"', '        )');
             }
             if (o.custom)
                 for (const [n, a] of geo.points.map) {

package/dist/io/gltf.d.ts

@@ -32,7 +32,7 @@ export interface GLTFImportOptions {
     /** EXT_mesh_gpu_instancing: 'packed' (default) = packed prims sharing one source per mesh; 'expand' = real geometry */
     instancing?: 'packed' | 'expand';
 }
-/** Geometry from a gltf-transform Document (all scenes' nodes flattened into world space). */
+/** Geometry from the default (or first) scene, traversed in scene order with world transforms. */
 export declare function documentToGeometry(doc: Document, warn?: (m: string) => void, instancing?: 'packed' | 'expand', base?: string): Geometry;
 /** Import GLB or .gltf bytes (external buffers through opts.readFile). */
 export declare function importGLTFAsync(bytes: Uint8Array, opts?: GLTFImportOptions): Promise<Geometry>;