usd-gltf 0.1.0 → 0.2.0

package/src/limits.js

@@ -0,0 +1,101 @@
+// Opt-in resource limits for a conversion: decoded data, serialized output and cancellation.
+
+/**
+ * Thrown when a conversion would go over one of its limits. The message and fields describe sizes
+ * only, never input data or file names.
+ */
+export class ConversionLimitError extends Error {
+
+	/**
+	 * @param {{ stage: string, resource: 'decoded'|'output', limit: number, attempted: number }} info
+	 */
+	constructor( { stage, resource, limit, attempted } ) {
+
+		super( `usd-gltf: ${ stage } needs ${ attempted } ${ resource } bytes, over the limit of ${ limit }` );
+		this.name = 'ConversionLimitError';
+		this.code = 'USD_GLTF_LIMIT';
+		this.stage = stage;
+		this.resource = resource;
+		this.limit = limit;
+		this.attempted = attempted;
+
+	}
+
+}
+
+/**
+ * Tracks one conversion against its limits. Readers and converters charge decoded bytes before
+ * allocating them; writers report their size before growing.
+ */
+export class Budget {
+
+	/**
+	 * @param {{ maxDecodedBytes?: number, maxOutputBytes?: number, signal?: AbortSignal }} [options]
+	 */
+	constructor( { maxDecodedBytes = Infinity, maxOutputBytes = Infinity, signal = null } = {} ) {
+
+		this.maxDecodedBytes = maxDecodedBytes;
+		this.maxOutputBytes = maxOutputBytes;
+		this.signal = signal;
+		this.decodedBytes = 0;
+
+	}
+
+	/**
+	 * Adds to the running total of decoded data: accessors, arrays, decompressed sections and images.
+	 * @param {number} bytes
+	 * @param {string} stage
+	 */
+	decode( bytes, stage ) {
+
+		checkSize( bytes, stage );
+		this.check();
+		this.decodedBytes += bytes;
+		if ( this.decodedBytes > this.maxDecodedBytes ) throw new ConversionLimitError( { stage, resource: 'decoded', limit: this.maxDecodedBytes, attempted: this.decodedBytes } );
+
+	}
+
+	/**
+	 * Checks the size an output buffer is about to reach.
+	 * @param {number} bytes
+	 * @param {string} stage
+	 */
+	output( bytes, stage ) {
+
+		checkSize( bytes, stage );
+		if ( bytes > this.maxOutputBytes ) throw new ConversionLimitError( { stage, resource: 'output', limit: this.maxOutputBytes, attempted: bytes } );
+
+	}
+
+	// Throws the signal's reason once the conversion is cancelled.
+	check() {
+
+		this.signal?.throwIfAborted();
+
+	}
+
+}
+
+/**
+ * A size computed from counts in a file must be a whole number of bytes that fits in an array.
+ * @param {number} bytes
+ * @param {string} stage
+ */
+export function checkSize( bytes, stage ) {
+
+	if ( ! Number.isSafeInteger( bytes ) || bytes < 0 ) throw new Error( `usd-gltf: invalid size in ${ stage }` );
+
+}
+
+/**
+ * For catch blocks that skip one failed asset and carry on: rethrows what must end the whole
+ * conversion instead, a limit error or cancellation.
+ * @param {unknown} error
+ * @param {Budget} [budget]
+ */
+export function rethrowFatal( error, budget ) {
+
+	budget?.check();
+	if ( error instanceof ConversionLimitError ) throw error;
+
+}

package/src/usd/stage.js

@@ -5,6 +5,7 @@
 import { AssetPath, ListOp } from './model.js';
 import { readUSDA } from './usda/reader.js';
 import { readZip } from './usdz.js';
+import { Budget, rethrowFatal } from '../limits.js';
 
 const decoder = new TextDecoder();
 
@@ -80,9 +81,10 @@ export function resolveAssetPath( anchor, assetPath ) {
 // Reads bytes for identifiers, opening packages (including nested ones) through the user resolver.
 export class AssetReader {
 
-	constructor( resolver ) {
+	constructor( resolver, budget = new Budget() ) {
 
 		this.resolver = resolver;
+		this.budget = budget;
 		this.packages = new Map();
 		this.files = new Map();
 
@@ -123,7 +125,7 @@ export class AssetReader {
 
 	async openPackage( identifier ) {
 
-		if ( ! this.packages.has( identifier ) ) this.packages.set( identifier, readZip( await this.read( identifier ) ) );
+		if ( ! this.packages.has( identifier ) ) this.packages.set( identifier, readZip( await this.read( identifier ), { budget: this.budget } ) );
 		return this.packages.get( identifier );
 
 	}
@@ -140,7 +142,7 @@ export class AssetReader {
 
 }
 
-async function parseLayer( bytes, identifier ) {
+async function parseLayer( bytes, identifier, budget ) {
 
 	const format = detectLayerFormat( bytes );
 	if ( format === 'usda' ) return readUSDA( bytes, identifier );
@@ -148,7 +150,7 @@ async function parseLayer( bytes, identifier ) {
 	if ( format === 'usdc' ) {
 
 		const { readUSDC } = await import( './usdc/reader.js' );
-		return readUSDC( bytes, identifier );
+		return readUSDC( bytes, identifier, { budget } );
 
 	}
 
@@ -185,14 +187,15 @@ async function loadLayers( reader, rootIdentifier ) {
 
 		if ( detectLayerFormat( bytes ) === 'usdz' ) {
 
-			reader.addPackage( identifier, readZip( bytes ) );
+			reader.addPackage( identifier, readZip( bytes, { budget: reader.budget } ) );
 			const root = await load( await reader.packageRoot( identifier ) );
 			layers.set( identifier, root );
 			return root;
 
 		}
 
-		const layer = await parseLayer( bytes, identifier );
+		reader.budget.check();
+		const layer = await parseLayer( bytes, identifier, reader.budget );
 		layers.set( identifier, layer );
 
 		const pending = [];
@@ -216,6 +219,7 @@ async function loadLayers( reader, rootIdentifier ) {
 
 			} catch ( e ) {
 
+				rethrowFatal( e, reader.budget );
 				console.warn( `usd-gltf: ${ e.message }` );
 
 			}
@@ -862,16 +866,16 @@ export class Stage {
 /**
  * Opens and composes a stage.
  * @param {Uint8Array} bytes usda, usdc or usdz
- * @param {{ identifier?: string, resolver?: ( identifier: string ) => Promise<Uint8Array>|Uint8Array, variants?: object }} [options]
+ * @param {{ identifier?: string, resolver?: ( identifier: string ) => Promise<Uint8Array>|Uint8Array, variants?: object, budget?: import('../limits.js').Budget }} [options]
  * @returns {Promise<Stage>}
  */
-export async function openStage( bytes, { identifier, resolver = null, variants = {} } = {} ) {
+export async function openStage( bytes, { identifier, resolver = null, variants = {}, budget = new Budget() } = {} ) {
 
 	const format = detectLayerFormat( bytes );
 	if ( ! format ) throw new Error( 'usd-gltf: input is not usda, usdc or usdz' );
 
 	identifier ??= 'root.' + format;
-	const reader = new AssetReader( resolver );
+	const reader = new AssetReader( resolver, budget );
 	reader.files.set( identifier, bytes );
 
 	const { root, layers } = await loadLayers( reader, identifier );

package/src/usd/usda/writer.js

@@ -357,11 +357,26 @@ function writePrim( out, spec, indent ) {
 
 /**
  * @param {import('../model.js').Layer} layer
+ * @param {{ budget?: import('../../limits.js').Budget }} [options] its output limit is checked as lines are written
  * @returns {string}
  */
-export function writeUSDA( layer ) {
+export function writeUSDA( layer, { budget = null } = {} ) {
 
 	const out = [ '#usda 1.0' ];
+
+	if ( budget ) {
+
+		let size = 0;
+		const push = out.push;
+		out.push = function ( ...lines ) {
+
+			for ( const line of lines ) size += line.length + 1;
+			budget.output( size, 'usda' );
+			return push.apply( this, lines );
+
+		};
+
+	}
 	const meta = formatMetadataEntries( layer.metadata, '    ', 'layer' );
 	if ( meta.length ) out.push( '(', ...meta, ')' );
 	out.push( '' );

package/src/usd/usdc/reader.js

@@ -4,6 +4,7 @@
 import { AssetPath, DICT_TYPES, Layer, ListOp, PrimSpec, PropertySpec, Reference } from '../model.js';
 import { decompressFast } from './lz4.js';
 import { decompressInts32, decompressInts64 } from './intcompression.js';
+import { Budget, checkSize } from '../../limits.js';
 import { PERMISSIONS, POD, SCALAR_SIZE, SPEC, SPECIFIERS, TYPE, TYPE_NAMES, halfArrayToFloat32, halfToFloat, isMatrixType, isVecType } from './types.js';
 
 const IS_ARRAY = 0x80000000;
@@ -42,9 +43,10 @@ const V0_8_0 = version( 0, 8, 0 );
 
 class CrateReader {
 
-	constructor( bytes ) {
+	constructor( bytes, budget = new Budget() ) {
 
 		this.bytes = bytes;
+		this.budget = budget;
 		this.view = new DataView( bytes.buffer, bytes.byteOffset, bytes.byteLength );
 		this.pos = 0;
 		this.timesCache = new Map();
@@ -136,7 +138,7 @@ class CrateReader {
 
 	take( n ) {
 
-		if ( this.pos + n > this.bytes.length ) throw new Error( 'usd-gltf: truncated usdc file' );
+		if ( ! Number.isSafeInteger( n ) || n < 0 || this.pos + n > this.bytes.length ) throw new Error( 'usd-gltf: truncated usdc file' );
 		const b = this.bytes.subarray( this.pos, this.pos + n );
 		this.pos += n;
 		return b;
@@ -145,8 +147,37 @@ class CrateReader {
 
 	compressedInts( n ) {
 
-		const size = this.u64();
-		return decompressInts32( this.take( size ), n );
+		const data = this.take( this.u64() );
+		this.decodedInts( n, 4, data );
+		return decompressInts32( data, n );
+
+	}
+
+	// Sizes come from the file, so they are checked before allocating. Compressed integers take at
+	// least 2 bits each before LZ4, which shrinks data at most 255 times.
+	decodedInts( n, size, data ) {
+
+		checkSize( n * size, 'usdc' );
+		if ( n > data.length * 1024 + 64 ) throw new Error( 'usd-gltf: corrupt compressed integers in usdc file' );
+		this.budget.decode( n * size * 2, 'usdc' ); // the encoded buffer and the decoded integers
+
+	}
+
+	lz4( data, size ) {
+
+		checkSize( size, 'usdc' );
+		if ( size > data.length * 256 + 64 ) throw new Error( 'usd-gltf: corrupt compressed data in usdc file' );
+		this.budget.decode( size, 'usdc' );
+		return decompressFast( data, size );
+
+	}
+
+	// count fixed size records at this.pos, which must all be in the file
+	records( count, size ) {
+
+		checkSize( count * size, 'usdc' );
+		if ( this.pos + count * size > this.bytes.length ) throw new Error( 'usd-gltf: truncated usdc file' );
+		this.budget.decode( count * size, 'usdc' );
 
 	}
 
@@ -176,7 +207,7 @@ class CrateReader {
 
 			const uncompressedSize = this.u64();
 			const compressedSize = this.u64();
-			chars = decompressFast( this.take( compressedSize ), uncompressedSize );
+			chars = this.lz4( this.take( compressedSize ), uncompressedSize );
 
 		}
 
@@ -213,6 +244,7 @@ class CrateReader {
 
 		if ( this.version < V0_4_0 ) {
 
+			this.records( count, 16 );
 			this.fieldTokens = new Uint32Array( count );
 			this.fieldReps = new Uint32Array( count * 2 );
 
@@ -229,7 +261,7 @@ class CrateReader {
 
 			this.fieldTokens = new Uint32Array( this.compressedInts( count ).buffer );
 			const repsSize = this.u64();
-			const reps = decompressFast( this.take( repsSize ), count * 8 );
+			const reps = this.lz4( this.take( repsSize ), count * 8 );
 			this.fieldReps = new Uint32Array( reps.buffer, 0, count * 2 );
 
 		}
@@ -245,6 +277,7 @@ class CrateReader {
 
 		if ( this.version < V0_4_0 ) {
 
+			this.records( count, 4 );
 			this.fieldSets = new Uint32Array( count );
 			for ( let i = 0; i < count; i ++ ) this.fieldSets[ i ] = this.u32();
 
@@ -263,16 +296,27 @@ class CrateReader {
 		if ( ! this.section( 'PATHS' ) ) return;
 
 		const numPaths = this.u64();
-		this.paths = new Array( numPaths ).fill( '' );
-		this.pathInfo = new Array( numPaths ).fill( null );
+		const allocate = () => {
+
+			this.budget.decode( numPaths * 16, 'usdc' );
+			this.paths = new Array( numPaths ).fill( '' );
+			this.pathInfo = new Array( numPaths ).fill( null );
+
+		};
 
 		if ( this.version < V0_4_0 ) {
 
-			this.readPathTree( this.version === version( 0, 0, 1 ) ? 16 : 12 );
+			const entry = this.version === version( 0, 0, 1 ) ? 16 : 12;
+			this.records( numPaths, entry ); // a tree entry per path
+			allocate();
+			this.readPathTree( entry );
 
 		} else {
 
 			const n = this.u64();
+			// the tree holds every path, sometimes leaving out an unused first slot
+			if ( ! Number.isSafeInteger( numPaths ) || numPaths > n + 1 ) throw new Error( 'usd-gltf: corrupt path table in usdc file' );
+			allocate();
 			const pathIndexes = this.compressedInts( n );
 			const elementTokens = this.compressedInts( n );
 			const jumps = this.compressedInts( n );
@@ -405,6 +449,7 @@ class CrateReader {
 		if ( this.version < V0_4_0 ) {
 
 			const pad = this.version === version( 0, 0, 1 ) ? 4 : 0;
+			this.records( count, 12 + pad );
 			this.specPaths = new Uint32Array( count );
 			this.specFieldSets = new Uint32Array( count );
 			this.specTypes = new Uint32Array( count );
@@ -591,6 +636,7 @@ class CrateReader {
 	readVector( item ) {
 
 		const n = this.u64();
+		this.records( n, 1 ); // every item takes at least a byte
 		const out = new Array( n );
 		for ( let i = 0; i < n; i ++ ) out[ i ] = item();
 		return out;
@@ -731,8 +777,8 @@ class CrateReader {
 
 			if ( compressed && pod[ 1 ] === 1 && n >= MIN_COMPRESSED_ARRAY_SIZE && ( kind === 'i32' || kind === 'u32' || kind === 'i64' || kind === 'u64' ) && this.version >= V0_5_0 ) {
 
-				const size = this.u64();
-				const data = this.take( size );
+				const data = this.take( this.u64() );
+				this.decodedInts( n, kind === 'i32' || kind === 'u32' ? 4 : 8, data );
 				if ( kind === 'i32' ) out = decompressInts32( data, n );
 				else if ( kind === 'u32' ) out = new Uint32Array( decompressInts32( data, n ).buffer );
 				else out = decompressInts64( data, n, kind === 'u64' );
@@ -769,6 +815,7 @@ class CrateReader {
 
 	readArrayItems( n, item ) {
 
+		this.records( n, 4 );
 		const out = new Array( n );
 		for ( let i = 0; i < n; i ++ ) out[ i ] = item();
 		return out;
@@ -778,6 +825,7 @@ class CrateReader {
 	readRawArray( kind, count ) {
 
 		const size = SCALAR_SIZE[ kind ];
+		this.records( count, kind === 'f16' || kind === 'i64' || kind === 'u64' ? size * 3 : size ); // and any widened copy
 		const bytes = this.take( count * size ).slice(); // copy, also aligns
 
 		switch ( kind ) {
@@ -812,6 +860,8 @@ class CrateReader {
 	readCompressedFloats( kind, n ) {
 
 		const code = this.u8();
+		checkSize( n * 8, 'usdc' );
+		this.budget.decode( n * ( kind === 'f64' ? 8 : 4 ), 'usdc' );
 		const out = typedArrayFor( kind, n );
 
 		if ( code === 105 ) { // 'i': integers
@@ -912,11 +962,11 @@ function payloadListOp( value ) {
  * @param {string} [identifier]
  * @returns {Layer}
  */
-export function readUSDC( data, identifier = 'anon.usdc' ) {
+export function readUSDC( data, identifier = 'anon.usdc', { budget = new Budget() } = {} ) {
 
 	// a plain view, so subarray().slice() copies even for a Node Buffer
 	const bytes = ArrayBuffer.isView( data ) ? new Uint8Array( data.buffer, data.byteOffset, data.byteLength ) : new Uint8Array( data );
-	const crate = new CrateReader( bytes );
+	const crate = new CrateReader( bytes, budget );
 	const layer = new Layer( identifier );
 
 	const specs = new Map(); // path index -> { type, object, order }

package/src/usd/usdc/writer.js

@@ -4,6 +4,7 @@
 import { AssetPath, DICT_TYPES, ListOp, Reference } from '../model.js';
 import { compressFast } from './lz4.js';
 import { compressInts32 } from './intcompression.js';
+import { Budget } from '../../limits.js';
 import { PERMISSIONS, POD, SPEC, SPECIFIERS, TYPE, crateTypeFor, floatToHalf, halfToFloat, isMatrixType, isVecType } from './types.js';
 
 const IS_ARRAY = 0x80000000;
@@ -148,8 +149,9 @@ function pathElements( path ) {
 
 class CrateWriter {
 
-	constructor() {
+	constructor( budget = new Budget() ) {
 
+		this.budget = budget;
 		this.buf = new Uint8Array( 1 << 16 );
 		this.view = new DataView( this.buf.buffer );
 		this.pos = 88; // bootstrap, written last
@@ -173,6 +175,7 @@ class CrateWriter {
 		this.fieldSetIndex = new Map();
 		this.specs = [];
 		this.timesCache = new Map();
+		this.arrayCache = new Map(); // hash -> [ { values, type, rep } ], so identical arrays are written once
 
 	}
 
@@ -181,8 +184,10 @@ class CrateWriter {
 	ensure( n ) {
 
 		if ( this.pos + n <= this.buf.length ) return;
+		this.budget.output( this.pos + n, 'usdc' );
 		let size = this.buf.length * 2;
 		while ( size < this.pos + n ) size *= 2;
+		size = Math.min( size, Math.max( this.pos + n, this.budget.maxOutputBytes ) );
 		const buf = new Uint8Array( size );
 		buf.set( this.buf );
 		this.buf = buf;
@@ -337,6 +342,7 @@ class CrateWriter {
 
 	addSpec( path, specType, fields ) {
 
+		this.budget.check();
 		const indexes = [];
 
 		for ( const [ name, [ lo, hi ] ] of fields ) {
@@ -690,8 +696,24 @@ class CrateWriter {
 
 	}
 
+	// Identical numeric arrays share one copy in the file, as OpenUSD's crate writer does.
 	packArray( values, type ) {
 
+		if ( ! POD[ type ] || values.length < MIN_COMPRESSED_ARRAY_SIZE ) return this.packArrayData( values, type );
+
+		const key = type + ':' + values.length + ':' + hashValues( values );
+		const entries = this.arrayCache.get( key ) || this.arrayCache.set( key, [] ).get( key );
+		const hit = entries.find( ( e ) => sameValues( e.values, values ) );
+		if ( hit ) return hit.rep;
+
+		const rep = this.packArrayData( values, type );
+		entries.push( { values, rep } );
+		return rep;
+
+	}
+
+	packArrayData( values, type ) {
+
 		const flags = ( IS_ARRAY | ( type << 16 ) ) >>> 0;
 		const pod = POD[ type ];
 		const total = values.length;
@@ -949,6 +971,33 @@ class CrateWriter {
 
 }
 
+const F64 = new Float64Array( 1 );
+const F64_WORDS = new Int32Array( F64.buffer );
+
+function hashValues( values ) {
+
+	let h = 0x811c9dc5;
+
+	for ( let i = 0; i < values.length; i ++ ) {
+
+		F64[ 0 ] = values[ i ];
+		h = Math.imul( h ^ F64_WORDS[ 0 ], 16777619 );
+		h = Math.imul( h ^ F64_WORDS[ 1 ], 16777619 );
+
+	}
+
+	return h >>> 0;
+
+}
+
+function sameValues( a, b ) {
+
+	if ( a.length !== b.length ) return false;
+	for ( let i = 0; i < a.length; i ++ ) if ( ! Object.is( a[ i ], b[ i ] ) ) return false;
+	return true;
+
+}
+
 function isInt8( v ) {
 
 	return Number.isInteger( v ) && v >= - 128 && v <= 127 && ! Object.is( v, - 0 );
@@ -1087,11 +1136,12 @@ function writeProperty( w, prop, path ) {
 /**
  * Writes a Layer as a binary USD crate file.
  * @param {import('../model.js').Layer} layer
+ * @param {{ budget?: import('../../limits.js').Budget }} [options]
  * @returns {Uint8Array}
  */
-export function writeUSDC( layer ) {
+export function writeUSDC( layer, { budget = new Budget() } = {} ) {
 
-	const w = new CrateWriter();
+	const w = new CrateWriter( budget );
 	const root = layer.root;
 
 	const fields = metadataFields( w, layer.metadata );

package/src/usd/usdz.js

@@ -3,6 +3,7 @@
 
 import { crc32 } from '../util/crc32.js';
 import { inflate } from '../image/inflate.js';
+import { Budget } from '../limits.js';
 
 const LOCAL = 0x04034b50;
 const CENTRAL = 0x02014b50;
@@ -12,9 +13,10 @@ const ALIGN = 64;
 /**
  * Lists the files in a zip, in archive order.
  * @param {Uint8Array} bytes
+ * @param {{ budget?: import('../limits.js').Budget }} [options]
  * @returns {Map<string, Uint8Array>}
  */
-export function readZip( bytes ) {
+export function readZip( bytes, { budget = new Budget() } = {} ) {
 
 	const view = new DataView( bytes.buffer, bytes.byteOffset, bytes.byteLength );
 	const files = new Map();
@@ -33,6 +35,7 @@ export function readZip( bytes ) {
 
 		const method = view.getUint16( offset + 10, true );
 		const compressedSize = view.getUint32( offset + 20, true );
+		const size = view.getUint32( offset + 24, true );
 		const nameLength = view.getUint16( offset + 28, true );
 		const extraLength = view.getUint16( offset + 30, true );
 		const commentLength = view.getUint16( offset + 32, true );
@@ -40,12 +43,13 @@ export function readZip( bytes ) {
 		const name = decoder.decode( bytes.subarray( offset + 46, offset + 46 + nameLength ) );
 
 		const dataStart = localOffset + 30 + view.getUint16( localOffset + 26, true ) + view.getUint16( localOffset + 28, true );
+		if ( dataStart + compressedSize > bytes.length ) throw new Error( 'usd-gltf: truncated zip archive' );
 		const data = bytes.subarray( dataStart, dataStart + compressedSize );
 
 		if ( ! name.endsWith( '/' ) ) {
 
 			if ( method === 0 ) files.set( name, data );
-			else if ( method === 8 ) files.set( name, inflate( data ) );
+			else if ( method === 8 ) { budget.decode( size, 'zip' ); files.set( name, inflate( data, size, size ) ); }
 			else throw new Error( `usd-gltf: unsupported zip compression ${ method } for ${ name }` );
 
 		}
@@ -61,9 +65,10 @@ export function readZip( bytes ) {
 /**
  * Writes a USDZ package. The first entry must be the root layer.
  * @param {Array<{ name: string, data: Uint8Array }>} entries
+ * @param {{ budget?: import('../limits.js').Budget }} [options]
  * @returns {Uint8Array}
  */
-export function writeUSDZ( entries ) {
+export function writeUSDZ( entries, { budget = new Budget() } = {} ) {
 
 	const encoder = new TextEncoder();
 	const locals = [];
@@ -127,6 +132,7 @@ export function writeUSDZ( entries ) {
 	ev.setUint32( 12, centralSize, true );
 	ev.setUint32( 16, offset, true );
 
+	budget.output( offset + centralSize + 22, 'usdz' );
 	const out = new Uint8Array( offset + centralSize + 22 );
 	let p = 0;
 	for ( const part of [ ...locals, ...centrals, end ] ) {