usd-gltf 0.0.1 → 0.2.0

package/src/usd/usdc/reader.js

@@ -4,6 +4,7 @@
 import { AssetPath, DICT_TYPES, Layer, ListOp, PrimSpec, PropertySpec, Reference } from '../model.js';
 import { decompressFast } from './lz4.js';
 import { decompressInts32, decompressInts64 } from './intcompression.js';
+import { Budget, checkSize } from '../../limits.js';
 import { PERMISSIONS, POD, SCALAR_SIZE, SPEC, SPECIFIERS, TYPE, TYPE_NAMES, halfArrayToFloat32, halfToFloat, isMatrixType, isVecType } from './types.js';
 
 const IS_ARRAY = 0x80000000;
@@ -42,9 +43,10 @@ const V0_8_0 = version( 0, 8, 0 );
 
 class CrateReader {
 
-	constructor( bytes ) {
+	constructor( bytes, budget = new Budget() ) {
 
 		this.bytes = bytes;
+		this.budget = budget;
 		this.view = new DataView( bytes.buffer, bytes.byteOffset, bytes.byteLength );
 		this.pos = 0;
 		this.timesCache = new Map();
@@ -136,7 +138,7 @@ class CrateReader {
 
 	take( n ) {
 
-		if ( this.pos + n > this.bytes.length ) throw new Error( 'usd-gltf: truncated usdc file' );
+		if ( ! Number.isSafeInteger( n ) || n < 0 || this.pos + n > this.bytes.length ) throw new Error( 'usd-gltf: truncated usdc file' );
 		const b = this.bytes.subarray( this.pos, this.pos + n );
 		this.pos += n;
 		return b;
@@ -145,8 +147,37 @@ class CrateReader {
 
 	compressedInts( n ) {
 
-		const size = this.u64();
-		return decompressInts32( this.take( size ), n );
+		const data = this.take( this.u64() );
+		this.decodedInts( n, 4, data );
+		return decompressInts32( data, n );
+
+	}
+
+	// Sizes come from the file, so they are checked before allocating. Compressed integers take at
+	// least 2 bits each before LZ4, which shrinks data at most 255 times.
+	decodedInts( n, size, data ) {
+
+		checkSize( n * size, 'usdc' );
+		if ( n > data.length * 1024 + 64 ) throw new Error( 'usd-gltf: corrupt compressed integers in usdc file' );
+		this.budget.decode( n * size * 2, 'usdc' ); // the encoded buffer and the decoded integers
+
+	}
+
+	lz4( data, size ) {
+
+		checkSize( size, 'usdc' );
+		if ( size > data.length * 256 + 64 ) throw new Error( 'usd-gltf: corrupt compressed data in usdc file' );
+		this.budget.decode( size, 'usdc' );
+		return decompressFast( data, size );
+
+	}
+
+	// count fixed size records at this.pos, which must all be in the file
+	records( count, size ) {
+
+		checkSize( count * size, 'usdc' );
+		if ( this.pos + count * size > this.bytes.length ) throw new Error( 'usd-gltf: truncated usdc file' );
+		this.budget.decode( count * size, 'usdc' );
 
 	}
 
@@ -176,7 +207,7 @@ class CrateReader {
 
 			const uncompressedSize = this.u64();
 			const compressedSize = this.u64();
-			chars = decompressFast( this.take( compressedSize ), uncompressedSize );
+			chars = this.lz4( this.take( compressedSize ), uncompressedSize );
 
 		}
 
@@ -213,6 +244,7 @@ class CrateReader {
 
 		if ( this.version < V0_4_0 ) {
 
+			this.records( count, 16 );
 			this.fieldTokens = new Uint32Array( count );
 			this.fieldReps = new Uint32Array( count * 2 );
 
@@ -229,7 +261,7 @@ class CrateReader {
 
 			this.fieldTokens = new Uint32Array( this.compressedInts( count ).buffer );
 			const repsSize = this.u64();
-			const reps = decompressFast( this.take( repsSize ), count * 8 );
+			const reps = this.lz4( this.take( repsSize ), count * 8 );
 			this.fieldReps = new Uint32Array( reps.buffer, 0, count * 2 );
 
 		}
@@ -245,6 +277,7 @@ class CrateReader {
 
 		if ( this.version < V0_4_0 ) {
 
+			this.records( count, 4 );
 			this.fieldSets = new Uint32Array( count );
 			for ( let i = 0; i < count; i ++ ) this.fieldSets[ i ] = this.u32();
 
@@ -263,16 +296,27 @@ class CrateReader {
 		if ( ! this.section( 'PATHS' ) ) return;
 
 		const numPaths = this.u64();
-		this.paths = new Array( numPaths ).fill( '' );
-		this.pathInfo = new Array( numPaths ).fill( null );
+		const allocate = () => {
+
+			this.budget.decode( numPaths * 16, 'usdc' );
+			this.paths = new Array( numPaths ).fill( '' );
+			this.pathInfo = new Array( numPaths ).fill( null );
+
+		};
 
 		if ( this.version < V0_4_0 ) {
 
-			this.readPathTree( this.version === version( 0, 0, 1 ) ? 16 : 12 );
+			const entry = this.version === version( 0, 0, 1 ) ? 16 : 12;
+			this.records( numPaths, entry ); // a tree entry per path
+			allocate();
+			this.readPathTree( entry );
 
 		} else {
 
 			const n = this.u64();
+			// the tree holds every path, sometimes leaving out an unused first slot
+			if ( ! Number.isSafeInteger( numPaths ) || numPaths > n + 1 ) throw new Error( 'usd-gltf: corrupt path table in usdc file' );
+			allocate();
 			const pathIndexes = this.compressedInts( n );
 			const elementTokens = this.compressedInts( n );
 			const jumps = this.compressedInts( n );
@@ -405,6 +449,7 @@ class CrateReader {
 		if ( this.version < V0_4_0 ) {
 
 			const pad = this.version === version( 0, 0, 1 ) ? 4 : 0;
+			this.records( count, 12 + pad );
 			this.specPaths = new Uint32Array( count );
 			this.specFieldSets = new Uint32Array( count );
 			this.specTypes = new Uint32Array( count );
@@ -591,6 +636,7 @@ class CrateReader {
 	readVector( item ) {
 
 		const n = this.u64();
+		this.records( n, 1 ); // every item takes at least a byte
 		const out = new Array( n );
 		for ( let i = 0; i < n; i ++ ) out[ i ] = item();
 		return out;
@@ -731,8 +777,8 @@ class CrateReader {
 
 			if ( compressed && pod[ 1 ] === 1 && n >= MIN_COMPRESSED_ARRAY_SIZE && ( kind === 'i32' || kind === 'u32' || kind === 'i64' || kind === 'u64' ) && this.version >= V0_5_0 ) {
 
-				const size = this.u64();
-				const data = this.take( size );
+				const data = this.take( this.u64() );
+				this.decodedInts( n, kind === 'i32' || kind === 'u32' ? 4 : 8, data );
 				if ( kind === 'i32' ) out = decompressInts32( data, n );
 				else if ( kind === 'u32' ) out = new Uint32Array( decompressInts32( data, n ).buffer );
 				else out = decompressInts64( data, n, kind === 'u64' );
@@ -769,6 +815,7 @@ class CrateReader {
 
 	readArrayItems( n, item ) {
 
+		this.records( n, 4 );
 		const out = new Array( n );
 		for ( let i = 0; i < n; i ++ ) out[ i ] = item();
 		return out;
@@ -778,6 +825,7 @@ class CrateReader {
 	readRawArray( kind, count ) {
 
 		const size = SCALAR_SIZE[ kind ];
+		this.records( count, kind === 'f16' || kind === 'i64' || kind === 'u64' ? size * 3 : size ); // and any widened copy
 		const bytes = this.take( count * size ).slice(); // copy, also aligns
 
 		switch ( kind ) {
@@ -812,6 +860,8 @@ class CrateReader {
 	readCompressedFloats( kind, n ) {
 
 		const code = this.u8();
+		checkSize( n * 8, 'usdc' );
+		this.budget.decode( n * ( kind === 'f64' ? 8 : 4 ), 'usdc' );
 		const out = typedArrayFor( kind, n );
 
 		if ( code === 105 ) { // 'i': integers
@@ -912,11 +962,11 @@ function payloadListOp( value ) {
  * @param {string} [identifier]
  * @returns {Layer}
  */
-export function readUSDC( data, identifier = 'anon.usdc' ) {
+export function readUSDC( data, identifier = 'anon.usdc', { budget = new Budget() } = {} ) {
 
 	// a plain view, so subarray().slice() copies even for a Node Buffer
 	const bytes = ArrayBuffer.isView( data ) ? new Uint8Array( data.buffer, data.byteOffset, data.byteLength ) : new Uint8Array( data );
-	const crate = new CrateReader( bytes );
+	const crate = new CrateReader( bytes, budget );
 	const layer = new Layer( identifier );
 
 	const specs = new Map(); // path index -> { type, object, order }

package/src/usd/usdc/writer.js

@@ -4,6 +4,7 @@
 import { AssetPath, DICT_TYPES, ListOp, Reference } from '../model.js';
 import { compressFast } from './lz4.js';
 import { compressInts32 } from './intcompression.js';
+import { Budget } from '../../limits.js';
 import { PERMISSIONS, POD, SPEC, SPECIFIERS, TYPE, crateTypeFor, floatToHalf, halfToFloat, isMatrixType, isVecType } from './types.js';
 
 const IS_ARRAY = 0x80000000;
@@ -148,8 +149,9 @@ function pathElements( path ) {
 
 class CrateWriter {
 
-	constructor() {
+	constructor( budget = new Budget() ) {
 
+		this.budget = budget;
 		this.buf = new Uint8Array( 1 << 16 );
 		this.view = new DataView( this.buf.buffer );
 		this.pos = 88; // bootstrap, written last
@@ -173,6 +175,7 @@ class CrateWriter {
 		this.fieldSetIndex = new Map();
 		this.specs = [];
 		this.timesCache = new Map();
+		this.arrayCache = new Map(); // hash -> [ { values, type, rep } ], so identical arrays are written once
 
 	}
 
@@ -181,8 +184,10 @@ class CrateWriter {
 	ensure( n ) {
 
 		if ( this.pos + n <= this.buf.length ) return;
+		this.budget.output( this.pos + n, 'usdc' );
 		let size = this.buf.length * 2;
 		while ( size < this.pos + n ) size *= 2;
+		size = Math.min( size, Math.max( this.pos + n, this.budget.maxOutputBytes ) );
 		const buf = new Uint8Array( size );
 		buf.set( this.buf );
 		this.buf = buf;
@@ -337,6 +342,7 @@ class CrateWriter {
 
 	addSpec( path, specType, fields ) {
 
+		this.budget.check();
 		const indexes = [];
 
 		for ( const [ name, [ lo, hi ] ] of fields ) {
@@ -690,8 +696,24 @@ class CrateWriter {
 
 	}
 
+	// Identical numeric arrays share one copy in the file, as OpenUSD's crate writer does.
 	packArray( values, type ) {
 
+		if ( ! POD[ type ] || values.length < MIN_COMPRESSED_ARRAY_SIZE ) return this.packArrayData( values, type );
+
+		const key = type + ':' + values.length + ':' + hashValues( values );
+		const entries = this.arrayCache.get( key ) || this.arrayCache.set( key, [] ).get( key );
+		const hit = entries.find( ( e ) => sameValues( e.values, values ) );
+		if ( hit ) return hit.rep;
+
+		const rep = this.packArrayData( values, type );
+		entries.push( { values, rep } );
+		return rep;
+
+	}
+
+	packArrayData( values, type ) {
+
 		const flags = ( IS_ARRAY | ( type << 16 ) ) >>> 0;
 		const pod = POD[ type ];
 		const total = values.length;
@@ -949,6 +971,33 @@ class CrateWriter {
 
 }
 
+const F64 = new Float64Array( 1 );
+const F64_WORDS = new Int32Array( F64.buffer );
+
+function hashValues( values ) {
+
+	let h = 0x811c9dc5;
+
+	for ( let i = 0; i < values.length; i ++ ) {
+
+		F64[ 0 ] = values[ i ];
+		h = Math.imul( h ^ F64_WORDS[ 0 ], 16777619 );
+		h = Math.imul( h ^ F64_WORDS[ 1 ], 16777619 );
+
+	}
+
+	return h >>> 0;
+
+}
+
+function sameValues( a, b ) {
+
+	if ( a.length !== b.length ) return false;
+	for ( let i = 0; i < a.length; i ++ ) if ( ! Object.is( a[ i ], b[ i ] ) ) return false;
+	return true;
+
+}
+
 function isInt8( v ) {
 
 	return Number.isInteger( v ) && v >= - 128 && v <= 127 && ! Object.is( v, - 0 );
@@ -1087,11 +1136,12 @@ function writeProperty( w, prop, path ) {
 /**
  * Writes a Layer as a binary USD crate file.
  * @param {import('../model.js').Layer} layer
+ * @param {{ budget?: import('../../limits.js').Budget }} [options]
  * @returns {Uint8Array}
  */
-export function writeUSDC( layer ) {
+export function writeUSDC( layer, { budget = new Budget() } = {} ) {
 
-	const w = new CrateWriter();
+	const w = new CrateWriter( budget );
 	const root = layer.root;
 
 	const fields = metadataFields( w, layer.metadata );

package/src/usd/usdz.js

@@ -3,6 +3,7 @@
 
 import { crc32 } from '../util/crc32.js';
 import { inflate } from '../image/inflate.js';
+import { Budget } from '../limits.js';
 
 const LOCAL = 0x04034b50;
 const CENTRAL = 0x02014b50;
@@ -12,9 +13,10 @@ const ALIGN = 64;
 /**
  * Lists the files in a zip, in archive order.
  * @param {Uint8Array} bytes
+ * @param {{ budget?: import('../limits.js').Budget }} [options]
  * @returns {Map<string, Uint8Array>}
  */
-export function readZip( bytes ) {
+export function readZip( bytes, { budget = new Budget() } = {} ) {
 
 	const view = new DataView( bytes.buffer, bytes.byteOffset, bytes.byteLength );
 	const files = new Map();
@@ -33,6 +35,7 @@ export function readZip( bytes ) {
 
 		const method = view.getUint16( offset + 10, true );
 		const compressedSize = view.getUint32( offset + 20, true );
+		const size = view.getUint32( offset + 24, true );
 		const nameLength = view.getUint16( offset + 28, true );
 		const extraLength = view.getUint16( offset + 30, true );
 		const commentLength = view.getUint16( offset + 32, true );
@@ -40,12 +43,13 @@ export function readZip( bytes ) {
 		const name = decoder.decode( bytes.subarray( offset + 46, offset + 46 + nameLength ) );
 
 		const dataStart = localOffset + 30 + view.getUint16( localOffset + 26, true ) + view.getUint16( localOffset + 28, true );
+		if ( dataStart + compressedSize > bytes.length ) throw new Error( 'usd-gltf: truncated zip archive' );
 		const data = bytes.subarray( dataStart, dataStart + compressedSize );
 
 		if ( ! name.endsWith( '/' ) ) {
 
 			if ( method === 0 ) files.set( name, data );
-			else if ( method === 8 ) files.set( name, inflate( data ) );
+			else if ( method === 8 ) { budget.decode( size, 'zip' ); files.set( name, inflate( data, size, size ) ); }
 			else throw new Error( `usd-gltf: unsupported zip compression ${ method } for ${ name }` );
 
 		}
@@ -61,9 +65,10 @@ export function readZip( bytes ) {
 /**
  * Writes a USDZ package. The first entry must be the root layer.
  * @param {Array<{ name: string, data: Uint8Array }>} entries
+ * @param {{ budget?: import('../limits.js').Budget }} [options]
  * @returns {Uint8Array}
  */
-export function writeUSDZ( entries ) {
+export function writeUSDZ( entries, { budget = new Budget() } = {} ) {
 
 	const encoder = new TextEncoder();
 	const locals = [];
@@ -127,6 +132,7 @@ export function writeUSDZ( entries ) {
 	ev.setUint32( 12, centralSize, true );
 	ev.setUint32( 16, offset, true );
 
+	budget.output( offset + centralSize + 22, 'usdz' );
 	const out = new Uint8Array( offset + centralSize + 22 );
 	let p = 0;
 	for ( const part of [ ...locals, ...centrals, end ] ) {