rayzee 9.0.0 → 9.1.0

package/src/Storage/BLASCache.js

@@ -0,0 +1,190 @@
+/**
+ * Stored per-mesh BVHs (BLASes), so reopening a scene skips building them. Keyed by the scene's
+ * source and the builder settings; every template's triangles are checksummed on both sides, so a
+ * stored BLAS is only ever used for exactly the triangles it was built over.
+ */
+
+import { ENGINE_AREAS } from './StorageManager.js';
+
+export const BLAS_CACHE_FORMAT = 2;
+
+const PIECE_RECORDS = 1 << 20;
+const YIELD_EVERY = 1 << 21;
+
+const pause = () => new Promise( ( resolve ) => setTimeout( resolve, 0 ) );
+
+// A BLAS depends on vertex positions alone: lanes 0-2, 4-6 and 8-10 of a triangle record.
+const POSITION_LANES = [ 0, 1, 2, 4, 5, 6, 8, 9, 10 ];
+
+/**
+ * Sum of per-record hashes over records [start, start + count): the same whatever order the
+ * records are in, so it compares triangles before and after a BLAS build reorders them.
+ * @param {number[]} [lanes] - which lanes to hash; the triangle positions by default
+ */
+export function rangeChecksum( records, start, count, lanes = POSITION_LANES ) {
+
+	const stride = records.lanesPerRecord;
+	const used = lanes.length;
+	let a = 0;
+	let b = 0;
+	let done = 0;
+
+	while ( done < count ) {
+
+		const rec = start + done;
+		const chunk = records.chunkFor( rec );
+		const n = Math.min( records.recordsPerChunk - ( rec % records.recordsPerChunk ), count - done );
+		let o = records.baseOf( rec );
+
+		for ( let i = 0; i < n; i ++, o += stride ) {
+
+			let h = 0x811c9dc5;
+			let g = 0x9e3779b9;
+			for ( let l = 0; l < used; l ++ ) {
+
+				const v = chunk[ o + lanes[ l ] ];
+				h = Math.imul( h ^ v, 0x01000193 );
+				g = Math.imul( ( g + v ) | 0, 0x85ebca6b ) ^ ( g >>> 13 );
+
+			}
+
+			a = ( a + h ) >>> 0;
+			b = ( b + g ) >>> 0;
+
+		}
+
+		done += n;
+
+	}
+
+	return `${a.toString( 16 )}.${b.toString( 16 )}`;
+
+}
+
+/** Checksums every owning template, yielding now and then so a big scene does not freeze the page. */
+export async function templateChecksums( records, templates ) {
+
+	const out = [];
+	let since = 0;
+	for ( const { triOffset, triCount } of templates ) {
+
+		out.push( rangeChecksum( records, triOffset, triCount ) );
+		since += triCount;
+		if ( since >= YIELD_EVERY ) {
+
+			since = 0;
+			await pause();
+
+		}
+
+	}
+
+	return out;
+
+}
+
+/**
+ * @param {import('./StorageManager.js').StorageManager} storage
+ * @returns {Promise<?{index: Object, nodes: File, order: File, release: function(): void}>}
+ */
+export async function openBLASCache( storage, key ) {
+
+	const entry = await storage?.area( ENGINE_AREAS.SCENES ).open( key );
+	if ( ! entry ) return null;
+
+	try {
+
+		const index = await entry.json( 'index.json' );
+		const nodes = await entry.file( 'nodes.f32' );
+		const order = await entry.file( 'order.u32' );
+		if ( index?.v !== BLAS_CACHE_FORMAT || ! nodes || ! order ) {
+
+			entry.release();
+			return null;
+
+		}
+
+		return { index, nodes, order, release: () => entry.release() };
+
+	} catch {
+
+		entry.release();
+		return null;
+
+	}
+
+}
+
+/**
+ * @param {{index: Object, bvh: import('../Processor/ChunkedRecords.js').ChunkedRecords,
+ *   orders: Array<?Uint32Array>, isStale: function(): boolean, label?: string}} build
+ * @returns {Promise<boolean>}
+ */
+export async function saveBLASCache( storage, key, { index, bvh, orders, isStale, label = '' } ) {
+
+	const area = storage?.area( ENGINE_AREAS.SCENES );
+	if ( ! area ) return false;
+
+	const nodeRecords = index.totalNodes - index.tlasNodeCount;
+	const orderBytes = orders.reduce( ( n, o ) => n + ( o?.byteLength ?? 0 ), 0 );
+	const writer = await area.create( key, { label: `${label} (BVH)`, expectedBytes: nodeRecords * 64 + orderBytes } );
+	if ( ! writer ) return false;
+
+	try {
+
+		for ( let rec = index.tlasNodeCount; rec < index.totalNodes; ) {
+
+			if ( isStale() ) throw new Error( 'scene changed while it was being stored' );
+			const chunk = bvh.chunkFor( rec );
+			const room = bvh.recordsPerChunk - ( rec % bvh.recordsPerChunk );
+			const n = Math.min( room, index.totalNodes - rec, PIECE_RECORDS );
+			const base = bvh.baseOf( rec );
+			await writer.write( 'nodes.f32', chunk.subarray( base, base + n * 16 ) );
+			rec += n;
+
+		}
+
+		for ( const order of orders ) {
+
+			if ( ! order ) continue;
+			for ( let at = 0; at < order.length; at += PIECE_RECORDS * 4 ) {
+
+				await writer.write( 'order.u32', order.subarray( at, Math.min( order.length, at + PIECE_RECORDS * 4 ) ) );
+
+			}
+
+		}
+
+		if ( orderBytes === 0 ) await writer.write( 'order.u32', new Uint8Array( 0 ) );
+		if ( isStale() ) throw new Error( 'scene changed while it was being stored' );
+		await writer.writeJSON( 'index.json', index );
+		await writer.commit();
+		return true;
+
+	} catch {
+
+		await writer.abort();
+		return false;
+
+	}
+
+}
+
+/** Reads records [from, to) of a stored node file into `bvh` at the same indices + `offset`. */
+export async function readNodesInto( file, bvh, firstRecord, recordCount ) {
+
+	for ( let done = 0; done < recordCount; done += PIECE_RECORDS ) {
+
+		const n = Math.min( PIECE_RECORDS, recordCount - done );
+		const buffer = await file.slice( done * 64, ( done + n ) * 64 ).arrayBuffer();
+		bvh.setRecords( firstRecord + done, new Float32Array( buffer ) );
+
+	}
+
+}
+
+export async function readOrder( file, offset, count ) {
+
+	return new Uint32Array( await file.slice( offset * 4, ( offset + count ) * 4 ).arrayBuffer() );
+
+}

package/src/Storage/CDFCache.js

@@ -0,0 +1,77 @@
+import { ENGINE_AREAS } from './StorageManager.js';
+import { sharedStorage } from './shared.js';
+import { getAssetConfig } from '../AssetConfig.js';
+
+const FORMAT = 1;
+
+/**
+ * Where an environment texture came from, recorded by the loader so its sampling tables can be
+ * found again. Pixels edited in place (skies, colour conversion) change the key or clear it.
+ */
+export function setEnvironmentSource( texture, source ) {
+
+	if ( texture && source ) texture.userData.__rayzeeSource = source;
+
+}
+
+function keyFor( texture ) {
+
+	const source = texture?.userData?.__rayzeeSource;
+	if ( ! source ) return null;
+	const { width = 0, height = 0 } = texture.image ?? {};
+	return `cdf:${FORMAT}:${source}|${width}x${height}|${texture.userData.__rayzeeColorSpace ?? 'native'}`;
+
+}
+
+const area = () => sharedStorage( getAssetConfig().cacheNamespace )?.area( ENGINE_AREAS.CDF ) ?? null;
+
+/** @returns {Promise<?{marginalData: Float32Array, conditionalData: Float32Array, totalSum: number, compensationDelta: number, width: number, height: number}>} */
+export async function loadCDF( texture ) {
+
+	const key = keyFor( texture );
+	const entry = key ? await area()?.open( key ) : null;
+	if ( ! entry ) return null;
+
+	try {
+
+		const [ marginal, conditional ] = await Promise.all( [ entry.file( 'marginal.f32' ), entry.file( 'conditional.f32' ) ] );
+		if ( ! marginal || ! conditional ) return null;
+		const { totalSum, compensationDelta, width, height } = entry.extra;
+		return {
+			marginalData: new Float32Array( await marginal.arrayBuffer() ),
+			conditionalData: new Float32Array( await conditional.arrayBuffer() ),
+			totalSum, compensationDelta, width, height,
+		};
+
+	} finally {
+
+		entry.release();
+
+	}
+
+}
+
+export async function saveCDF( texture, { marginalData, conditionalData, totalSum, compensationDelta, width, height } ) {
+
+	const key = keyFor( texture );
+	const target = key ? area() : null;
+	if ( ! target || ! marginalData || ! conditionalData ) return false;
+
+	const writer = await target.create( key, { label: texture.name || 'environment', expectedBytes: marginalData.byteLength + conditionalData.byteLength } );
+	if ( ! writer ) return false;
+
+	try {
+
+		await writer.write( 'marginal.f32', marginalData );
+		await writer.write( 'conditional.f32', conditionalData );
+		await writer.commit( { totalSum, compensationDelta, width, height } );
+		return true;
+
+	} catch {
+
+		await writer.abort();
+		return false;
+
+	}
+
+}

package/src/Storage/DownloadCache.js

@@ -0,0 +1,266 @@
+import { ENGINE_AREAS } from './StorageManager.js';
+import { sharedStorage } from './shared.js';
+import { getAssetConfig } from '../AssetConfig.js';
+
+const DAY_MS = 86_400_000;
+
+/**
+ * `immutable`: a cached copy is always right (versioned paths). `revalidate`: use the cached copy
+ * at once, then check the server in the background at most once per `maxAgeMs`. `network`: skip
+ * the cache.
+ */
+export const DOWNLOAD_POLICY = Object.freeze( { IMMUTABLE: 'immutable', REVALIDATE: 'revalidate', NETWORK: 'network' } );
+
+const progressEvent = ( loaded, total ) => ( { loaded, total, lengthComputable: total > 0 && loaded <= total } );
+
+const header = ( response, name ) => response.headers?.get?.( name ) ?? null;
+
+const abortError = ( signal ) => signal.reason ?? new DOMException( 'aborted', 'AbortError' );
+
+export function nameFromUrl( url ) {
+
+	try {
+
+		const path = new URL( url, globalThis.location?.href ?? 'http://localhost/' ).pathname;
+		return decodeURIComponent( path.split( '/' ).pop() ) || 'download';
+
+	} catch {
+
+		return 'download';
+
+	}
+
+}
+
+async function readBody( response, { onProgress, signal } ) {
+
+	const total = Number( header( response, 'content-length' ) ) || 0;
+	const type = header( response, 'content-type' ) ?? '';
+	if ( ! response.body ) return new Blob( [ await response.arrayBuffer() ], { type } );
+
+	const reader = response.body.getReader();
+	const chunks = [];
+	let loaded = 0;
+	for ( ;; ) {
+
+		if ( signal?.aborted ) {
+
+			reader.cancel().catch( () => {} );
+			throw abortError( signal );
+
+		}
+
+		const { done, value } = await reader.read();
+		if ( done ) break;
+		chunks.push( value );
+		loaded += value.length;
+		onProgress?.( progressEvent( loaded, total ) );
+
+	}
+
+	return new Blob( chunks, { type } );
+
+}
+
+const asFile = ( blob, name, type ) => new File( [ blob ], name, { type: type || blob.type, lastModified: blob.lastModified ?? Date.now() } );
+
+const noop = () => {};
+
+/**
+ * Downloads that survive reloads. Without storage (or when it is full) a download lands in
+ * memory exactly as before, so callers never branch on it.
+ */
+export class DownloadCache {
+
+	/** @param {?import('./StorageManager.js').StorageManager} storage */
+	constructor( storage ) {
+
+		this._storage = storage;
+		this._revalidating = new Set();
+
+	}
+
+	get _area() {
+
+		return this._storage?.area( ENGINE_AREAS.DOWNLOADS ) ?? null;
+
+	}
+
+	/**
+	 * Hold `release()` until done reading the File: it keeps another tab from replacing the entry.
+	 * @returns {Promise<{file: File, release: function(): void, fromCache: boolean}>}
+	 */
+	async fetch( url, {
+		key = url,
+		name = nameFromUrl( url ),
+		policy = DOWNLOAD_POLICY.REVALIDATE,
+		maxAgeMs = DAY_MS,
+		onProgress = null,
+		signal = null,
+		fetchOptions = {},
+	} = {} ) {
+
+		const area = this._area;
+
+		if ( area && policy !== DOWNLOAD_POLICY.NETWORK ) {
+
+			const entry = await area.open( key );
+			if ( entry ) {
+
+				const blob = await entry.file( 'data' );
+				if ( blob ) {
+
+					if ( policy === DOWNLOAD_POLICY.REVALIDATE && Date.now() - ( entry.extra.checkedAt ?? 0 ) >= maxAgeMs ) {
+
+						this._revalidate( url, key, name, entry.extra, fetchOptions );
+
+					}
+
+					onProgress?.( progressEvent( blob.size, blob.size ) );
+					return { file: asFile( blob, name, entry.extra.contentType ), release: () => entry.release(), fromCache: true };
+
+				}
+
+				entry.release();
+
+			}
+
+		}
+
+		return this._download( url, key, name, { onProgress, signal, fetchOptions, cache: !! area } );
+
+	}
+
+	/** Whether a copy is stored, without opening it for reading. */
+	async has( key ) {
+
+		return ( await this._area?.has( key ) ) === true;
+
+	}
+
+	async _download( url, key, name, { onProgress, signal, fetchOptions, cache } ) {
+
+		if ( signal?.aborted ) throw abortError( signal );
+		const response = await fetch( url, { ...fetchOptions, signal } );
+		if ( ! response.ok ) throw new Error( `HTTP ${response.status} fetching ${url}` );
+
+		const total = Number( header( response, 'content-length' ) ) || 0;
+		const info = {
+			url,
+			contentType: header( response, 'content-type' ) ?? '',
+			lastModified: header( response, 'last-modified' ),
+			contentLength: total || null,
+			checkedAt: Date.now(),
+		};
+
+		const writer = cache && response.body ? await this._area.create( key, { label: name, expectedBytes: total } ) : null;
+		if ( ! writer ) {
+
+			return { file: asFile( await readBody( response, { onProgress, signal } ), name, info.contentType ), release: noop, fromCache: false };
+
+		}
+
+		try {
+
+			await writer.writeStream( 'data', response.body, { signal, onProgress: ( loaded ) => onProgress?.( progressEvent( loaded, total ) ) } );
+			await writer.commit( info );
+
+		} catch ( error ) {
+
+			await writer.abort();
+			if ( error?.name !== 'QuotaExceededError' ) throw error;
+			return this._download( url, key, name, { onProgress, signal, fetchOptions, cache: false } );
+
+		}
+
+		const entry = await this._area.open( key, { wait: true } );
+		const blob = entry && await entry.file( 'data' );
+		if ( ! blob ) {
+
+			entry?.release();
+			throw new Error( `storage: ${name} vanished right after it was saved` );
+
+		}
+
+		return { file: asFile( blob, name, info.contentType ), release: () => entry.release(), fromCache: false };
+
+	}
+
+	async _revalidate( url, key, name, extra, fetchOptions ) {
+
+		if ( this._revalidating.has( key ) ) return;
+		this._revalidating.add( key );
+
+		try {
+
+			const head = await fetch( url, { ...fetchOptions, method: 'HEAD' } );
+			if ( ! head.ok ) return;
+
+			const lastModified = header( head, 'last-modified' );
+			const length = Number( header( head, 'content-length' ) ) || null;
+			const changed = ( lastModified && extra.lastModified && lastModified !== extra.lastModified )
+				|| ( length && extra.contentLength && length !== extra.contentLength );
+
+			if ( changed ) {
+
+				const { release } = await this._download( url, key, name, { fetchOptions, cache: true } );
+				release();
+
+			} else {
+
+				await this._area.patchMeta( key, { extra: { ...extra, checkedAt: Date.now() } } );
+
+			}
+
+		} catch {
+
+			// Offline, or the server refuses HEAD: keep serving the stored copy.
+
+		} finally {
+
+			this._revalidating.delete( key );
+
+		}
+
+	}
+
+}
+
+/** One-off download into memory or storage, for code that holds no DownloadCache. */
+export function fetchFile( url, storage, options ) {
+
+	return new DownloadCache( storage ).fetch( url, options );
+
+}
+
+/** Downloads through whatever storage this page's apps opened; plain fetches when none. */
+export function sharedDownloads() {
+
+	return new DownloadCache( sharedStorage( getAssetConfig().cacheNamespace ) );
+
+}
+
+/**
+ * A URL a three.js loader can take in place of `url`: an object URL of the stored copy when there
+ * is storage, the network URL untouched otherwise (so the loader's own progress still works).
+ * Call `release()` once the loader has read it.
+ * @returns {Promise<{url: string, release: function(): void, cached: boolean}>}
+ */
+export async function cachedObjectURL( url, { downloads = sharedDownloads(), ...options } = {} ) {
+
+	if ( ! downloads._storage || ! /^https?:/i.test( url ) ) return { url, release: noop, cached: false };
+
+	const { file, release } = await downloads.fetch( url, options );
+	const objectUrl = URL.createObjectURL( file );
+	return {
+		url: objectUrl,
+		cached: true,
+		release: () => {
+
+			URL.revokeObjectURL( objectUrl );
+			release();
+
+		},
+	};
+
+}