rayzee 7.23.2 → 7.24.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rayzee",
-  "version": "7.23.2",
+  "version": "7.24.0",
   "type": "module",
   "description": "Real-time WebGPU path tracing engine built on Three.js",
   "main": "dist/rayzee.umd.js",
@@ -38,7 +38,7 @@
     "three": ">=0.185.0"
   },
   "optionalDependencies": {
-    "oidn-web": ">=0.3.0",
+    "oidn-web": ">=0.4.0",
     "onnxruntime-web": ">=1.0.0"
   }
 }

package/src/Passes/AIUpscaler.js

@@ -107,6 +107,8 @@ export class AIUpscaler extends EventDispatcher {
 		this.hdr = false;
 		this.scaleFactor = options.scaleFactor || 2;
 		this.quality = options.quality || 'fast';
+		// refreshInput: () => void — re-renders the compositor so the WebGPU canvas is readable
+		this.refreshInput = options.refreshInput || null;
 		this.tileSize = options.tileSize || MODEL_CONFIG.TILE_SIZE;
 		this._tileSizeOverride = !! options.tileSize;
 
@@ -254,6 +256,14 @@ export class AIUpscaler extends EventDispatcher {
 
 	}
 
+	// See OIDNDenoiser._revealOutput. Here the resize below also clears the bitmap.
+	_revealOutput() {
+
+		this.output.style.display = 'block';
+		this.input.style.opacity = '0';
+
+	}
+
 	async execute() {
 
 		if ( ! this.enabled ) return false;
@@ -261,10 +271,6 @@ export class AIUpscaler extends EventDispatcher {
 		this.state.abortController = new AbortController();
 		this.state.isUpscaling = true;
 
-		// Show output canvas and hide input (matching OIDN lifecycle)
-		this.input.style.opacity = '0';
-		this.output.style.display = 'block';
-
 		// Capture source image SYNCHRONOUSLY before any async work.
 		// WebGPU canvas textures expire after each compositor frame,
 		// so we must grab the pixels before awaiting model load.
@@ -292,6 +298,7 @@ export class AIUpscaler extends EventDispatcher {
 		ctx.imageSmoothingEnabled = true;
 		ctx.imageSmoothingQuality = 'high';
 		ctx.drawImage( this._backupCanvas, 0, 0, outW, outH );
+		this._revealOutput();
 
 		try {
 
@@ -467,10 +474,11 @@ export class AIUpscaler extends EventDispatcher {
 
 		if ( sourceCanvas && sourceCanvas !== this.output ) {
 
-			// Source is the WebGPU canvas — must copy to 2D canvas first.
-			// WebGPU canvases expire their texture after each compositor frame,
-			// so the caller (PathTracerApp) should re-render the display stage
-			// before triggering the upscaler when OIDN is not used.
+			// Source is the WebGPU canvas — must copy to a 2D canvas first, and it only reads
+			// back non-empty straight after a compositor pass. Enforced here rather than left to
+			// the caller: any awaited GPU work in between silently yields a fully transparent
+			// capture, and the upscaler would then upscale nothing at all.
+			this.refreshInput?.();
 			const offscreen = document.createElement( 'canvas' );
 			offscreen.width = sourceCanvas.width;
 			offscreen.height = sourceCanvas.height;

package/src/Passes/OIDNDenoiser.js

@@ -4,17 +4,11 @@ import { createLogger, fmt } from '../utils/Logger.js';
 const log = createLogger( 'oidn' );
 
 let _initUNetFromURL = null;
-let _tfEngine = null;
 async function getInitUNetFromURL() {
 
 	if ( ! _initUNetFromURL ) {
 
-		const [ oidnMod, tfMod ] = await Promise.all( [
-			import( 'oidn-web' ),
-			import( '@tensorflow/tfjs-core' )
-		] );
-		_initUNetFromURL = oidnMod.initUNetFromURL;
-		_tfEngine = tfMod.engine;
+		_initUNetFromURL = ( await import( 'oidn-web' ) ).initUNetFromURL;
 
 	}
 
@@ -22,29 +16,6 @@ async function getInitUNetFromURL() {
 
 }
 
-// oidn-web caches its WebGPUBackend in TFJS's global ENGINE under 'webgpu-oidn'.
-// On dispose, drop it so the next instance binds to the new GPUDevice instead of
-// reusing the destroyed one (which would produce black tiles).
-function removeOidnTfjsBackend() {
-
-	if ( ! _tfEngine ) return;
-	try {
-
-		const eng = _tfEngine();
-		if ( eng?.registryFactory && 'webgpu-oidn' in eng.registryFactory ) {
-
-			eng.removeBackend( 'webgpu-oidn' );
-
-		}
-
-	} catch ( e ) {
-
-		log.warn( 'failed to clear cached TFJS backend', e );
-
-	}
-
-}
-
 import { TONE_MAP_FNS, linearToSRGB, applySaturation } from '../Processor/ToneMapCPU.js';
 import { getAssetConfig } from '../AssetConfig.js';
 
@@ -52,21 +23,29 @@ import { getAssetConfig } from '../AssetConfig.js';
 const _tmOut = new Float32Array( 3 );
 
 const MODEL_CONFIG = {
-	// No cleanAux flag in oidn-web, so the blob is it — and must match what setCleanAuxNormal
-	// feeds: `fast` point-samples the normal (noisy aux), balance/high accumulate it. calb_cnrm
-	// is also the only model with a `_large` blob.
+	// No cleanAux flag in oidn-web, so the blob is it — and it must match what setCleanAuxNormal
+	// feeds the aux MRT: an `alb_nrm` model wants the point-sampled normal, a `calb_cnrm` model
+	// wants the accumulated one. `cleanAux` is carried here rather than inferred from the tier
+	// name, which stopped being possible once a tier other than `fast` used a noisy-aux model.
+	//
+	// `fast-clean` is the same topology and channel width as `fast` (both 641 KB, widest conv 64)
+	// so it costs the same to run, and only differs in being trained for a clean guide.
 	QUALITY_MODELS: {
-		fast: 'rt_hdr_alb_nrm_small',
-		balance: 'rt_hdr_calb_cnrm',
-		high: 'rt_hdr_calb_cnrm_large'
+		fast: { model: 'rt_hdr_alb_nrm_small', cleanAux: false },
+		'fast-clean': { model: 'rt_hdr_calb_cnrm_small', cleanAux: true },
+		balance: { model: 'rt_hdr_calb_cnrm', cleanAux: true },
+		high: { model: 'rt_hdr_calb_cnrm_large', cleanAux: true }
 	},
 	DEFAULT_OPTIONS: {
 		enableOIDN: true,
 		oidnQuality: 'fast',
 		debugGbufferMaps: true,
-		// Tiles run at tileSize + 2*overlap (96px, 112 for _large), so small tiles pay that padding
-		// every tile: 256 puts 3.06x the pixels through the UNet at 512². Measured 3.64x faster.
-		tileSize: 512
+		// A cap, not a fixed size — the effective tile is min( max( w, h ), this ), so a frame
+		// that fits in one tile pays no overlap padding at all. Every tile otherwise runs at
+		// tileSize + 2*overlap (96px, 112 for _large): at 1024²/high, 4x512 tiles measured
+		// 422ms against 229ms for one 1024 tile. 1024 caps the one-time activation
+		// allocation at ~430MB; beyond it larger frames tile and stay bounded.
+		tileSize: 1024
 	}
 };
 
@@ -140,6 +119,8 @@ export class OIDNDenoiser extends EventDispatcher {
 		// getToneMapping: () => number (Three.js ToneMapping constant)
 		this.backendParamsGetter = options.backendParams || null;
 		this.getGPUTextures = options.getGPUTextures || null;
+		// refreshInput: () => void — re-renders the compositor so the WebGPU canvas is readable
+		this.refreshInput = options.refreshInput || null;
 		this.getExposure = options.getExposure || ( () => 1.0 );
 		this.getToneMapping = options.getToneMapping || ( () => ACESFilmicToneMapping );
 		this.getSaturation = options.getSaturation || ( () => 1.0 );
@@ -179,8 +160,8 @@ export class OIDNDenoiser extends EventDispatcher {
 		this.enabled = this.config.enableOIDN;
 		this.quality = this.config.oidnQuality;
 		this.debugGbufferMaps = this.config.debugGbufferMaps;
-		this.tileSize = this.config.tileSize;
-		// tileSize actually baked into the live UNet, so a runtime change forces a rebuild.
+		this.maxTileSize = this.config.tileSize;
+		// The size actually baked into the live UNet, so a change forces a rebuild.
 		this._activeTileSize = 0;
 
 		// State management
@@ -257,6 +238,63 @@ export class OIDNDenoiser extends EventDispatcher {
 
 	}
 
+	/**
+	 * Whether the tier's model expects a clean (accumulated) auxiliary guide rather than a
+	 * point-sampled one. The aux MRT has to be fed to match, or the guide fights the weights.
+	 * @param {string} [quality] defaults to the active tier
+	 * @returns {boolean}
+	 */
+	expectsCleanAux( quality = this.quality ) {
+
+		const { QUALITY_MODELS } = MODEL_CONFIG;
+		return ( QUALITY_MODELS[ quality ] || QUALITY_MODELS.balance ).cleanAux;
+
+	}
+
+	/**
+	 * Engine, precision, model, kernel, tile state and resource counts from the live UNet.
+	 * @returns {Object|null} null before the weights load
+	 */
+	getRuntimeInfo() {
+
+		return this.unet?.getRuntimeInfo?.() ?? null;
+
+	}
+
+	/**
+	 * Arms per-layer GPU timestamping for the next denoise. Needs `timestamp-query` on the
+	 * device; one denoise only, so call it again for each capture.
+	 * @returns {boolean} whether the capture was armed
+	 */
+	profileNextDenoise() {
+
+		return this.unet?.profileNextExecution?.() ?? false;
+
+	}
+
+	/**
+	 * Per-layer GPU timings from the last denoise armed by {@link profileNextDenoise}.
+	 * @returns {Promise<Object|null>}
+	 */
+	async getLastDenoiseProfile() {
+
+		return ( await this.unet?.getLastExecutionProfile?.() ) ?? null;
+
+	}
+
+	/**
+	 * Largest tile that still covers the frame, bounded by the configured cap. One tile means
+	 * zero overlap, which is the cheapest configuration oidn-web has.
+	 * @returns {number}
+	 */
+	_resolveTileSize() {
+
+		const longest = Math.max( this.output?.width || 0, this.output?.height || 0 );
+		if ( ! longest ) return this.maxTileSize;
+		return Math.min( longest, this.maxTileSize );
+
+	}
+
 	async _setupUNetDenoiser() {
 
 		if ( this.state.isLoading ) return;
@@ -264,8 +302,10 @@ export class OIDNDenoiser extends EventDispatcher {
 		this.state.isLoading = true;
 		const tzaUrl = this._generateTzaUrl();
 
+		const tileSize = this._resolveTileSize();
+
 		// maxTileSize is a constructor arg, so comparing the URL alone left tileSize changes inert.
-		if ( this.currentTZAUrl === tzaUrl && this._activeTileSize === this.tileSize && this.unet ) {
+		if ( this.currentTZAUrl === tzaUrl && this._activeTileSize === tileSize && this.unet ) {
 
 			this.state.isLoading = false;
 			return;
@@ -299,11 +339,14 @@ export class OIDNDenoiser extends EventDispatcher {
 			this.unet = await initFn( tzaUrl, backendParams, {
 				aux: true,
 				hdr: true,
-				maxTileSize: this.tileSize
+				maxTileSize: tileSize,
+				// Adaptive tiling ignores maxTileSize, starts at 384, and cannot see that overlap
+				// padding only vanishes once a tile covers the image — so it settles smaller.
+				dynamicTile: false
 			} );
 
 			this.currentTZAUrl = tzaUrl;
-			this._activeTileSize = this.tileSize;
+			this._activeTileSize = tileSize;
 			this.dispatchEvent( { type: 'loaded' } );
 			log.debug( 'UNet weights loaded:', tzaUrl );
 
@@ -333,8 +376,8 @@ export class OIDNDenoiser extends EventDispatcher {
 
 		const { oidnWeightsBaseUrl } = getAssetConfig();
 		const { QUALITY_MODELS } = MODEL_CONFIG;
-		const modelName = QUALITY_MODELS[ this.quality ] || QUALITY_MODELS.balance;
-		return `${oidnWeightsBaseUrl}${modelName}.tza`;
+		const tier = QUALITY_MODELS[ this.quality ] || QUALITY_MODELS.balance;
+		return `${oidnWeightsBaseUrl}${tier.model}.tza`;
 
 	}
 
@@ -349,7 +392,7 @@ export class OIDNDenoiser extends EventDispatcher {
 		Object.assign( this.config, newConfig );
 		this.quality = this.config.oidnQuality;
 		this.debugGbufferMaps = this.config.debugGbufferMaps;
-		this.tileSize = this.config.tileSize;
+		this.maxTileSize = this.config.tileSize;
 
 		// Reload denoiser if necessary
 		await this._setupUNetDenoiser();
@@ -376,6 +419,14 @@ export class OIDNDenoiser extends EventDispatcher {
 
 		}
 
+		// The tile tracks the frame size, so a resolution change since the last build has to be
+		// rebuilt into the UNet before denoising — otherwise the frame tiles when it need not.
+		if ( this.unet && ! this.state.isLoading && this._activeTileSize !== this._resolveTileSize() ) {
+
+			await this._setupUNetDenoiser();
+
+		}
+
 		// UNet weights still loading — defer the start (fired from _setupUNetDenoiser's
 		// finally) rather than silently dropping this frame's denoise request.
 		if ( this.state.isLoading ) {
@@ -404,6 +455,15 @@ export class OIDNDenoiser extends EventDispatcher {
 
 	}
 
+	// Revealing before the first paint shows the output's transparent bitmap — and its
+	// checker CSS background — over the now-hidden render canvas.
+	_revealOutput() {
+
+		this.output.style.display = 'block';
+		this.input.style.opacity = '0';
+
+	}
+
 	async execute() {
 
 		if ( ! this.enabled || ! this.unet ) return false;
@@ -411,8 +471,6 @@ export class OIDNDenoiser extends EventDispatcher {
 		// Create abort controller for this execution
 		this.state.abortController = new AbortController();
 		this.state.isDenoising = true;
-		this.input.style.opacity = '0';
-		this.output.style.display = 'block';
 
 		try {
 
@@ -485,6 +543,12 @@ export class OIDNDenoiser extends EventDispatcher {
 
 		}
 
+		// Capture the base now: the readback below awaits, and a presented WebGPU canvas only
+		// reads back non-empty straight after a compositor pass — hence the refresh.
+		this.refreshInput?.();
+		this.ctx.drawImage( this.input, 0, 0, width, height );
+		this._revealOutput();
+
 		// Ensure storage buffers are sized correctly (recreate on resolution change)
 		this._ensureGPUInputBuffers( width, height );
 
@@ -543,8 +607,6 @@ export class OIDNDenoiser extends EventDispatcher {
 		if ( globalThis.__OIDN_NO_INPUT_SCALE ) this._oidnInputScale = 1.0;
 		this._applyColorScale( device, width * height );
 
-		// Draw the current noisy frame as the base — denoised tiles paint on top progressively
-		this.ctx.drawImage( this.input, 0, 0, width, height );
 		// Pass GPU storage buffers to oidn-web (GPUBuffer path, well-tested)
 		const config = {
 			color: { data: this._gpuInputBuffers.color, width, height },
@@ -1094,9 +1156,6 @@ export class OIDNDenoiser extends EventDispatcher {
 
 		// Dispose resources
 		this.unet?.dispose();
-		// Must precede renderer.dispose() so the GPUDevice is still alive when
-		// TFJS tears down the cached backend's buffers/textures.
-		removeOidnTfjsBackend();
 		this._destroyGPUInputBuffers();
 		this._colorScalePipeline = null;
 

package/src/PathTracerApp.js

@@ -1808,6 +1808,39 @@ export class PathTracerApp extends EventDispatcher {
 
 	}
 
+	/**
+	 * Arms per-layer GPU timestamping inside the OIDN denoiser for the next denoise only.
+	 * Read the result with {@link PathTracerApp#getDenoiseProfile}.
+	 *
+	 * `getGPUTimings()` cannot see the denoise: oidn-web submits on its own command encoders,
+	 * outside the stages three.js times.
+	 *
+	 * @returns {boolean} whether the capture was armed
+	 */
+	profileNextDenoise() {
+
+		return this.denoisingManager?.denoiser?.profileNextDenoise() ?? false;
+
+	}
+
+	/**
+	 * Per-layer GPU timings from the denoise armed by {@link PathTracerApp#profileNextDenoise},
+	 * plus the live engine/precision/model/tile diagnostics.
+	 *
+	 * @returns {Promise<{ profile: Object|null, runtime: Object|null }|null>}
+	 */
+	async getDenoiseProfile() {
+
+		const denoiser = this.denoisingManager?.denoiser;
+		if ( ! denoiser ) return null;
+
+		return {
+			profile: await denoiser.getLastDenoiseProfile(),
+			runtime: denoiser.getRuntimeInfo(),
+		};
+
+	}
+
 	/**
 	 * Real GPU milliseconds for the last resolved frame, from WebGPU timestamp queries.
 	 * Returns null unless {@link PathTracerApp#enableGPUTiming} was called.

package/src/managers/DenoisingManager.js

@@ -138,6 +138,13 @@ export class DenoisingManager extends EventDispatcher {
 				adapterInfo: null
 			} ),
 
+			refreshInput: () => {
+
+				const ctx = this.pipeline?.context;
+				if ( this._stages.compositor && ctx ) this._stages.compositor.render( ctx );
+
+			},
+
 			getGPUTextures: () => {
 
 				if ( ! pt?.storageTextures?.readTarget ) return null;
@@ -190,6 +197,13 @@ export class DenoisingManager extends EventDispatcher {
 
 			},
 
+			refreshInput: () => {
+
+				const ctx = this.pipeline?.context;
+				if ( this._stages.compositor && ctx ) this._stages.compositor.render( ctx );
+
+			},
+
 			getGPUTextures: () => {
 
 				if ( ! pt?.storageTextures?.readTarget ) return null;
@@ -428,13 +442,6 @@ export class DenoisingManager extends EventDispatcher {
 		// Remove any stale completion-chain listener from a previous render cycle
 		this._cleanupCompletionListener();
 
-		// Show post-process canvas if any post-process is enabled
-		if ( ( this.denoiser?.enabled || this.upscaler?.enabled ) && this.denoiserCanvas ) {
-
-			this.denoiserCanvas.style.display = 'block';
-
-		}
-
 		// Chain: denoise first (if enabled), then upscale (if enabled)
 		const startUpscaler = () => {
 
@@ -458,13 +465,6 @@ export class DenoisingManager extends EventDispatcher {
 
 		} else {
 
-			// Re-render compositor stage so WebGPU canvas has valid content
-			if ( this.upscaler?.enabled && this._stages.compositor && context ) {
-
-				this._stages.compositor.render( context );
-
-			}
-
 			startUpscaler();
 
 		}
@@ -638,8 +638,10 @@ export class DenoisingManager extends EventDispatcher {
 	setOIDNQuality( quality ) {
 
 		this.denoiser?.updateQuality( quality );
-		// Clean-aux normal follows the model: balanced/high (clean-aux models) → accumulate; fast → keep bump.
-		this._stages.pathTracer?.setCleanAuxNormal?.( !! this.denoiser?.enabled && quality !== 'fast' );
+		// Clean-aux normal follows the model, not the tier name — see OIDNDenoiser.QUALITY_MODELS.
+		this._stages.pathTracer?.setCleanAuxNormal?.(
+			!! this.denoiser?.enabled && !! this.denoiser?.expectsCleanAux( quality )
+		);
 		this._onPostProcessRefresh?.();
 
 	}