rayzee 7.23.3 → 7.25.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rayzee",
-  "version": "7.23.3",
+  "version": "7.25.0",
   "type": "module",
   "description": "Real-time WebGPU path tracing engine built on Three.js",
   "main": "dist/rayzee.umd.js",
@@ -18,7 +18,7 @@
   ],
   "repository": {
     "type": "git",
-    "url": "https://github.com/atul-mourya/RayTracing.git",
+    "url": "git+https://github.com/atul-mourya/rayzee-renderer.git",
     "directory": "rayzee"
   },
   "keywords": [
@@ -38,7 +38,7 @@
     "three": ">=0.185.0"
   },
   "optionalDependencies": {
-    "oidn-web": ">=0.3.0",
+    "oidn-web": ">=0.4.0",
     "onnxruntime-web": ">=1.0.0"
   }
 }

package/src/Passes/AIUpscaler.js

@@ -1,5 +1,5 @@
 import { EventDispatcher, ACESFilmicToneMapping } from 'three';
-import { TONE_MAP_FNS, SRGB_GAMMA, applySaturation } from '../Processor/ToneMapCPU.js';
+import { TONE_MAP_FNS, SRGB_GAMMA, applySaturation, effectiveExposure } from '../Processor/ToneMapCPU.js';
 import { getAssetConfig } from '../AssetConfig.js';
 import AIUpscalerWorker from '../Processor/Workers/AIUpscalerWorker.js?worker&inline';
 
@@ -107,6 +107,8 @@ export class AIUpscaler extends EventDispatcher {
 		this.hdr = false;
 		this.scaleFactor = options.scaleFactor || 2;
 		this.quality = options.quality || 'fast';
+		// refreshInput: () => void — re-renders the compositor so the WebGPU canvas is readable
+		this.refreshInput = options.refreshInput || null;
 		this.tileSize = options.tileSize || MODEL_CONFIG.TILE_SIZE;
 		this._tileSizeOverride = !! options.tileSize;
 
@@ -254,6 +256,14 @@ export class AIUpscaler extends EventDispatcher {
 
 	}
 
+	// See OIDNDenoiser._revealOutput. Here the resize below also clears the bitmap.
+	_revealOutput() {
+
+		this.output.style.display = 'block';
+		this.input.style.opacity = '0';
+
+	}
+
 	async execute() {
 
 		if ( ! this.enabled ) return false;
@@ -261,10 +271,6 @@ export class AIUpscaler extends EventDispatcher {
 		this.state.abortController = new AbortController();
 		this.state.isUpscaling = true;
 
-		// Show output canvas and hide input (matching OIDN lifecycle)
-		this.input.style.opacity = '0';
-		this.output.style.display = 'block';
-
 		// Capture source image SYNCHRONOUSLY before any async work.
 		// WebGPU canvas textures expire after each compositor frame,
 		// so we must grab the pixels before awaiting model load.
@@ -292,6 +298,7 @@ export class AIUpscaler extends EventDispatcher {
 		ctx.imageSmoothingEnabled = true;
 		ctx.imageSmoothingQuality = 'high';
 		ctx.drawImage( this._backupCanvas, 0, 0, outW, outH );
+		this._revealOutput();
 
 		try {
 
@@ -393,7 +400,7 @@ export class AIUpscaler extends EventDispatcher {
 		if ( sourceImageData.isHDR ) {
 
 			this._hdrToneMapFn = TONE_MAP_FNS.get( this.getToneMapping() ) || TONE_MAP_FNS.get( ACESFilmicToneMapping );
-			this._hdrExposure = this.getExposure();
+			this._hdrExposure = effectiveExposure( this.getExposure(), this.getToneMapping() );
 			this._hdrSaturation = this.getSaturation();
 			this._tmOut = new Float32Array( 3 );
 
@@ -467,10 +474,11 @@ export class AIUpscaler extends EventDispatcher {
 
 		if ( sourceCanvas && sourceCanvas !== this.output ) {
 
-			// Source is the WebGPU canvas — must copy to 2D canvas first.
-			// WebGPU canvases expire their texture after each compositor frame,
-			// so the caller (PathTracerApp) should re-render the display stage
-			// before triggering the upscaler when OIDN is not used.
+			// Source is the WebGPU canvas — must copy to a 2D canvas first, and it only reads
+			// back non-empty straight after a compositor pass. Enforced here rather than left to
+			// the caller: any awaited GPU work in between silently yields a fully transparent
+			// capture, and the upscaler would then upscale nothing at all.
+			this.refreshInput?.();
 			const offscreen = document.createElement( 'canvas' );
 			offscreen.width = sourceCanvas.width;
 			offscreen.height = sourceCanvas.height;
@@ -785,7 +793,7 @@ export class AIUpscaler extends EventDispatcher {
 			const imageData = ctx.createImageData( width, height );
 			const pixels = imageData.data;
 			const tmFn = TONE_MAP_FNS.get( this.getToneMapping() ) || TONE_MAP_FNS.get( ACESFilmicToneMapping );
-			const exposure = this.getExposure();
+			const exposure = effectiveExposure( this.getExposure(), this.getToneMapping() );
 			const saturation = this.getSaturation();
 			const out = new Float32Array( 3 );
 

package/src/Passes/OIDNDenoiser.js

@@ -4,17 +4,11 @@ import { createLogger, fmt } from '../utils/Logger.js';
 const log = createLogger( 'oidn' );
 
 let _initUNetFromURL = null;
-let _tfEngine = null;
 async function getInitUNetFromURL() {
 
 	if ( ! _initUNetFromURL ) {
 
-		const [ oidnMod, tfMod ] = await Promise.all( [
-			import( 'oidn-web' ),
-			import( '@tensorflow/tfjs-core' )
-		] );
-		_initUNetFromURL = oidnMod.initUNetFromURL;
-		_tfEngine = tfMod.engine;
+		_initUNetFromURL = ( await import( 'oidn-web' ) ).initUNetFromURL;
 
 	}
 
@@ -22,51 +16,36 @@ async function getInitUNetFromURL() {
 
 }
 
-// oidn-web caches its WebGPUBackend in TFJS's global ENGINE under 'webgpu-oidn'.
-// On dispose, drop it so the next instance binds to the new GPUDevice instead of
-// reusing the destroyed one (which would produce black tiles).
-function removeOidnTfjsBackend() {
-
-	if ( ! _tfEngine ) return;
-	try {
-
-		const eng = _tfEngine();
-		if ( eng?.registryFactory && 'webgpu-oidn' in eng.registryFactory ) {
-
-			eng.removeBackend( 'webgpu-oidn' );
-
-		}
-
-	} catch ( e ) {
-
-		log.warn( 'failed to clear cached TFJS backend', e );
-
-	}
-
-}
-
-import { TONE_MAP_FNS, linearToSRGB, applySaturation } from '../Processor/ToneMapCPU.js';
+import { TONE_MAP_FNS, linearToSRGB, applySaturation, effectiveExposure } from '../Processor/ToneMapCPU.js';
 import { getAssetConfig } from '../AssetConfig.js';
 
 /** Reusable RGB output buffer (avoids per-pixel allocation). */
 const _tmOut = new Float32Array( 3 );
 
 const MODEL_CONFIG = {
-	// No cleanAux flag in oidn-web, so the blob is it — and must match what setCleanAuxNormal
-	// feeds: `fast` point-samples the normal (noisy aux), balance/high accumulate it. calb_cnrm
-	// is also the only model with a `_large` blob.
+	// No cleanAux flag in oidn-web, so the blob is it — and it must match what setCleanAuxNormal
+	// feeds the aux MRT: an `alb_nrm` model wants the point-sampled normal, a `calb_cnrm` model
+	// wants the accumulated one. `cleanAux` is carried here rather than inferred from the tier
+	// name, which stopped being possible once a tier other than `fast` used a noisy-aux model.
+	//
+	// `fast-clean` is the same topology and channel width as `fast` (both 641 KB, widest conv 64)
+	// so it costs the same to run, and only differs in being trained for a clean guide.
 	QUALITY_MODELS: {
-		fast: 'rt_hdr_alb_nrm_small',
-		balance: 'rt_hdr_calb_cnrm',
-		high: 'rt_hdr_calb_cnrm_large'
+		fast: { model: 'rt_hdr_alb_nrm_small', cleanAux: false },
+		'fast-clean': { model: 'rt_hdr_calb_cnrm_small', cleanAux: true },
+		balance: { model: 'rt_hdr_calb_cnrm', cleanAux: true },
+		high: { model: 'rt_hdr_calb_cnrm_large', cleanAux: true }
 	},
 	DEFAULT_OPTIONS: {
 		enableOIDN: true,
 		oidnQuality: 'fast',
 		debugGbufferMaps: true,
-		// Tiles run at tileSize + 2*overlap (96px, 112 for _large), so small tiles pay that padding
-		// every tile: 256 puts 3.06x the pixels through the UNet at 512². Measured 3.64x faster.
-		tileSize: 512
+		// A cap, not a fixed size — the effective tile is min( max( w, h ), this ), so a frame
+		// that fits in one tile pays no overlap padding at all. Every tile otherwise runs at
+		// tileSize + 2*overlap (96px, 112 for _large): at 1024²/high, 4x512 tiles measured
+		// 422ms against 229ms for one 1024 tile. 1024 caps the one-time activation
+		// allocation at ~430MB; beyond it larger frames tile and stay bounded.
+		tileSize: 1024
 	}
 };
 
@@ -140,6 +119,8 @@ export class OIDNDenoiser extends EventDispatcher {
 		// getToneMapping: () => number (Three.js ToneMapping constant)
 		this.backendParamsGetter = options.backendParams || null;
 		this.getGPUTextures = options.getGPUTextures || null;
+		// refreshInput: () => void — re-renders the compositor so the WebGPU canvas is readable
+		this.refreshInput = options.refreshInput || null;
 		this.getExposure = options.getExposure || ( () => 1.0 );
 		this.getToneMapping = options.getToneMapping || ( () => ACESFilmicToneMapping );
 		this.getSaturation = options.getSaturation || ( () => 1.0 );
@@ -179,8 +160,8 @@ export class OIDNDenoiser extends EventDispatcher {
 		this.enabled = this.config.enableOIDN;
 		this.quality = this.config.oidnQuality;
 		this.debugGbufferMaps = this.config.debugGbufferMaps;
-		this.tileSize = this.config.tileSize;
-		// tileSize actually baked into the live UNet, so a runtime change forces a rebuild.
+		this.maxTileSize = this.config.tileSize;
+		// The size actually baked into the live UNet, so a change forces a rebuild.
 		this._activeTileSize = 0;
 
 		// State management
@@ -257,6 +238,63 @@ export class OIDNDenoiser extends EventDispatcher {
 
 	}
 
+	/**
+	 * Whether the tier's model expects a clean (accumulated) auxiliary guide rather than a
+	 * point-sampled one. The aux MRT has to be fed to match, or the guide fights the weights.
+	 * @param {string} [quality] defaults to the active tier
+	 * @returns {boolean}
+	 */
+	expectsCleanAux( quality = this.quality ) {
+
+		const { QUALITY_MODELS } = MODEL_CONFIG;
+		return ( QUALITY_MODELS[ quality ] || QUALITY_MODELS.balance ).cleanAux;
+
+	}
+
+	/**
+	 * Engine, precision, model, kernel, tile state and resource counts from the live UNet.
+	 * @returns {Object|null} null before the weights load
+	 */
+	getRuntimeInfo() {
+
+		return this.unet?.getRuntimeInfo?.() ?? null;
+
+	}
+
+	/**
+	 * Arms per-layer GPU timestamping for the next denoise. Needs `timestamp-query` on the
+	 * device; one denoise only, so call it again for each capture.
+	 * @returns {boolean} whether the capture was armed
+	 */
+	profileNextDenoise() {
+
+		return this.unet?.profileNextExecution?.() ?? false;
+
+	}
+
+	/**
+	 * Per-layer GPU timings from the last denoise armed by {@link profileNextDenoise}.
+	 * @returns {Promise<Object|null>}
+	 */
+	async getLastDenoiseProfile() {
+
+		return ( await this.unet?.getLastExecutionProfile?.() ) ?? null;
+
+	}
+
+	/**
+	 * Largest tile that still covers the frame, bounded by the configured cap. One tile means
+	 * zero overlap, which is the cheapest configuration oidn-web has.
+	 * @returns {number}
+	 */
+	_resolveTileSize() {
+
+		const longest = Math.max( this.output?.width || 0, this.output?.height || 0 );
+		if ( ! longest ) return this.maxTileSize;
+		return Math.min( longest, this.maxTileSize );
+
+	}
+
 	async _setupUNetDenoiser() {
 
 		if ( this.state.isLoading ) return;
@@ -264,8 +302,10 @@ export class OIDNDenoiser extends EventDispatcher {
 		this.state.isLoading = true;
 		const tzaUrl = this._generateTzaUrl();
 
+		const tileSize = this._resolveTileSize();
+
 		// maxTileSize is a constructor arg, so comparing the URL alone left tileSize changes inert.
-		if ( this.currentTZAUrl === tzaUrl && this._activeTileSize === this.tileSize && this.unet ) {
+		if ( this.currentTZAUrl === tzaUrl && this._activeTileSize === tileSize && this.unet ) {
 
 			this.state.isLoading = false;
 			return;
@@ -299,11 +339,14 @@ export class OIDNDenoiser extends EventDispatcher {
 			this.unet = await initFn( tzaUrl, backendParams, {
 				aux: true,
 				hdr: true,
-				maxTileSize: this.tileSize
+				maxTileSize: tileSize,
+				// Adaptive tiling ignores maxTileSize, starts at 384, and cannot see that overlap
+				// padding only vanishes once a tile covers the image — so it settles smaller.
+				dynamicTile: false
 			} );
 
 			this.currentTZAUrl = tzaUrl;
-			this._activeTileSize = this.tileSize;
+			this._activeTileSize = tileSize;
 			this.dispatchEvent( { type: 'loaded' } );
 			log.debug( 'UNet weights loaded:', tzaUrl );
 
@@ -333,8 +376,8 @@ export class OIDNDenoiser extends EventDispatcher {
 
 		const { oidnWeightsBaseUrl } = getAssetConfig();
 		const { QUALITY_MODELS } = MODEL_CONFIG;
-		const modelName = QUALITY_MODELS[ this.quality ] || QUALITY_MODELS.balance;
-		return `${oidnWeightsBaseUrl}${modelName}.tza`;
+		const tier = QUALITY_MODELS[ this.quality ] || QUALITY_MODELS.balance;
+		return `${oidnWeightsBaseUrl}${tier.model}.tza`;
 
 	}
 
@@ -349,7 +392,7 @@ export class OIDNDenoiser extends EventDispatcher {
 		Object.assign( this.config, newConfig );
 		this.quality = this.config.oidnQuality;
 		this.debugGbufferMaps = this.config.debugGbufferMaps;
-		this.tileSize = this.config.tileSize;
+		this.maxTileSize = this.config.tileSize;
 
 		// Reload denoiser if necessary
 		await this._setupUNetDenoiser();
@@ -376,6 +419,14 @@ export class OIDNDenoiser extends EventDispatcher {
 
 		}
 
+		// The tile tracks the frame size, so a resolution change since the last build has to be
+		// rebuilt into the UNet before denoising — otherwise the frame tiles when it need not.
+		if ( this.unet && ! this.state.isLoading && this._activeTileSize !== this._resolveTileSize() ) {
+
+			await this._setupUNetDenoiser();
+
+		}
+
 		// UNet weights still loading — defer the start (fired from _setupUNetDenoiser's
 		// finally) rather than silently dropping this frame's denoise request.
 		if ( this.state.isLoading ) {
@@ -404,6 +455,15 @@ export class OIDNDenoiser extends EventDispatcher {
 
 	}
 
+	// Revealing before the first paint shows the output's transparent bitmap — and its
+	// checker CSS background — over the now-hidden render canvas.
+	_revealOutput() {
+
+		this.output.style.display = 'block';
+		this.input.style.opacity = '0';
+
+	}
+
 	async execute() {
 
 		if ( ! this.enabled || ! this.unet ) return false;
@@ -411,8 +471,6 @@ export class OIDNDenoiser extends EventDispatcher {
 		// Create abort controller for this execution
 		this.state.abortController = new AbortController();
 		this.state.isDenoising = true;
-		this.input.style.opacity = '0';
-		this.output.style.display = 'block';
 
 		try {
 
@@ -485,6 +543,12 @@ export class OIDNDenoiser extends EventDispatcher {
 
 		}
 
+		// Capture the base now: the readback below awaits, and a presented WebGPU canvas only
+		// reads back non-empty straight after a compositor pass — hence the refresh.
+		this.refreshInput?.();
+		this.ctx.drawImage( this.input, 0, 0, width, height );
+		this._revealOutput();
+
 		// Ensure storage buffers are sized correctly (recreate on resolution change)
 		this._ensureGPUInputBuffers( width, height );
 
@@ -543,8 +607,6 @@ export class OIDNDenoiser extends EventDispatcher {
 		if ( globalThis.__OIDN_NO_INPUT_SCALE ) this._oidnInputScale = 1.0;
 		this._applyColorScale( device, width * height );
 
-		// Draw the current noisy frame as the base — denoised tiles paint on top progressively
-		this.ctx.drawImage( this.input, 0, 0, width, height );
 		// Pass GPU storage buffers to oidn-web (GPUBuffer path, well-tested)
 		const config = {
 			color: { data: this._gpuInputBuffers.color, width, height },
@@ -702,7 +764,7 @@ export class OIDNDenoiser extends EventDispatcher {
 	/** Display exposure with the input scale divided back out — output arrives pre-multiplied. */
 	_outputExposure() {
 
-		return this.getExposure() / this._oidnInputScale;
+		return effectiveExposure( this.getExposure(), this.getToneMapping() ) / this._oidnInputScale;
 
 	}
 
@@ -1094,9 +1156,6 @@ export class OIDNDenoiser extends EventDispatcher {
 
 		// Dispose resources
 		this.unet?.dispose();
-		// Must precede renderer.dispose() so the GPUDevice is still alive when
-		// TFJS tears down the cached backend's buffers/textures.
-		removeOidnTfjsBackend();
 		this._destroyGPUInputBuffers();
 		this._colorScalePipeline = null;
 

package/src/PathTracerApp.js

@@ -366,8 +366,11 @@ export class PathTracerApp extends EventDispatcher {
 				memoryPeak: tracker?.peak ?? 0,
 			} );
 
-			// Check time limit
-			if ( this.completion.isTimeLimitReached( this.settings.get( 'renderLimitMode' ), this.settings.get( 'renderTimeLimit' ) ) ) {
+			// Only the wall-clock stop — PathTracer.render() retires the ceiling and convergence
+			// itself, so whichever of the three arrives first wins.
+			if ( this.completion.isTimeLimitReached(
+				this.stages.pathTracer, this.settings.get( 'renderLimitMode' ), this.settings.get( 'renderTimeLimit' )
+			) ) {
 
 				this.stages.pathTracer.isComplete = true;
 
@@ -381,8 +384,14 @@ export class PathTracerApp extends EventDispatcher {
 					context: this.pipeline?.context,
 				} );
 
-				this.dispatchEvent( { type: 'RenderComplete' } );
-				this.dispatchEvent( { type: EngineEvents.RENDER_COMPLETE } );
+				const completionInfo = {
+					samples: this.stages.pathTracer.frameCount,
+					timeElapsed: this.completion.timeElapsed,
+					budgetOverrun: this.completion.budgetOverrun,
+				};
+
+				this.dispatchEvent( { type: 'RenderComplete', ...completionInfo } );
+				this.dispatchEvent( { type: EngineEvents.RENDER_COMPLETE, ...completionInfo } );
 
 			}
 
@@ -1808,6 +1817,39 @@ export class PathTracerApp extends EventDispatcher {
 
 	}
 
+	/**
+	 * Arms per-layer GPU timestamping inside the OIDN denoiser for the next denoise only.
+	 * Read the result with {@link PathTracerApp#getDenoiseProfile}.
+	 *
+	 * `getGPUTimings()` cannot see the denoise: oidn-web submits on its own command encoders,
+	 * outside the stages three.js times.
+	 *
+	 * @returns {boolean} whether the capture was armed
+	 */
+	profileNextDenoise() {
+
+		return this.denoisingManager?.denoiser?.profileNextDenoise() ?? false;
+
+	}
+
+	/**
+	 * Per-layer GPU timings from the denoise armed by {@link PathTracerApp#profileNextDenoise},
+	 * plus the live engine/precision/model/tile diagnostics.
+	 *
+	 * @returns {Promise<{ profile: Object|null, runtime: Object|null }|null>}
+	 */
+	async getDenoiseProfile() {
+
+		const denoiser = this.denoisingManager?.denoiser;
+		if ( ! denoiser ) return null;
+
+		return {
+			profile: await denoiser.getLastDenoiseProfile(),
+			runtime: denoiser.getRuntimeInfo(),
+		};
+
+	}
+
 	/**
 	 * Real GPU milliseconds for the last resolved frame, from WebGPU timestamp queries.
 	 * Returns null unless {@link PathTracerApp#enableGPUTiming} was called.

package/src/Pipeline/CompletionTracker.js

@@ -1,3 +1,7 @@
+import { createLogger } from '../utils/Logger.js';
+
+const log = createLogger( 'engine' );
+
 /**
  * Tracks render completion state, time limits, and sample limits.
  *
@@ -12,6 +16,10 @@ export class CompletionTracker {
 		this.lastResetTime = performance.now();
 		this.renderCompleteDispatched = false;
 
+		/** Whether the deadline — rather than the ceiling or convergence — retired the frame. */
+		this.budgetOverrun = false;
+		this._budgetHeldWarned = false;
+
 	}
 
 	/**
@@ -24,19 +32,48 @@ export class CompletionTracker {
 	}
 
 	/**
-	 * Checks whether the time-based render limit has been reached.
-	 * @param {string} renderLimitMode - 'time' or 'samples'
+	 * Checks whether the wall-clock render budget has expired. An additional stop condition,
+	 * never a replacement — the sample ceiling and convergence still bind when they come first.
+	 *
+	 * Floored at one sample: frameCount does not advance while the stage compiles kernels or
+	 * while the camera is in interaction mode, and retiring in that window denoises an empty
+	 * buffer and captures a blank canvas.
+	 *
+	 * @param {Object} pathTracer - The PathTracer stage
+	 * @param {string} renderLimitMode - 'time' or 'frames'
 	 * @param {number} renderTimeLimit - Time limit in seconds
 	 * @returns {boolean}
 	 */
-	isTimeLimitReached( renderLimitMode, renderTimeLimit ) {
+	isTimeLimitReached( pathTracer, renderLimitMode, renderTimeLimit ) {
+
+		if ( renderLimitMode !== 'time' || ! ( renderTimeLimit > 0 ) ) return false;
+		if ( this.timeElapsed < renderTimeLimit ) return false;
+
+		if ( ! ( pathTracer?.frameCount > 0 ) ) {
 
-		return renderLimitMode === 'time' && renderTimeLimit > 0 && this.timeElapsed >= renderTimeLimit;
+			if ( ! this._budgetHeldWarned ) {
+
+				this._budgetHeldWarned = true;
+				log.warn(
+					`render time limit (${renderTimeLimit}s) expired before the first sample landed — ` +
+					'holding the deadline open to avoid capturing an empty frame'
+				);
+
+			}
+
+			return false;
+
+		}
+
+		this.budgetOverrun = true;
+		return true;
 
 	}
 
 	/**
-	 * Checks whether ANY render limit (time or samples) is reached.
+	 * Checks whether ANY render limit is reached: sample ceiling, convergence, or time budget,
+	 * whichever arrives first.
+	 *
 	 * @param {Object} pathTracer - The PathTracer stage
 	 * @param {string} renderLimitMode
 	 * @param {number} renderTimeLimit
@@ -46,7 +83,7 @@ export class CompletionTracker {
 
 		if ( ! pathTracer ) return false;
 
-		if ( this.isTimeLimitReached( renderLimitMode, renderTimeLimit ) ) return true;
+		if ( this.isTimeLimitReached( pathTracer, renderLimitMode, renderTimeLimit ) ) return true;
 
 		// Tier-1 convergence early-stop must agree with PathTracer.render()'s own check, else the app-level
 		// reconcile would flip isComplete back off and keep dispatching after the frame converged.
@@ -75,6 +112,8 @@ export class CompletionTracker {
 		this.timeElapsed = 0;
 		this.lastResetTime = performance.now();
 		this.renderCompleteDispatched = false;
+		this.budgetOverrun = false;
+		this._budgetHeldWarned = false;
 
 	}