rayzee 8.6.0 → 8.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -53
- package/dist/{DLSSNeural-BhBCBgKf.js → NeuralRetouch-CYOJ21Wn.js} +31 -31
- package/dist/{DLSSSuperRes-BCA7F_k5.js → NeuralSuperRes-Dc6jRYke.js} +23 -23
- package/dist/{ToneMapGPU-BzcLheoR.js → ToneMapGPU-BVE852iN.js} +2 -2
- package/dist/rayzee.es.js +49 -39
- package/dist/rayzee.umd.js +5 -5
- package/package.json +1 -1
- package/src/AssetConfig.js +6 -5
- package/src/Passes/OIDNDenoiser.js +13 -5
- package/src/index.js +3 -3
- package/src/managers/DenoisingManager.js +60 -44
- package/src/{dlss/DLSSNeural.js → neural/NeuralRetouch.js} +42 -41
- package/src/{dlss/DLSSSuperRes.js → neural/NeuralSuperRes.js} +27 -26
package/package.json
CHANGED
package/src/AssetConfig.js
CHANGED
|
@@ -45,12 +45,13 @@ const config = {
|
|
|
45
45
|
// AI upscaler ONNX model base URL. Quality presets resolve relative paths against this.
|
|
46
46
|
upscalerModelBaseUrl: 'https://huggingface.co/notaneimu/onnx-image-models/resolve/main/',
|
|
47
47
|
|
|
48
|
-
//
|
|
49
|
-
// `globalThis.
|
|
50
|
-
|
|
48
|
+
// Neural post-pass runtime. A plain script the host serves, not a module: it installs
|
|
49
|
+
// `globalThis.NeuralRuntime`, so it cannot be imported and the host must publish it somewhere.
|
|
50
|
+
neuralRuntimeUrl: '/neural/neural-runtime.js',
|
|
51
51
|
|
|
52
|
-
// Base for the
|
|
53
|
-
|
|
52
|
+
// Base for the neural weight manifests the runtime fetches (`generated/...` beneath this). No
|
|
53
|
+
// default: the weights are not redistributable, so a host supplies its own copy.
|
|
54
|
+
neuralAssetBaseUrl: null,
|
|
54
55
|
|
|
55
56
|
// Prefix used when the engine writes to client-side stores (IndexedDB, etc).
|
|
56
57
|
// Set to a unique value to avoid collisions when multiple apps embed the engine on the same origin.
|
|
@@ -688,6 +688,14 @@ export class OIDNDenoiser extends EventDispatcher {
|
|
|
688
688
|
this._computeInputScale( device, width * height );
|
|
689
689
|
this._applyColorScale( device, width * height );
|
|
690
690
|
|
|
691
|
+
// A final denoise is shown tile by tile, so the tiles still to come must show this render
|
|
692
|
+
// rather than the blank or previous-view picture the output holds between runs.
|
|
693
|
+
if ( ! continuous && ! this._hasOutput ) {
|
|
694
|
+
|
|
695
|
+
this._unpackToTexture( this._gpuInputBuffers.color, width, { x: 0, y: 0, width, height }, false );
|
|
696
|
+
|
|
697
|
+
}
|
|
698
|
+
|
|
691
699
|
// Pass GPU storage buffers to oidn-web (GPUBuffer path, well-tested)
|
|
692
700
|
const config = {
|
|
693
701
|
color: { data: this._gpuInputBuffers.color, width, height },
|
|
@@ -996,11 +1004,12 @@ export class OIDNDenoiser extends EventDispatcher {
|
|
|
996
1004
|
/**
|
|
997
1005
|
* Writes a rectangle of the denoised buffer into the output picture, on the card.
|
|
998
1006
|
*
|
|
999
|
-
* @param {GPUBuffer} src -
|
|
1007
|
+
* @param {GPUBuffer} src - rgba32float, full image, scaled by the input autoexposure
|
|
1000
1008
|
* @param {number} srcWidth
|
|
1001
1009
|
* @param {{x: number, y: number, width: number, height: number}} rect
|
|
1010
|
+
* @param {boolean} [denoised=true] - false for the raw render laid under a tiled run
|
|
1002
1011
|
*/
|
|
1003
|
-
_unpackToTexture( src, srcWidth, rect ) {
|
|
1012
|
+
_unpackToTexture( src, srcWidth, rect, denoised = true ) {
|
|
1004
1013
|
|
|
1005
1014
|
const device = this.gpuDevice;
|
|
1006
1015
|
if ( ! device || ! this._ensureScalePipelines( device ) ) return;
|
|
@@ -1047,7 +1056,7 @@ export class OIDNDenoiser extends EventDispatcher {
|
|
|
1047
1056
|
// Set here rather than when the run resolves: 'end' is dispatched from execute()'s
|
|
1048
1057
|
// finally, which runs first, so a listener that waits for the run would see the very
|
|
1049
1058
|
// first denoise as having produced nothing.
|
|
1050
|
-
this._hasOutput = true;
|
|
1059
|
+
if ( denoised ) this._hasOutput = true;
|
|
1051
1060
|
|
|
1052
1061
|
} finally {
|
|
1053
1062
|
|
|
@@ -1060,8 +1069,7 @@ export class OIDNDenoiser extends EventDispatcher {
|
|
|
1060
1069
|
|
|
1061
1070
|
/**
|
|
1062
1071
|
* Promise wrapper around tileExecute for the GPU path. Each tile is written straight into the
|
|
1063
|
-
* output picture on the card
|
|
1064
|
-
* being visible.
|
|
1072
|
+
* output picture on the card; 'tileProgress' is what puts it on screen.
|
|
1065
1073
|
*/
|
|
1066
1074
|
_executeWithAbortGPU( config, continuous = false ) {
|
|
1067
1075
|
|
package/src/index.js
CHANGED
|
@@ -99,11 +99,11 @@ export { VideoRenderManager } from './managers/VideoRenderManager.js';
|
|
|
99
99
|
// Interaction
|
|
100
100
|
export { InteractionManager } from './managers/InteractionManager.js';
|
|
101
101
|
|
|
102
|
-
//
|
|
102
|
+
// Neural post passes. The app drives these through `denoisingManager` — what is exported here is
|
|
103
103
|
// the offline entry point (`renderUpscaled`, used by the regression bench) plus the two contracts a
|
|
104
104
|
// host needs to build UI against.
|
|
105
|
-
export { renderUpscaled, SR_SCALE, SR_MAX_INPUT } from './
|
|
106
|
-
export {
|
|
105
|
+
export { renderUpscaled, SR_SCALE, SR_MAX_INPUT } from './neural/NeuralSuperRes.js';
|
|
106
|
+
export { RETOUCH_DEFAULTS, RETOUCH_RANGES, RETOUCH_MAX_PIXELS } from './neural/NeuralRetouch.js';
|
|
107
107
|
|
|
108
108
|
// ── Colour management ───────────────────────────────────────────────────────────────────────────
|
|
109
109
|
// `app.color` is the instance a host normally uses; these are for building UI against it, and for
|
|
@@ -4,9 +4,9 @@ import { AIUpscaler } from '../Passes/AIUpscaler.js';
|
|
|
4
4
|
import { EngineEvents } from '../EngineEvents.js';
|
|
5
5
|
import { createLogger } from '../utils/Logger.js';
|
|
6
6
|
|
|
7
|
-
// The neural passes live in `../
|
|
8
|
-
// share one namespace: `rayzee.log.only( '
|
|
9
|
-
const
|
|
7
|
+
// The neural passes live in `../neural/` but report through the manager that drives them, so they
|
|
8
|
+
// share one namespace: `rayzee.log.only( 'neural' )` shows the whole chain.
|
|
9
|
+
const neuralLog = createLogger( 'neural' );
|
|
10
10
|
import { ENGINE_DEFAULTS as DEFAULT_STATE, ASVGF_QUALITY_PRESETS, NRD_DEFAULTS, NRD_QUALITY_PRESETS, NRD_PRESET_KEYS } from '../EngineDefaults.js';
|
|
11
11
|
|
|
12
12
|
// A refresh slower than this is a slideshow, not a live view, so the cadence swaps to a cheaper
|
|
@@ -112,16 +112,16 @@ export class DenoisingManager extends EventDispatcher {
|
|
|
112
112
|
|
|
113
113
|
this.denoiser = null;
|
|
114
114
|
this.upscaler = null;
|
|
115
|
-
// Which model the AI upscaler runs. 'esrgan' is the ONNX chain; '
|
|
115
|
+
// Which model the AI upscaler runs. 'esrgan' is the ONNX chain; 'neural' is the neural
|
|
116
116
|
// super-resolution pass, which is fixed at 2x and needs a denoised source.
|
|
117
117
|
this.upscalerBackend = 'esrgan';
|
|
118
|
-
this.
|
|
118
|
+
this._neuralUpscaler = null;
|
|
119
119
|
// The neural-rendering (detail) pass is independent of the upscaler: it changes appearance,
|
|
120
120
|
// not resolution, and runs last because its result cannot be read back.
|
|
121
121
|
this.neuralRendering = false;
|
|
122
122
|
this.neuralRenderingSettings = {};
|
|
123
|
-
this.
|
|
124
|
-
this.
|
|
123
|
+
this._retouch = null;
|
|
124
|
+
this._superResModule = null;
|
|
125
125
|
this._neuralPostBusy = false;
|
|
126
126
|
|
|
127
127
|
// The tier the finished image uses. The loaded tier is not always this one: while the
|
|
@@ -162,6 +162,7 @@ export class DenoisingManager extends EventDispatcher {
|
|
|
162
162
|
// Bound event forwarding handlers (stored for removal on re-setup / dispose)
|
|
163
163
|
this._denoiserStartHandler = null;
|
|
164
164
|
this._denoiserEndHandler = null;
|
|
165
|
+
this._denoiserTileHandler = null;
|
|
165
166
|
this._upscalerResChangedHandler = null;
|
|
166
167
|
this._upscalerStartHandler = null;
|
|
167
168
|
this._upscalerProgressHandler = null;
|
|
@@ -298,8 +299,21 @@ export class DenoisingManager extends EventDispatcher {
|
|
|
298
299
|
|
|
299
300
|
};
|
|
300
301
|
|
|
302
|
+
// The loop has stopped by the time a final denoise runs, so each tile has to redraw the
|
|
303
|
+
// canvas itself. Not through the display-refresh callback: that restarts the loop, which a
|
|
304
|
+
// video export drives by hand.
|
|
305
|
+
this._denoiserTileHandler = e => {
|
|
306
|
+
|
|
307
|
+
if ( e.continuous ) return;
|
|
308
|
+
this._publishOutput();
|
|
309
|
+
const ctx = this.pipeline?.context;
|
|
310
|
+
if ( this._stages.compositor && ctx ) this._stages.compositor.render( ctx );
|
|
311
|
+
|
|
312
|
+
};
|
|
313
|
+
|
|
301
314
|
this.denoiser.addEventListener( 'start', this._denoiserStartHandler );
|
|
302
315
|
this.denoiser.addEventListener( 'end', this._denoiserEndHandler );
|
|
316
|
+
this.denoiser.addEventListener( 'tileProgress', this._denoiserTileHandler );
|
|
303
317
|
|
|
304
318
|
}
|
|
305
319
|
|
|
@@ -804,20 +818,20 @@ export class DenoisingManager extends EventDispatcher {
|
|
|
804
818
|
}
|
|
805
819
|
|
|
806
820
|
/**
|
|
807
|
-
* Selects the AI upscaler's model. '
|
|
821
|
+
* Selects the AI upscaler's model. 'neural' is fixed at 2x and reads the denoised picture, so it
|
|
808
822
|
* is a no-op without a denoiser — feeding it raw Monte-Carlo noise measurably loses to bilinear.
|
|
809
823
|
*
|
|
810
|
-
* @param {'esrgan'|'
|
|
824
|
+
* @param {'esrgan'|'neural'} backend
|
|
811
825
|
*/
|
|
812
826
|
setUpscalerBackend( backend ) {
|
|
813
827
|
|
|
814
|
-
const next = backend === '
|
|
828
|
+
const next = backend === 'neural' ? 'neural' : 'esrgan';
|
|
815
829
|
if ( next === this.upscalerBackend ) return;
|
|
816
830
|
|
|
817
831
|
this.upscalerBackend = next;
|
|
818
832
|
// Whatever is on screen came from the model being switched away from.
|
|
819
833
|
this._restoreRenderDisplay();
|
|
820
|
-
if ( next !== '
|
|
834
|
+
if ( next !== 'neural' ) this._releaseNeuralUpscaler();
|
|
821
835
|
|
|
822
836
|
}
|
|
823
837
|
|
|
@@ -847,15 +861,15 @@ export class DenoisingManager extends EventDispatcher {
|
|
|
847
861
|
|
|
848
862
|
}
|
|
849
863
|
|
|
850
|
-
|
|
864
|
+
_releaseNeuralUpscaler() {
|
|
851
865
|
|
|
852
|
-
this.
|
|
853
|
-
this.
|
|
866
|
+
this._neuralUpscaler?.dispose();
|
|
867
|
+
this._neuralUpscaler = null;
|
|
854
868
|
|
|
855
869
|
// The frame packer lives on the module, not on the upscaler, and holds two buffers the size
|
|
856
870
|
// of the render. Nothing else reaches it, so this is its only release point. Guarded on the
|
|
857
871
|
// module having been loaded, so a teardown never pulls in 1 MB of runtime to call a no-op.
|
|
858
|
-
this.
|
|
872
|
+
this._superResModule?.releaseDenoisedReader();
|
|
859
873
|
|
|
860
874
|
}
|
|
861
875
|
|
|
@@ -866,17 +880,17 @@ export class DenoisingManager extends EventDispatcher {
|
|
|
866
880
|
if ( settings ) this.neuralRenderingSettings = { ...this.neuralRenderingSettings, ...settings };
|
|
867
881
|
if ( ! this.neuralRendering ) {
|
|
868
882
|
|
|
869
|
-
this.
|
|
883
|
+
this._releaseNeuralRetouch();
|
|
870
884
|
this._restoreRenderDisplay();
|
|
871
885
|
|
|
872
886
|
}
|
|
873
887
|
|
|
874
888
|
}
|
|
875
889
|
|
|
876
|
-
|
|
890
|
+
_releaseNeuralRetouch() {
|
|
877
891
|
|
|
878
|
-
this.
|
|
879
|
-
this.
|
|
892
|
+
this._retouch?.dispose();
|
|
893
|
+
this._retouch = null;
|
|
880
894
|
|
|
881
895
|
}
|
|
882
896
|
|
|
@@ -894,7 +908,7 @@ export class DenoisingManager extends EventDispatcher {
|
|
|
894
908
|
*/
|
|
895
909
|
async _runNeuralPost( isStillComplete ) {
|
|
896
910
|
|
|
897
|
-
const wantSR = this.upscaler?.enabled && this.upscalerBackend === '
|
|
911
|
+
const wantSR = this.upscaler?.enabled && this.upscalerBackend === 'neural';
|
|
898
912
|
const wantNR = this.neuralRendering;
|
|
899
913
|
if ( ! wantSR && ! wantNR ) return;
|
|
900
914
|
|
|
@@ -904,14 +918,14 @@ export class DenoisingManager extends EventDispatcher {
|
|
|
904
918
|
|
|
905
919
|
if ( ! this.upscalerCanvas ) {
|
|
906
920
|
|
|
907
|
-
|
|
921
|
+
neuralLog.warn( 'neural pass skipped: the engine has no overlay canvas to present on' );
|
|
908
922
|
return;
|
|
909
923
|
|
|
910
924
|
}
|
|
911
925
|
|
|
912
926
|
if ( ! this.pipeline?.context?.getTexture( 'oidn:output' ) ) {
|
|
913
927
|
|
|
914
|
-
|
|
928
|
+
neuralLog.warn( 'neural pass skipped: no denoised picture — it needs OIDN' );
|
|
915
929
|
return;
|
|
916
930
|
|
|
917
931
|
}
|
|
@@ -921,26 +935,26 @@ export class DenoisingManager extends EventDispatcher {
|
|
|
921
935
|
|
|
922
936
|
try {
|
|
923
937
|
|
|
924
|
-
const sr = await import( '../
|
|
925
|
-
this.
|
|
938
|
+
const sr = await import( '../neural/NeuralSuperRes.js' );
|
|
939
|
+
this._superResModule = sr;
|
|
926
940
|
|
|
927
941
|
// The detail pass has a size ceiling — above it a run takes minutes and then loses every GPU
|
|
928
942
|
// device in the page. Running it FIRST is what keeps it under: at render size it sees a
|
|
929
943
|
// quarter of the pixels it would after a 2x upscale, and the render reserve already caps
|
|
930
944
|
// that at 2048. The check stays as a floor under a raised reserve.
|
|
931
|
-
const nrModule = wantNR ? await import( '../
|
|
945
|
+
const nrModule = wantNR ? await import( '../neural/NeuralRetouch.js' ) : null;
|
|
932
946
|
const srcSize = this.denoiser?._outTexSize;
|
|
933
947
|
const nrPixels = ( srcSize?.width ?? 0 ) * ( srcSize?.height ?? 0 );
|
|
934
|
-
const runNR = wantNR && nrPixels <= nrModule.
|
|
948
|
+
const runNR = wantNR && nrPixels <= nrModule.RETOUCH_MAX_PIXELS;
|
|
935
949
|
|
|
936
950
|
if ( wantNR && ! runNR ) {
|
|
937
951
|
|
|
938
|
-
|
|
952
|
+
neuralLog.warn(
|
|
939
953
|
`Neural rendering skipped: the render is ${( nrPixels / 1e6 ).toFixed( 1 )} MP, above the ` +
|
|
940
|
-
`${( nrModule.
|
|
954
|
+
`${( nrModule.RETOUCH_MAX_PIXELS / 1e6 ).toFixed( 1 )} MP the pass survives.` +
|
|
941
955
|
( wantSR ? ' The upscale still ran.' : '' )
|
|
942
956
|
);
|
|
943
|
-
this.
|
|
957
|
+
this._releaseNeuralRetouch();
|
|
944
958
|
|
|
945
959
|
}
|
|
946
960
|
|
|
@@ -961,16 +975,16 @@ export class DenoisingManager extends EventDispatcher {
|
|
|
961
975
|
|
|
962
976
|
if ( wantSR ) {
|
|
963
977
|
|
|
964
|
-
if ( this.
|
|
965
|
-
&& ( this.
|
|
978
|
+
if ( this._neuralUpscaler
|
|
979
|
+
&& ( this._neuralUpscaler.inputWidth !== source.width || this._neuralUpscaler.inputHeight !== source.height ) ) {
|
|
966
980
|
|
|
967
|
-
this.
|
|
981
|
+
this._releaseNeuralUpscaler();
|
|
968
982
|
|
|
969
983
|
}
|
|
970
984
|
|
|
971
|
-
if ( ! this.
|
|
985
|
+
if ( ! this._neuralUpscaler ) {
|
|
972
986
|
|
|
973
|
-
this.
|
|
987
|
+
this._neuralUpscaler = await sr.NeuralSuperRes.create( { width: source.width, height: source.height } );
|
|
974
988
|
|
|
975
989
|
}
|
|
976
990
|
|
|
@@ -979,23 +993,23 @@ export class DenoisingManager extends EventDispatcher {
|
|
|
979
993
|
let image = source;
|
|
980
994
|
|
|
981
995
|
// Detail first, upscale second. The detail pass hands back scene-referred light (see
|
|
982
|
-
// `
|
|
996
|
+
// the runtime's `rayzee-patch` edits), so the upscaler still gets the linear HDR it expects —
|
|
983
997
|
// which is what makes this order possible at all.
|
|
984
998
|
if ( runNR ) {
|
|
985
999
|
|
|
986
1000
|
const result = await nrModule.enhanceFrame( {
|
|
987
1001
|
source: image,
|
|
988
1002
|
settings: this.neuralRenderingSettings,
|
|
989
|
-
instance: this.
|
|
1003
|
+
instance: this._retouch,
|
|
990
1004
|
exposure: tone.exposure,
|
|
991
1005
|
tone: wantSR ? null : tone,
|
|
992
1006
|
} );
|
|
993
|
-
this.
|
|
1007
|
+
this._retouch = result.instance;
|
|
994
1008
|
image = result;
|
|
995
1009
|
|
|
996
1010
|
}
|
|
997
1011
|
|
|
998
|
-
if ( wantSR ) image = await this.
|
|
1012
|
+
if ( wantSR ) image = await this._neuralUpscaler.upscaleToRGBA8( image, tone );
|
|
999
1013
|
|
|
1000
1014
|
if ( ! isStillComplete() ) {
|
|
1001
1015
|
|
|
@@ -1024,7 +1038,7 @@ export class DenoisingManager extends EventDispatcher {
|
|
|
1024
1038
|
|
|
1025
1039
|
} catch ( error ) {
|
|
1026
1040
|
|
|
1027
|
-
|
|
1041
|
+
neuralLog.warn( 'neural pass failed', error );
|
|
1028
1042
|
|
|
1029
1043
|
} finally {
|
|
1030
1044
|
|
|
@@ -1060,7 +1074,7 @@ export class DenoisingManager extends EventDispatcher {
|
|
|
1060
1074
|
|
|
1061
1075
|
// One owner of the overlay canvas, so the neural chain and the ONNX upscaler are
|
|
1062
1076
|
// exclusive. The neural chain also runs on its own when only its detail pass is on.
|
|
1063
|
-
if ( this.neuralRendering || ( this.upscaler?.enabled && this.upscalerBackend === '
|
|
1077
|
+
if ( this.neuralRendering || ( this.upscaler?.enabled && this.upscalerBackend === 'neural' ) ) {
|
|
1064
1078
|
|
|
1065
1079
|
this._runNeuralPost( isStillComplete );
|
|
1066
1080
|
return;
|
|
@@ -1184,8 +1198,8 @@ export class DenoisingManager extends EventDispatcher {
|
|
|
1184
1198
|
// Remove pending completion-chain listener
|
|
1185
1199
|
this._cleanupCompletionListener();
|
|
1186
1200
|
// Owns a second GPUDevice plus 3 MB of weights, so it must not outlive the manager.
|
|
1187
|
-
this.
|
|
1188
|
-
this.
|
|
1201
|
+
this._releaseNeuralUpscaler();
|
|
1202
|
+
this._releaseNeuralRetouch();
|
|
1189
1203
|
|
|
1190
1204
|
// Before the denoiser destroys the picture the Compositor would otherwise still sample.
|
|
1191
1205
|
this._unpublishOutput();
|
|
@@ -1194,6 +1208,7 @@ export class DenoisingManager extends EventDispatcher {
|
|
|
1194
1208
|
|
|
1195
1209
|
if ( this._denoiserStartHandler ) this.denoiser.removeEventListener( 'start', this._denoiserStartHandler );
|
|
1196
1210
|
if ( this._denoiserEndHandler ) this.denoiser.removeEventListener( 'end', this._denoiserEndHandler );
|
|
1211
|
+
if ( this._denoiserTileHandler ) this.denoiser.removeEventListener( 'tileProgress', this._denoiserTileHandler );
|
|
1197
1212
|
this.denoiser.dispose();
|
|
1198
1213
|
this.denoiser = null;
|
|
1199
1214
|
|
|
@@ -1207,15 +1222,16 @@ export class DenoisingManager extends EventDispatcher {
|
|
|
1207
1222
|
if ( this._upscalerEndHandler ) this.upscaler.removeEventListener( 'end', this._upscalerEndHandler );
|
|
1208
1223
|
this.upscaler.dispose();
|
|
1209
1224
|
this.upscaler = null;
|
|
1210
|
-
// Which model the AI upscaler runs. 'esrgan' is the ONNX chain; '
|
|
1225
|
+
// Which model the AI upscaler runs. 'esrgan' is the ONNX chain; 'neural' is the neural
|
|
1211
1226
|
// super-resolution pass, which is fixed at 2x and needs a denoised source.
|
|
1212
1227
|
this.upscalerBackend = 'esrgan';
|
|
1213
|
-
this.
|
|
1228
|
+
this._neuralUpscaler = null;
|
|
1214
1229
|
|
|
1215
1230
|
}
|
|
1216
1231
|
|
|
1217
1232
|
this._denoiserStartHandler = null;
|
|
1218
1233
|
this._denoiserEndHandler = null;
|
|
1234
|
+
this._denoiserTileHandler = null;
|
|
1219
1235
|
this._upscalerResChangedHandler = null;
|
|
1220
1236
|
this._upscalerStartHandler = null;
|
|
1221
1237
|
this._upscalerProgressHandler = null;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* Retouch — the neural-rendering (detail) pass.
|
|
3
3
|
*
|
|
4
4
|
* Runs over a finished, denoised image and adjusts its appearance: local tone and structure, and a
|
|
5
5
|
* skin-specific term. It does not change resolution and it does not denoise. Measured on path-traced
|
|
@@ -36,7 +36,7 @@ import { getAssetConfig } from '../AssetConfig.js';
|
|
|
36
36
|
* The model's own settings contract. Every field is numeric and validated hard by the runtime —
|
|
37
37
|
* passing a string throws `"<name> must be finite"`.
|
|
38
38
|
*/
|
|
39
|
-
export const
|
|
39
|
+
export const RETOUCH_DEFAULTS = Object.freeze( {
|
|
40
40
|
enabled: true,
|
|
41
41
|
intensity: 1,
|
|
42
42
|
localTone: 1,
|
|
@@ -52,7 +52,7 @@ export const DLSS_NR_DEFAULTS = Object.freeze( {
|
|
|
52
52
|
* How much of the network's own colour to take, 0..1. Ours, not the runtime's.
|
|
53
53
|
*
|
|
54
54
|
* ⚠️ It must be kept OUT of the object handed to the runtime: that one is validated by key and
|
|
55
|
-
* throws `Unknown
|
|
55
|
+
* throws `Unknown Retouch setting: colorStrength`. `splitSettings` does the separating.
|
|
56
56
|
*
|
|
57
57
|
* It is `color_strength` in the composition kernel, blending between the original chroma relit to
|
|
58
58
|
* the network's luminance (`original * ratio`, at 0) and the network's full colour (at 1). At 0 the
|
|
@@ -65,13 +65,13 @@ export const DLSS_NR_DEFAULTS = Object.freeze( {
|
|
|
65
65
|
* are the same at all three, so nothing the pass is actually for is given up. The renderer's chroma
|
|
66
66
|
* is ground truth here; the network's is a guess about a photograph.
|
|
67
67
|
*/
|
|
68
|
-
export const
|
|
68
|
+
export const RETOUCH_COLOR_STRENGTH = 0;
|
|
69
69
|
|
|
70
70
|
/** @returns {{runtime: object, colorStrength: number}} */
|
|
71
71
|
function splitSettings( settings = {} ) {
|
|
72
72
|
|
|
73
|
-
const { colorStrength =
|
|
74
|
-
return { runtime: { ...
|
|
73
|
+
const { colorStrength = RETOUCH_COLOR_STRENGTH, ...rest } = settings;
|
|
74
|
+
return { runtime: { ...RETOUCH_DEFAULTS, ...rest }, colorStrength };
|
|
75
75
|
|
|
76
76
|
}
|
|
77
77
|
|
|
@@ -95,12 +95,12 @@ function splitSettings( settings = {} ) {
|
|
|
95
95
|
* reserve (`MAX_STORAGE_TEXTURE_SIZE`, 2048) already caps that at exactly 4.19 MP. The check below
|
|
96
96
|
* is therefore unreachable today and exists as a floor under a raised reserve.
|
|
97
97
|
*/
|
|
98
|
-
const log = createLogger( '
|
|
98
|
+
const log = createLogger( 'neural' );
|
|
99
99
|
|
|
100
|
-
export const
|
|
100
|
+
export const RETOUCH_MAX_PIXELS = 2048 * 2048;
|
|
101
101
|
|
|
102
102
|
/** Ranges the runtime clamps to. `skinStructure: -1` means "follow localStructure". */
|
|
103
|
-
export const
|
|
103
|
+
export const RETOUCH_RANGES = Object.freeze( {
|
|
104
104
|
intensity: [ 0, 1 ],
|
|
105
105
|
localTone: [ 0, 2 ],
|
|
106
106
|
localStructure: [ 0, 2 ],
|
|
@@ -112,31 +112,32 @@ export const DLSS_NR_RANGES = Object.freeze( {
|
|
|
112
112
|
let _runtimePromise = null;
|
|
113
113
|
|
|
114
114
|
/**
|
|
115
|
-
* The runtime is a plain script that installs `globalThis.
|
|
115
|
+
* The runtime is a plain script that installs `globalThis.NeuralRuntime`, not a module, so it has to
|
|
116
116
|
* be injected rather than imported. Cached: a second tag would re-run a 1 MB bundle.
|
|
117
117
|
*/
|
|
118
118
|
function loadRuntime() {
|
|
119
119
|
|
|
120
120
|
if ( _runtimePromise ) return _runtimePromise;
|
|
121
121
|
|
|
122
|
-
const {
|
|
123
|
-
|
|
122
|
+
const { neuralRuntimeUrl, neuralAssetBaseUrl } = getAssetConfig();
|
|
123
|
+
if ( ! neuralAssetBaseUrl ) return Promise.reject( new Error( 'No neural model location: set neuralAssetBaseUrl with configureAssets()' ) );
|
|
124
|
+
globalThis.__NEURAL_ASSET_BASE__ = neuralAssetBaseUrl;
|
|
124
125
|
|
|
125
126
|
_runtimePromise = new Promise( ( resolve, reject ) => {
|
|
126
127
|
|
|
127
|
-
if ( globalThis.
|
|
128
|
+
if ( globalThis.NeuralRuntime?.getSrNrChain ) return resolve( globalThis.NeuralRuntime.getSrNrChain() );
|
|
128
129
|
|
|
129
130
|
const el = document.createElement( 'script' );
|
|
130
|
-
el.src =
|
|
131
|
+
el.src = neuralRuntimeUrl;
|
|
131
132
|
el.onload = () => {
|
|
132
133
|
|
|
133
|
-
const factory = globalThis.
|
|
134
|
-
if ( ! factory ) return reject( new Error( '
|
|
134
|
+
const factory = globalThis.NeuralRuntime?.getSrNrChain;
|
|
135
|
+
if ( ! factory ) return reject( new Error( 'Neural runtime loaded but exposed no retouch entry' ) );
|
|
135
136
|
resolve( factory() );
|
|
136
137
|
|
|
137
138
|
};
|
|
138
139
|
|
|
139
|
-
el.onerror = () => reject( new Error( `Failed to load the
|
|
140
|
+
el.onerror = () => reject( new Error( `Failed to load the neural runtime from ${neuralRuntimeUrl}` ) );
|
|
140
141
|
document.head.appendChild( el );
|
|
141
142
|
|
|
142
143
|
} );
|
|
@@ -183,7 +184,7 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
|
|
|
183
184
|
* Holds a GPUDevice of its own (the runtime builds it) plus ~140 MB of weights, so it must be
|
|
184
185
|
* disposed rather than dropped.
|
|
185
186
|
*/
|
|
186
|
-
export class
|
|
187
|
+
export class NeuralRetouch {
|
|
187
188
|
|
|
188
189
|
constructor( chain, canvas ) {
|
|
189
190
|
|
|
@@ -232,15 +233,15 @@ export class DLSSNeural {
|
|
|
232
233
|
|
|
233
234
|
if ( ! Number.isInteger( width ) || ! Number.isInteger( height ) || width < 1 || height < 1 ) {
|
|
234
235
|
|
|
235
|
-
throw new Error( '
|
|
236
|
+
throw new Error( 'NeuralRetouch.create: width and height must be positive integers' );
|
|
236
237
|
|
|
237
238
|
}
|
|
238
239
|
|
|
239
|
-
if ( width * height >
|
|
240
|
+
if ( width * height > RETOUCH_MAX_PIXELS ) {
|
|
240
241
|
|
|
241
242
|
throw new Error(
|
|
242
|
-
`
|
|
243
|
-
`${(
|
|
243
|
+
`NeuralRetouch.create: ${width}x${height} is ${( width * height / 1e6 ).toFixed( 1 )} MP, above the ` +
|
|
244
|
+
`${( RETOUCH_MAX_PIXELS / 1e6 ).toFixed( 1 )} MP this pass survives. Past it the run takes minutes ` +
|
|
244
245
|
'and then loses every GPU device in the page.'
|
|
245
246
|
);
|
|
246
247
|
|
|
@@ -256,7 +257,7 @@ export class DLSSNeural {
|
|
|
256
257
|
surface, width, height, onProgress, splitSettings( settings ).runtime,
|
|
257
258
|
);
|
|
258
259
|
|
|
259
|
-
const instance = new
|
|
260
|
+
const instance = new NeuralRetouch( chain, surface );
|
|
260
261
|
instance._allocate();
|
|
261
262
|
return instance;
|
|
262
263
|
|
|
@@ -267,7 +268,7 @@ export class DLSSNeural {
|
|
|
267
268
|
const { validWidth, validHeight } = this.chain.pipeline.geometry;
|
|
268
269
|
|
|
269
270
|
this.source = this.device.createBuffer( {
|
|
270
|
-
label: 'rayzee:
|
|
271
|
+
label: 'rayzee:retouch-source',
|
|
271
272
|
size: validWidth * validHeight * 8,
|
|
272
273
|
usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
|
|
273
274
|
} );
|
|
@@ -276,34 +277,34 @@ export class DLSSNeural {
|
|
|
276
277
|
|
|
277
278
|
const bytes = validWidth * validHeight * 4;
|
|
278
279
|
this._readbackStorage = this.device.createBuffer( {
|
|
279
|
-
label: 'rayzee:
|
|
280
|
+
label: 'rayzee:retouch-readback',
|
|
280
281
|
size: bytes,
|
|
281
282
|
usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC,
|
|
282
283
|
} );
|
|
283
284
|
this._readbackMap = this.device.createBuffer( {
|
|
284
|
-
label: 'rayzee:
|
|
285
|
+
label: 'rayzee:retouch-readback-map',
|
|
285
286
|
size: bytes,
|
|
286
287
|
usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ,
|
|
287
288
|
} );
|
|
288
289
|
this._readbackPipeline = this.device.createComputePipeline( {
|
|
289
|
-
label: 'rayzee:
|
|
290
|
+
label: 'rayzee:retouch-readback',
|
|
290
291
|
layout: 'auto',
|
|
291
292
|
compute: { module: this.device.createShaderModule( { code: READBACK_WGSL } ), entryPoint: 'main' },
|
|
292
293
|
} );
|
|
293
294
|
|
|
294
295
|
const hdrBytes = validWidth * validHeight * 8;
|
|
295
296
|
this._hdrStorage = this.device.createBuffer( {
|
|
296
|
-
label: 'rayzee:
|
|
297
|
+
label: 'rayzee:retouch-hdr',
|
|
297
298
|
size: hdrBytes,
|
|
298
299
|
usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC,
|
|
299
300
|
} );
|
|
300
301
|
this._hdrMap = this.device.createBuffer( {
|
|
301
|
-
label: 'rayzee:
|
|
302
|
+
label: 'rayzee:retouch-hdr-map',
|
|
302
303
|
size: hdrBytes,
|
|
303
304
|
usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ,
|
|
304
305
|
} );
|
|
305
306
|
this._hdrPipeline = this.device.createComputePipeline( {
|
|
306
|
-
label: 'rayzee:
|
|
307
|
+
label: 'rayzee:retouch-hdr',
|
|
307
308
|
layout: 'auto',
|
|
308
309
|
compute: { module: this.device.createShaderModule( { code: HDR_READBACK_WGSL } ), entryPoint: 'main' },
|
|
309
310
|
} );
|
|
@@ -317,8 +318,8 @@ export class DLSSNeural {
|
|
|
317
318
|
if ( ! texture ) {
|
|
318
319
|
|
|
319
320
|
throw new Error(
|
|
320
|
-
'
|
|
321
|
-
+ 'or
|
|
321
|
+
'NeuralRetouch: the runtime has no scene-referred output. Either no frame has been produced yet, '
|
|
322
|
+
+ 'or neural-runtime.js was re-vendored without its `rayzee-patch` edits.'
|
|
322
323
|
);
|
|
323
324
|
|
|
324
325
|
}
|
|
@@ -331,7 +332,7 @@ export class DLSSNeural {
|
|
|
331
332
|
],
|
|
332
333
|
} );
|
|
333
334
|
|
|
334
|
-
const encoder = this.device.createCommandEncoder( { label: 'rayzee:
|
|
335
|
+
const encoder = this.device.createCommandEncoder( { label: 'rayzee:retouch-hdr' } );
|
|
335
336
|
const pass = encoder.beginComputePass();
|
|
336
337
|
pass.setPipeline( this._hdrPipeline );
|
|
337
338
|
pass.setBindGroup( 0, group );
|
|
@@ -346,7 +347,7 @@ export class DLSSNeural {
|
|
|
346
347
|
* The pass's result as scene-referred light, packed the way the upscaler wants it.
|
|
347
348
|
*
|
|
348
349
|
* Reads `hdrOutput`, the `rgba16float` texture the vendored runtime gained in
|
|
349
|
-
* `
|
|
350
|
+
* its `rayzee-patch` edits — the value one statement before its own 8-bit store, so nothing
|
|
350
351
|
* has been clamped, quantised or pushed through the model's hardcoded ACES curve yet.
|
|
351
352
|
*
|
|
352
353
|
* This is what lets the pass run FIRST, at render size, where it is ~10x cheaper than on an
|
|
@@ -381,7 +382,7 @@ export class DLSSNeural {
|
|
|
381
382
|
const { validWidth: w, validHeight: h } = this.chain.pipeline.geometry;
|
|
382
383
|
await this._packHDR( w, h );
|
|
383
384
|
|
|
384
|
-
this._toneMapper ??= new PackedToneMapper( this.device, 'rayzee:
|
|
385
|
+
this._toneMapper ??= new PackedToneMapper( this.device, 'rayzee:retouch-tonemap' );
|
|
385
386
|
this._toneMapper.ensureSize( w, h );
|
|
386
387
|
|
|
387
388
|
const rgba8 = await this._toneMapper.toRGBA8( this._hdrStorage, {
|
|
@@ -403,7 +404,7 @@ export class DLSSNeural {
|
|
|
403
404
|
|
|
404
405
|
const { validWidth: w, validHeight: h } = this.chain.pipeline.geometry;
|
|
405
406
|
const texture = this.chain.pipeline.slots[ this.slot ]?.output;
|
|
406
|
-
if ( ! texture ) throw new Error( '
|
|
407
|
+
if ( ! texture ) throw new Error( 'NeuralRetouch.readOutput: the pass has not produced a frame yet' );
|
|
407
408
|
|
|
408
409
|
const group = this.device.createBindGroup( {
|
|
409
410
|
layout: this._readbackPipeline.getBindGroupLayout( 0 ),
|
|
@@ -413,7 +414,7 @@ export class DLSSNeural {
|
|
|
413
414
|
],
|
|
414
415
|
} );
|
|
415
416
|
|
|
416
|
-
const encoder = this.device.createCommandEncoder( { label: 'rayzee:
|
|
417
|
+
const encoder = this.device.createCommandEncoder( { label: 'rayzee:retouch-readback' } );
|
|
417
418
|
const pass = encoder.beginComputePass();
|
|
418
419
|
pass.setPipeline( this._readbackPipeline );
|
|
419
420
|
pass.setBindGroup( 0, group );
|
|
@@ -477,7 +478,7 @@ const MAX_EXPOSURE = 20;
|
|
|
477
478
|
* @param {{half: Uint16Array, width: number, height: number}} opts.source
|
|
478
479
|
* @param {number} [opts.exposure=1] clamped to `MAX_EXPOSURE`
|
|
479
480
|
* @param {{exposure: number, toneMapping: number, saturation: number}} [opts.tone] engine display state
|
|
480
|
-
* @param {
|
|
481
|
+
* @param {NeuralRetouch} [opts.instance] reuse a network already built for this size
|
|
481
482
|
* @returns {Promise<object>} `{ instance, width, height, ms }` plus `rgba8` or `half`
|
|
482
483
|
*/
|
|
483
484
|
export async function enhanceFrame( {
|
|
@@ -494,7 +495,7 @@ export async function enhanceFrame( {
|
|
|
494
495
|
|
|
495
496
|
if ( ! nr ) {
|
|
496
497
|
|
|
497
|
-
nr = await
|
|
498
|
+
nr = await NeuralRetouch.create( {
|
|
498
499
|
width: source.width, height: source.height, settings, onProgress,
|
|
499
500
|
} );
|
|
500
501
|
|
|
@@ -505,7 +506,7 @@ export async function enhanceFrame( {
|
|
|
505
506
|
|
|
506
507
|
if ( ! source.half || source.half.length !== dst.length ) {
|
|
507
508
|
|
|
508
|
-
throw new Error( `
|
|
509
|
+
throw new Error( `NeuralRetouch: expected ${dst.length} packed halfs, got ${source.half?.length}` );
|
|
509
510
|
|
|
510
511
|
}
|
|
511
512
|
|
|
@@ -519,7 +520,7 @@ export async function enhanceFrame( {
|
|
|
519
520
|
nr.chain.pipeline.colorStrength = colorStrength;
|
|
520
521
|
|
|
521
522
|
const started = performance.now();
|
|
522
|
-
const encoder = nr.device.createCommandEncoder( { label: 'rayzee:
|
|
523
|
+
const encoder = nr.device.createCommandEncoder( { label: 'rayzee:retouch' } );
|
|
523
524
|
await nr.chain.encode( encoder, nr.slot, { buffer: nr.source }, merged, () => {}, { reset: true } );
|
|
524
525
|
nr.device.queue.submit( [ encoder.finish() ] );
|
|
525
526
|
await nr.device.queue.onSubmittedWorkDone();
|