supervision 0.2.8 → 0.2.9

package/dist/index.js

@@ -10650,6 +10650,260 @@ function matches(left, right) {
         left.every((value, index) => Object.is(value, right[index])));
 }
 
+const BINS = 256;
+const TILE_COUNT = 8;
+const CLIP_LIMIT = 2;
+function computeMediaClahe(rgba, width, height) {
+    const tilesX = Math.min(TILE_COUNT, width);
+    const tilesY = Math.min(TILE_COUNT, height);
+    const luts = new Uint8Array(tilesX * tilesY * BINS);
+    const histogram = new Uint32Array(BINS);
+    for (let ty = 0; ty < tilesY; ty++) {
+        const y0 = Math.floor((ty * height) / tilesY);
+        const y1 = Math.floor(((ty + 1) * height) / tilesY);
+        for (let tx = 0; tx < tilesX; tx++) {
+            const x0 = Math.floor((tx * width) / tilesX);
+            const x1 = Math.floor(((tx + 1) * width) / tilesX);
+            histogram.fill(0);
+            for (let y = y0; y < y1; y++) {
+                for (let x = x0; x < x1; x++) {
+                    const pixel = (y * width + x) * 4;
+                    const luma = Math.round(0.299 * rgba[pixel] +
+                        0.587 * rgba[pixel + 1] +
+                        0.114 * rgba[pixel + 2]);
+                    histogram[luma]++;
+                }
+            }
+            const area = (x1 - x0) * (y1 - y0);
+            const limit = Math.max(1, Math.floor((CLIP_LIMIT * area) / BINS));
+            let excess = 0;
+            for (let i = 0; i < BINS; i++) {
+                if (histogram[i] > limit) {
+                    excess += histogram[i] - limit;
+                    histogram[i] = limit;
+                }
+            }
+            const perBin = Math.floor(excess / BINS);
+            let residual = excess - perBin * BINS;
+            for (let i = 0; i < BINS; i++)
+                histogram[i] += perBin;
+            const residualStep = Math.max(Math.floor(BINS / Math.max(residual, 1)), 1);
+            for (let i = 0; i < BINS && residual > 0; i += residualStep, residual--) {
+                histogram[i]++;
+            }
+            const offset = (ty * tilesX + tx) * BINS;
+            let cumulative = 0;
+            for (let i = 0; i < BINS; i++) {
+                cumulative += histogram[i];
+                luts[offset + i] = Math.round((cumulative * (BINS - 1)) / area);
+            }
+        }
+    }
+    return { luts, tilesX, tilesY };
+}
+
+/**
+ * Pixi keys its program cache by source, so one program is shared by every
+ * shader built from it. `destroy(true)` takes that program with it, leaving
+ * both the peers still rendering through it and the next shader built from the
+ * same source bound to nothing.
+ */
+function destroyShaderKeepingProgram(shader) {
+    try {
+        shader.destroy();
+    }
+    catch {
+        // Pixi has already invalidated this shader's resource group.
+    }
+}
+/**
+ * Pixi uploads a placeholder while it builds a shader's first bind group, and
+ * WebGPU rejects a canvas that was never given a rendering context.
+ */
+function createShaderPlaceholderCanvas() {
+    if (typeof document === "undefined") {
+        return { height: 1, width: 1 };
+    }
+    const canvas = document.createElement("canvas");
+    canvas.height = 1;
+    canvas.width = 1;
+    canvas.getContext("2d");
+    return canvas;
+}
+
+const HISTOGRAM_MAX_DIMENSION = 1024;
+const vertex = `
+in vec2 aPosition;
+out vec2 vTextureCoord;
+out vec2 vImageCoord;
+uniform vec4 uInputSize;
+uniform vec4 uOutputFrame;
+uniform vec4 uOutputTexture;
+uniform mat3 uImageMatrix;
+void main() {
+  vec2 position = aPosition * uOutputFrame.zw + uOutputFrame.xy;
+  position.x = position.x * (2.0 / uOutputTexture.x) - 1.0;
+  position.y = position.y * (2.0 * uOutputTexture.z / uOutputTexture.y) - uOutputTexture.z;
+  gl_Position = vec4(position, 0.0, 1.0);
+  vTextureCoord = aPosition * (uOutputFrame.zw * uInputSize.zw);
+  vImageCoord = (uImageMatrix * vec3(vTextureCoord, 1.0)).xy;
+}
+`;
+const fragment = `
+in vec2 vTextureCoord;
+in vec2 vImageCoord;
+out vec4 finalColor;
+uniform sampler2D uTexture;
+uniform sampler2D uLut;
+uniform vec2 uTiles;
+uniform float uGamma;
+uniform float uAuto;
+float lookup(vec2 tile, float luma) {
+  float row = tile.y * uTiles.x + tile.x;
+  return texture(uLut, vec2((luma * 255.0 + 0.5) / 256.0,
+    (row + 0.5) / (uTiles.x * uTiles.y))).r;
+}
+void main() {
+  vec4 color = texture(uTexture, vTextureCoord);
+  if (color.a == 0.0) { finalColor = color; return; }
+  vec3 rgb = color.rgb / color.a;
+  if (uAuto > 0.5) {
+    float luma = dot(rgb, vec3(0.299, 0.587, 0.114));
+    vec2 tilePosition = clamp(vImageCoord, 0.0, 1.0) * uTiles - 0.5;
+    vec2 t0 = floor(tilePosition);
+    vec2 f = tilePosition - t0;
+    vec2 t1 = clamp(t0 + 1.0, vec2(0.0), uTiles - 1.0);
+    t0 = clamp(t0, vec2(0.0), uTiles - 1.0);
+    float top = mix(lookup(t0, luma), lookup(vec2(t1.x, t0.y), luma), f.x);
+    float bottom = mix(lookup(vec2(t0.x, t1.y), luma), lookup(t1, luma), f.x);
+    rgb = clamp(rgb + (mix(top, bottom, f.y) - luma), 0.0, 1.0);
+  }
+  finalColor = vec4(pow(rgb, vec3(1.0 / uGamma)) * color.a, color.a);
+}
+`;
+const gpuSource = `
+struct GlobalFilterUniforms {
+  uInputSize: vec4<f32>, uInputPixel: vec4<f32>, uInputClamp: vec4<f32>,
+  uOutputFrame: vec4<f32>, uGlobalFrame: vec4<f32>, uOutputTexture: vec4<f32>,
+};
+struct InspectionUniforms {
+  uImageMatrix: mat3x3<f32>, uTiles: vec2<f32>, uGamma: f32, uAuto: f32,
+};
+@group(0) @binding(0) var<uniform> gfu: GlobalFilterUniforms;
+@group(0) @binding(1) var uTexture: texture_2d<f32>;
+@group(0) @binding(2) var uSampler: sampler;
+@group(1) @binding(0) var<uniform> inspectionUniforms: InspectionUniforms;
+@group(1) @binding(1) var uLut: texture_2d<f32>;
+@group(1) @binding(2) var uLutSampler: sampler;
+struct VertexOutput {
+  @builtin(position) position: vec4<f32>,
+  @location(0) uv: vec2<f32>, @location(1) image: vec2<f32>,
+};
+@vertex fn mainVertex(@location(0) aPosition: vec2<f32>) -> VertexOutput {
+  var position = aPosition * gfu.uOutputFrame.zw + gfu.uOutputFrame.xy;
+  position.x = position.x * (2.0 / gfu.uOutputTexture.x) - 1.0;
+  position.y = position.y * (2.0 * gfu.uOutputTexture.z / gfu.uOutputTexture.y) - gfu.uOutputTexture.z;
+  let uv = aPosition * (gfu.uOutputFrame.zw * gfu.uInputSize.zw);
+  return VertexOutput(vec4(position, 0.0, 1.0), uv,
+    (inspectionUniforms.uImageMatrix * vec3(uv, 1.0)).xy);
+}
+fn lookup(tile: vec2<f32>, luma: f32) -> f32 {
+  let tiles = inspectionUniforms.uTiles;
+  let row = tile.y * tiles.x + tile.x;
+  return textureSampleLevel(uLut, uLutSampler,
+    vec2((luma * 255.0 + 0.5) / 256.0, (row + 0.5) / (tiles.x * tiles.y)), 0.0).r;
+}
+@fragment fn mainFragment(input: VertexOutput) -> @location(0) vec4<f32> {
+  let color = textureSample(uTexture, uSampler, input.uv);
+  if (color.a == 0.0) { return color; }
+  var rgb = color.rgb / color.a;
+  if (inspectionUniforms.uAuto > 0.5) {
+    let luma = dot(rgb, vec3(0.299, 0.587, 0.114));
+    let tiles = inspectionUniforms.uTiles;
+    let tilePosition = clamp(input.image, vec2(0.0), vec2(1.0)) * tiles - 0.5;
+    let unbounded = floor(tilePosition);
+    let f = tilePosition - unbounded;
+    let t0 = clamp(unbounded, vec2(0.0), tiles - 1.0);
+    let t1 = clamp(unbounded + 1.0, vec2(0.0), tiles - 1.0);
+    let top = mix(lookup(t0, luma), lookup(vec2(t1.x, t0.y), luma), f.x);
+    let bottom = mix(lookup(vec2(t0.x, t1.y), luma), lookup(t1, luma), f.x);
+    rgb = clamp(rgb + (mix(top, bottom, f.y) - luma), vec3(0.0), vec3(1.0));
+  }
+  return vec4(pow(rgb, vec3(1.0 / inspectionUniforms.uGamma)) * color.a, color.a);
+}
+`;
+function createPixiMediaInspectionFilter(sprite, pixi) {
+    const { BufferImageSource, Filter, Matrix } = pixi;
+    const pixels = new Uint8Array(256 * 64 * 4);
+    const lutSource = new BufferImageSource({
+        resource: pixels,
+        width: 256,
+        height: 64,
+        scaleMode: "linear",
+        alphaMode: "no-premultiply-alpha",
+    });
+    const filter = Filter.from({
+        gl: { vertex: vertex, fragment: fragment },
+        gpu: {
+            vertex: { source: gpuSource, entryPoint: "mainVertex" },
+            fragment: { source: gpuSource, entryPoint: "mainFragment" },
+        },
+        resolution: "inherit",
+        resources: {
+            inspectionUniforms: {
+                uImageMatrix: { value: new Matrix(), type: "mat3x3<f32>" },
+                uTiles: { value: new Float32Array([8, 8]), type: "vec2<f32>" },
+                uGamma: { value: 1, type: "f32" },
+                uAuto: { value: 0, type: "f32" },
+            },
+            uLut: lutSource,
+            uLutSampler: lutSource.style,
+        },
+    });
+    const uniforms = filter.resources.inspectionUniforms.uniforms;
+    let hasHistogram = false;
+    filter.apply = (manager, input, output, clear) => {
+        // The filter input is clipped by the viewport; lookup tiles stay anchored
+        // to the full image when the user zooms or pans.
+        manager.calculateSpriteMatrix(uniforms.uImageMatrix, sprite);
+        manager.applyFilter(filter, input, output, clear);
+    };
+    return {
+        filter,
+        invalidateImage() {
+            hasHistogram = false;
+        },
+        update(gamma, autoContrast, image) {
+            uniforms.uGamma = Math.max(gamma, 0.0001);
+            if (autoContrast && image && !hasHistogram) {
+                const scale = Math.min(1, HISTOGRAM_MAX_DIMENSION / Math.max(image.width, image.height));
+                const sample = document.createElement("canvas");
+                sample.width = Math.max(1, Math.round(image.width * scale));
+                sample.height = Math.max(1, Math.round(image.height * scale));
+                const context = sample.getContext("2d", { willReadFrequently: true });
+                context.drawImage(image, 0, 0, sample.width, sample.height);
+                const { luts, tilesX, tilesY } = computeMediaClahe(context.getImageData(0, 0, sample.width, sample.height).data, sample.width, sample.height);
+                for (let i = 0; i < luts.length; i++) {
+                    pixels[i * 4] = luts[i];
+                    pixels[i * 4 + 1] = luts[i];
+                    pixels[i * 4 + 2] = luts[i];
+                    pixels[i * 4 + 3] = 255;
+                }
+                uniforms.uTiles.set([tilesX, tilesY]);
+                lutSource.resource = pixels.subarray(0, luts.length * 4);
+                lutSource.resize(256, tilesX * tilesY);
+                lutSource.update();
+                hasHistogram = true;
+            }
+            uniforms.uAuto = autoContrast && hasHistogram ? 1 : 0;
+        },
+        destroy() {
+            destroyShaderKeepingProgram(filter);
+            lutSource.destroy();
+        },
+    };
+}
+
 /**
  * Which frame a media time draws and how much of it is cooked, as two separate
  * facts. Reporting readiness is what the snapshot and the token are for; a
@@ -10789,35 +11043,6 @@ void main(void) {
 }
 `;
 
-/**
- * Pixi keys its program cache by source, so one program is shared by every
- * shader built from it. `destroy(true)` takes that program with it, leaving
- * both the peers still rendering through it and the next shader built from the
- * same source bound to nothing.
- */
-function destroyShaderKeepingProgram(shader) {
-    try {
-        shader.destroy();
-    }
-    catch {
-        // Pixi has already invalidated this shader's resource group.
-    }
-}
-/**
- * Pixi uploads a placeholder while it builds a shader's first bind group, and
- * WebGPU rejects a canvas that was never given a rendering context.
- */
-function createShaderPlaceholderCanvas() {
-    if (typeof document === "undefined") {
-        return { height: 1, width: 1 };
-    }
-    const canvas = document.createElement("canvas");
-    canvas.height = 1;
-    canvas.width = 1;
-    canvas.getContext("2d");
-    return canvas;
-}
-
 /**
  * The halo a detection actually paints, or `undefined`. A style may answer an
  * instruction that draws nothing, and both the coverage the halo prepares and
@@ -18572,6 +18797,9 @@ async function createPixiMediaScene(options) {
     let isScrubbing = false;
     let displayBrightness = 1;
     let displayContrast = 1;
+    let displayGamma = 1;
+    let displayAutoContrast = false;
+    let inspectionFilter;
     let viewportScale = 1;
     let hasPresentedSample = false;
     let mediaHeight = 0;
@@ -19429,6 +19657,7 @@ async function createPixiMediaScene(options) {
                 if (!collectFrameTimings) {
                     sample.draw(stagingContext, 0, 0, mediaWidth, mediaHeight);
                     presentedSampleTimestamp = sample.timestamp;
+                    refreshInspectionImage();
                     stagingTextureSource?.update();
                     stagingTexture?.update();
                     const { boxState, regionState } = drawFramePresentLayers(annotationDrawLayers, sample.timestamp);
@@ -19444,6 +19673,7 @@ async function createPixiMediaScene(options) {
                 const mediaUploadMs = measure(() => {
                     sample.draw(stagingContext, 0, 0, mediaWidth, mediaHeight);
                     presentedSampleTimestamp = sample.timestamp;
+                    refreshInspectionImage();
                     stagingTextureSource?.update();
                     stagingTexture?.update();
                 });
@@ -19507,15 +19737,27 @@ async function createPixiMediaScene(options) {
         },
         setDisplayAdjustments(adjustments) {
             void import('pixi.js').then(({ ColorMatrixFilter }) => {
+                if (isDestroyed)
+                    return;
                 const brightness = adjustments.brightness ?? 1;
                 const contrast = adjustments.contrast ?? 1;
                 const filter = new ColorMatrixFilter();
                 // Preserve the renderer's pixel grid through the adjustment pass.
                 filter.resolution = "inherit";
                 filter.brightness(brightness, false);
-                filter.contrast(contrast - 1, true);
-                if (mediaSprite)
-                    mediaSprite.filters = [filter];
+                displayGamma = adjustments.gamma ?? 1;
+                displayAutoContrast = stillFrame && adjustments.autoContrast === true;
+                filter.contrast(displayAutoContrast ? 0 : contrast - 1, true);
+                if (mediaSprite) {
+                    if (displayGamma !== 1 || displayAutoContrast) {
+                        inspectionFilter ??= createPixiMediaInspectionFilter(mediaSprite, pixi);
+                        inspectionFilter.update(displayGamma, displayAutoContrast, hasPresentedSample ? stagingCanvas : null);
+                        mediaSprite.filters = [inspectionFilter.filter, filter];
+                    }
+                    else {
+                        mediaSprite.filters = [filter];
+                    }
+                }
                 displayBrightness = brightness;
                 displayContrast = contrast;
                 renderOnChange();
@@ -19822,6 +20064,7 @@ async function createPixiMediaScene(options) {
             // a decoder buffer in a producer that outlives the scene.
             frameChannel?.onPresentedFrame((presented) => presented.frame.close());
             destroyBatchTextureBindings();
+            inspectionFilter?.destroy();
             mediaCompositor?.destroy();
             interactionLayer?.destroy();
             interactionPresentationLayer?.destroy();
@@ -20532,6 +20775,12 @@ async function createPixiMediaScene(options) {
             app.ticker.remove(drawAnnotationOverlayNow);
         }
     }
+    function refreshInspectionImage() {
+        if (!stillFrame || !inspectionFilter)
+            return;
+        inspectionFilter.invalidateImage();
+        inspectionFilter.update(displayGamma, displayAutoContrast, stagingCanvas);
+    }
     function hasLoadingDetections() {
         const loading = currentVisibility?.loadingDetectionIds;
         if (!loading)
@@ -20563,6 +20812,8 @@ async function createPixiMediaScene(options) {
             app.renderer.resolution,
             displayBrightness,
             displayContrast,
+            displayGamma,
+            displayAutoContrast,
             ...RENDERED_PRESENTATION_FIELDS.map((field) => appliedPresentation?.[field]),
             // Styles above compare by identity; mask opacity is applied after
             // preparation, so a host can move it inside a style object it keeps.