phoria2 1.2.0 → 1.4.0

package/src/webgl/resources.js

@@ -10,6 +10,9 @@ import { isVideo } from '../material/Texture.js'
 
 // uploads bind textures here, out of the way of the units programs sample from
 const UPLOAD_UNIT = 15
+// past this many changed regions since the last upload, those in each cell of a GRID x GRID split of
+// the image go as one rectangle round them - nearby changes together, far ones still apart
+const MOST_REGIONS = 16, GRID = 4
 
 /**
  * @typedef {object} MeshBuffers
@@ -109,6 +112,11 @@ export function uvMatrix(t) {
 
 // a quad as a triangle strip of [along, across] corners, shared by every Lines draw
 const LINE_CORNERS = new Float32Array([0, -1, 0, 1, 1, -1, 1, 1])
+// a point drawn as a square: position, colour, size, opacity, angle and index, as [location, floats,
+// offset in floats]
+const SPRITE_LAYOUT = [[ATTRIBUTES.position, 3, 0], [ATTRIBUTES.color, 3, 3], [ATTRIBUTES.pointSize, 1, 6],
+  [ATTRIBUTES.pointOpacity, 1, 7], [ATTRIBUTES.pointAngle, 1, 8], [ATTRIBUTES.pointIndex, 1, 9]]
+const SPRITE_STRIDE = 10
 
 export class Resources {
   /** @type {WebGL2RenderingContext} */ gl
@@ -116,7 +124,8 @@ export class Resources {
   /** @type {WeakMap<object, MeshBuffers>} */ #triangles = new WeakMap()
   /** @type {WeakMap<object, MeshBuffers>} */ #points = new WeakMap()
   /** @type {WeakMap<object, MeshBuffers>} */ #lines = new WeakMap()
-  /** @type {WeakMap<Texture, { texture: WebGLTexture, version: number }>} */ #textures = new WeakMap()
+  /** Points drawn as squares: an instance a point. @type {WeakMap<object, MeshBuffers & { capacity: number, sorted: boolean, data: Float32Array }>} */ #sprites = new WeakMap()
+  /** The GPU copy of each texture: the version it holds, and the storage it was made with. @type {WeakMap<Texture, { texture: WebGLTexture, version: number, width: number, height: number, internal: number }>} */ #textures = new WeakMap()
   /** @type {WebGLTexture | null} */ #white = null
   /** @type {WebGLBuffer | null} */ #corners = null
   /** Counts for renderer.info. */
@@ -284,8 +293,8 @@ export class Resources {
   }
 
   /**
-   * Vertex array for Lines: one instance per segment with its two endpoints (and colours),
-   * expanded to a screen-space quad in the vertex shader.
+   * Vertex array for Lines: one instance per segment with its two endpoints (and colours) and where
+   * it is along its chain, expanded to a screen-space quad - and arrowheads - in the vertex shader.
    * @param {Geometry} geometry
    * @returns {MeshBuffers}
    */
@@ -301,6 +310,9 @@ export class Resources {
     const segments = geometry.elementCount / 2
     const ends = new Float32Array(segments * 6)
     const colors = new Float32Array(segments * 6).fill(1)
+    // chains: runs of segments each starting where the one before ended - a polyline. Dashes run
+    // on along a chain, and arrowheads go at its ends
+    const info = new Float32Array(segments * 3)
     for (let s = 0; s < segments; s++) {
       const i0 = idx ? idx[s * 2] : s * 2, i1 = idx ? idx[s * 2 + 1] : s * 2 + 1
       for (let k = 0; k < 3; k++) {
@@ -311,6 +323,12 @@ export class Resources {
           colors[s * 6 + 3 + k] = c[i1 * 3 + k]
         }
       }
+      const o = s * 6, joined = s > 0 && ends[o - 3] === ends[o] && ends[o - 2] === ends[o + 1] && ends[o - 1] === ends[o + 2]
+      if (joined) {
+        info[s * 3] = info[s * 3 - 3] + Math.hypot(ends[o - 3] - ends[o - 6], ends[o - 2] - ends[o - 5], ends[o - 1] - ends[o - 4])
+        info[s * 3 - 1] = 0
+      } else info[s * 3 + 1] = 1
+      info[s * 3 + 2] = 1
     }
 
     const vao = /** @type {WebGLVertexArrayObject} */ (gl.createVertexArray())
@@ -330,6 +348,13 @@ export class Resources {
     }
     interleavedPair(ends, ATTRIBUTES.segmentStart, ATTRIBUTES.segmentEnd)
     interleavedPair(colors, ATTRIBUTES.segmentColorStart, ATTRIBUTES.segmentColorEnd)
+    const chains = /** @type {WebGLBuffer} */ (gl.createBuffer())
+    gl.bindBuffer(gl.ARRAY_BUFFER, chains)
+    gl.bufferData(gl.ARRAY_BUFFER, info, gl.STATIC_DRAW)
+    gl.enableVertexAttribArray(ATTRIBUTES.segmentInfo)
+    gl.vertexAttribPointer(ATTRIBUTES.segmentInfo, 3, gl.FLOAT, false, 0, 0)
+    gl.vertexAttribDivisor(ATTRIBUTES.segmentInfo, 1)
+    buffers.push(chains)
     gl.bindBuffer(gl.ARRAY_BUFFER, this.#cornerBuffer())
     gl.enableVertexAttribArray(ATTRIBUTES.corner)
     gl.vertexAttribPointer(ATTRIBUTES.corner, 2, gl.FLOAT, false, 0, 0)
@@ -354,7 +379,7 @@ export class Resources {
     // a video with no frame yet: what it showed before, or white - and try again next frame
     if (isVideo(texture.image) && /** @type {HTMLVideoElement} */ (texture.image).readyState < 2) return entry?.texture ?? this.#whiteTexture()
     if (!entry) {
-      entry = { texture: /** @type {WebGLTexture} */ (gl.createTexture()), version: -1 }
+      entry = { texture: /** @type {WebGLTexture} */ (gl.createTexture()), version: -1, width: 0, height: 0, internal: 0 }
       this.#textures.set(texture, entry)
       this.counts.textures++
       texture.onDispose(() => this.#freeTexture(texture))
@@ -364,16 +389,37 @@ export class Resources {
     gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, texture.flipY)
     gl.pixelStorei(gl.UNPACK_PREMULTIPLY_ALPHA_WEBGL, false)
     const image = texture.image
-    if ('data' in image) {
-      // raw pixels: bytes as usual, floats (HDR) as half floats, which filter everywhere
-      const d = /** @type {DataImage} */ (image)
-      const float = d.data instanceof Float32Array
-      const internal = float ? gl.RGBA16F : texture.colorSpace === 'srgb' ? gl.SRGB8_ALPHA8 : gl.RGBA8
-      gl.texImage2D(gl.TEXTURE_2D, 0, internal, d.width, d.height, 0, gl.RGBA, float ? gl.FLOAT : gl.UNSIGNED_BYTE, d.data)
+    const data = 'data' in image ? /** @type {DataImage} */ (image) : null
+    // raw pixels: bytes as usual, floats (HDR) as half floats, which filter everywhere; sRGB textures
+    // are stored as sRGB so sampling returns linear values, filtered correctly
+    const float = !!data && data.data instanceof Float32Array
+    const internal = float ? gl.RGBA16F : texture.colorSpace === 'srgb' ? gl.SRGB8_ALPHA8 : gl.RGBA8
+    const type = float ? gl.FLOAT : gl.UNSIGNED_BYTE
+    const [width, height] = sizeOf(image)
+    if (entry.version < 0 || entry.width !== width || entry.height !== height || entry.internal !== internal) {
+      // new storage: the first upload, or the image changed size or kind
+      if (data) gl.texImage2D(gl.TEXTURE_2D, 0, internal, width, height, 0, gl.RGBA, type, data.data)
+      else gl.texImage2D(gl.TEXTURE_2D, 0, internal, gl.RGBA, type, /** @type {TexImageSource} */ (image))
+      Object.assign(entry, { width, height, internal })
     } else {
-      // sRGB textures are stored as sRGB so sampling returns linear values, filtered correctly
-      const internal = texture.colorSpace === 'srgb' ? gl.SRGB8_ALPHA8 : gl.RGBA8
-      gl.texImage2D(gl.TEXTURE_2D, 0, internal, gl.RGBA, gl.UNSIGNED_BYTE, /** @type {TexImageSource} */ (image))
+      // the same storage, refilled: just the parts that changed, when the updates since said - a
+      // canvas drawn on a little at a time - or the whole image (a video's next frame). A flipped
+      // image's rows run the other way, so it goes whole
+      let regions = texture.flipY ? null : texture.changedSince(entry.version)
+      if (regions && regions.length > MOST_REGIONS) regions = byCell(regions, width, height)
+      for (const r of regions ?? [{ x: 0, y: 0, width, height }]) {
+        // inside the image: a region over its edge is cut there, one off it sends nothing
+        const x = Math.max(0, r.x), y = Math.max(0, r.y)
+        const w = Math.min(width, r.x + r.width) - x, h = Math.min(height, r.y + r.height) - y
+        if (!(w > 0 && h > 0)) continue
+        gl.pixelStorei(gl.UNPACK_SKIP_PIXELS, x)
+        gl.pixelStorei(gl.UNPACK_SKIP_ROWS, y)
+        if (data) gl.pixelStorei(gl.UNPACK_ROW_LENGTH, width)
+        gl.texSubImage2D(gl.TEXTURE_2D, 0, x, y, w, h, gl.RGBA, type, /** @type {any} */ (data ? data.data : image))
+      }
+      gl.pixelStorei(gl.UNPACK_SKIP_PIXELS, 0)
+      gl.pixelStorei(gl.UNPACK_SKIP_ROWS, 0)
+      gl.pixelStorei(gl.UNPACK_ROW_LENGTH, 0)
     }
     const wrap = { repeat: gl.REPEAT, clamp: gl.CLAMP_TO_EDGE, mirror: gl.MIRRORED_REPEAT }[texture.wrap]
     gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, wrap)
@@ -409,13 +455,13 @@ export class Resources {
       if (!d.isDrawable) return
       keys.add(d.geometry)
       if ('instanceMatrices' in d) keys.add(d)
-      for (const [, t] of d.material.textures) keys.add(t)
+      for (const [, t] of d.material.allTextures) keys.add(t)
     })
     for (const key of keys) {
       const users = this.#usedBy.get(key)
       if (!users?.delete(owner) || users.size) continue
       this.#usedBy.delete(key)
-      for (const cache of [this.#triangles, this.#points, this.#lines]) {
+      for (const cache of [this.#triangles, this.#points, this.#lines, this.#sprites]) {
         const entry = cache.get(key)
         if (!entry) continue
         this.#free(entry)
@@ -434,6 +480,64 @@ export class Resources {
     users.add(this.owner)
   }
 
+  /**
+   * Vertex array for points drawn as squares: one instance a point - its position, colour, size,
+   * opacity, angle and index - over the corner strip lines use. Given a model-view matrix, the points
+   * go back to front (furthest first) - re-sorted every call, as the camera may have moved; without
+   * one, in their own order; with undefined, in whatever order they were last sent (picking, which
+   * reads each point's index and doesn't mind).
+   * @param {Geometry} geometry @param {ArrayLike<number> | null} [modelView]
+   * @returns {MeshBuffers}
+   */
+  sprites(geometry, modelView) {
+    this.#mark(geometry)
+    const gl = this.gl
+    let entry = this.#sprites.get(geometry)
+    const sorting = !!modelView
+    if (entry && entry.version === geometry.version && !sorting && (modelView === undefined || !entry.sorted)) return entry
+    const n = geometry.drawnVertices, count = geometry.vertexCount
+    if (!entry || entry.capacity < count) {
+      if (entry) this.#free(entry)
+      else this.#watch(geometry, geometry, this.#sprites)
+      const vao = /** @type {WebGLVertexArrayObject} */ (gl.createVertexArray())
+      gl.bindVertexArray(vao)
+      const buffer = /** @type {WebGLBuffer} */ (gl.createBuffer())
+      gl.bindBuffer(gl.ARRAY_BUFFER, buffer)
+      gl.bufferData(gl.ARRAY_BUFFER, count * SPRITE_STRIDE * 4, gl.DYNAMIC_DRAW)
+      for (const [location, size, offset] of SPRITE_LAYOUT) {
+        gl.enableVertexAttribArray(location)
+        gl.vertexAttribPointer(location, size, gl.FLOAT, false, SPRITE_STRIDE * 4, offset * 4)
+        gl.vertexAttribDivisor(location, 1)
+      }
+      gl.bindBuffer(gl.ARRAY_BUFFER, this.#cornerBuffer())
+      gl.enableVertexAttribArray(ATTRIBUTES.corner)
+      gl.vertexAttribPointer(ATTRIBUTES.corner, 2, gl.FLOAT, false, 0, 0)
+      gl.bindVertexArray(null)
+      entry = { vao, buffers: [buffer], count: 0, indexType: 0, geometry, version: -1, instanceVersion: 0, capacity: count, sorted: false, data: new Float32Array(count * SPRITE_STRIDE) }
+      this.#sprites.set(geometry, entry)
+    }
+    const order = sorting ? backToFront(geometry.positions, n, /** @type {ArrayLike<number>} */ (modelView)) : null
+    const p = geometry.positions, c = geometry.colors, sizes = geometry.sizes, opacities = geometry.opacities, angles = geometry.angles
+    const data = entry.data
+    for (let j = 0; j < n; j++) {
+      const i = order ? order[j] : j, o = j * SPRITE_STRIDE
+      data[o] = p[i * 3]
+      data[o + 1] = p[i * 3 + 1]
+      data[o + 2] = p[i * 3 + 2]
+      data[o + 3] = c ? c[i * 3] : 1
+      data[o + 4] = c ? c[i * 3 + 1] : 1
+      data[o + 5] = c ? c[i * 3 + 2] : 1
+      data[o + 6] = sizes ? sizes[i] : 1
+      data[o + 7] = opacities ? opacities[i] : 1
+      data[o + 8] = angles ? angles[i] : 0
+      data[o + 9] = i
+    }
+    gl.bindBuffer(gl.ARRAY_BUFFER, entry.buffers[0])
+    gl.bufferSubData(gl.ARRAY_BUFFER, 0, data, 0, n * SPRITE_STRIDE)
+    Object.assign(entry, { count: n, version: geometry.version, sorted: sorting })
+    return entry
+  }
+
   /** @param {Texture} texture */
   #freeTexture(texture) {
     const entry = this.#textures.get(texture)
@@ -448,6 +552,7 @@ export class Resources {
     this.#triangles = new WeakMap()
     this.#points = new WeakMap()
     this.#lines = new WeakMap()
+    this.#sprites = new WeakMap()
     this.#textures = new WeakMap()
     this.#usedBy = new WeakMap()
     this.#white = null
@@ -497,3 +602,52 @@ export class Resources {
     return this.#corners
   }
 }
+
+/**
+ * An image's size in pixels - a video's frames, an <img>'s own pixels rather than its size on the page.
+ * @param {any} image @returns {[number, number]}
+ */
+function sizeOf(image) {
+  if (isVideo(image)) return [image.videoWidth, image.videoHeight]
+  if ('naturalWidth' in image) return [image.naturalWidth, image.naturalHeight]
+  return [image.width, image.height]
+}
+
+/**
+ * Many regions as a few: those whose middles fall in the same cell of a GRID x GRID split of the
+ * image, as one rectangle round them.
+ * @param {import('../material/Texture.js').TextureRegion[]} regions @param {number} width @param {number} height
+ */
+function byCell(regions, width, height) {
+  /** @type {Map<number, number[]>} x0, y0, x1, y1 round each cell's regions */
+  const cells = new Map()
+  for (const r of regions) {
+    const cx = Math.min(GRID - 1, Math.max(0, Math.floor((r.x + r.width / 2) / width * GRID)))
+    const cy = Math.min(GRID - 1, Math.max(0, Math.floor((r.y + r.height / 2) / height * GRID)))
+    const key = cy * GRID + cx, box = cells.get(key)
+    if (!box) cells.set(key, [r.x, r.y, r.x + r.width, r.y + r.height])
+    else {
+      box[0] = Math.min(box[0], r.x)
+      box[1] = Math.min(box[1], r.y)
+      box[2] = Math.max(box[2], r.x + r.width)
+      box[3] = Math.max(box[3], r.y + r.height)
+    }
+  }
+  return [...cells.values()].map(([x0, y0, x1, y1]) => ({ x: x0, y: y0, width: x1 - x0, height: y1 - y0 }))
+}
+
+/**
+ * The first n points' order furthest from the camera first: by their depth in view, given the
+ * model-view matrix.
+ * @param {Float32Array} p @param {number} n @param {ArrayLike<number>} m
+ * @returns {Uint32Array}
+ */
+function backToFront(p, n, m) {
+  const depth = new Float32Array(n), order = new Uint32Array(n)
+  for (let i = 0; i < n; i++) {
+    // view space z: more negative is further away
+    depth[i] = m[2] * p[i * 3] + m[6] * p[i * 3 + 1] + m[10] * p[i * 3 + 2] + m[14]
+    order[i] = i
+  }
+  return order.sort((a, b) => depth[a] - depth[b])
+}