reze-engine 0.21.2 → 0.22.1

package/src/camera.ts

@@ -1,4 +1,5 @@
-import { Mat4, Vec3 } from "./math"
+import { Mat4, Quat, Vec3 } from "./math"
+import type { CameraPose } from "./camera-animation"
 
 /** Far cap / zoom limit; large enough for wide shots without clipping distant ground */
 const FAR_CAP = 8000
@@ -41,6 +42,16 @@ export class Camera {
   private _viewMat = new Mat4(new Float32Array(16))
   private _projMat = new Mat4(new Float32Array(16))
 
+  // ── VMD camera drive ──
+  // When vmdDriven, getViewMatrix builds the shot from a sampled MMD camera pose (target /
+  // rotation-euler / distance / fov) instead of the orbit params. Toggled by the engine.
+  vmdDriven: boolean = false
+  private _vmdTarget = new Vec3(0, 0, 0)
+  private _vmdRotation = new Vec3(0, 0, 0) // euler radians
+  private _vmdDistance = -45
+  private _savedFov = Math.PI / 4
+  private _quatScratch = new Float32Array(16)
+
   constructor(alpha: number, beta: number, radius: number, target: Vec3, fov: number = Math.PI / 4) {
     this.alpha = alpha
     this.beta = beta
@@ -68,7 +79,41 @@ export class Camera {
     return new Vec3(x, y, z)
   }
 
+  /** Enter/leave VMD-camera drive. Backs up the orbit fov on enter, restores it on leave
+   *  (the VMD camera animates fov, so orbit's value would otherwise be clobbered). */
+  setVmdDriven(enabled: boolean): void {
+    if (enabled === this.vmdDriven) return
+    if (enabled) this._savedFov = this.fov
+    else this.fov = this._savedFov
+    this.vmdDriven = enabled
+  }
+
+  /** Feed the next sampled MMD camera pose (engine calls this each frame while driving). */
+  setVmdPose(pose: CameraPose): void {
+    this._vmdTarget.set(pose.target)
+    this._vmdRotation.set(pose.rotation)
+    this._vmdDistance = pose.distance
+    this.fov = pose.fov // drives the projection
+  }
+
   getViewMatrix(): Mat4 {
+    if (this.vmdDriven) {
+      // MMD camera: look at `target` from `distance` back, oriented by the euler rotation.
+      // forward = q·(0,0,1) (LH), up = q·(0,1,0) — read as columns 2 and 1 of the rot matrix.
+      // eye = target + forward·distance (distance is negative in VMD, so eye sits behind).
+      // Euler is NEGATED on all three axes: babylon-mmd builds the shot with
+      // RotationYawPitchRoll(-ry, -rx, -rz), and our fromEuler(rx,ry,rz) equals
+      // RotationYawPitchRoll(ry,rx,rz) — so the MMD-authentic rotation is fromEuler(-r).
+      const r = this._vmdRotation
+      const q = Quat.fromEuler(-r.x, -r.y, -r.z)
+      Mat4.fromQuatInto(q.x, q.y, q.z, q.w, this._quatScratch, 0)
+      const s = this._quatScratch
+      const t = this._vmdTarget
+      const d = this._vmdDistance
+      const ex = t.x + s[8] * d, ey = t.y + s[9] * d, ez = t.z + s[10] * d
+      Mat4.lookAtInto(this._viewMat.values, ex, ey, ez, t.x, t.y, t.z, s[4], s[5], s[6])
+      return this._viewMat
+    }
     const eye = this.getPosition()
     const t = this.target
     Mat4.lookAtInto(this._viewMat.values, eye.x, eye.y, eye.z, t.x, t.y, t.z, 0, 1, 0)
@@ -200,7 +245,7 @@ export class Camera {
   }
 
   private onMouseMove(e: MouseEvent) {
-    if (this.inputLocked) return
+    if (this.inputLocked || this.vmdDriven) return // VMD owns the camera; orbit/pan is inert
     if (!this.isDragging) return
 
     const deltaX = e.clientX - this.lastMousePos.x
@@ -228,6 +273,7 @@ export class Camera {
 
   private onWheel(e: WheelEvent) {
     e.preventDefault()
+    if (this.vmdDriven) return // VMD owns the camera; zoom is inert
 
     // Update camera radius (zoom)
     this.radius += e.deltaY * this.wheelPrecision
@@ -274,6 +320,7 @@ export class Camera {
   private onTouchMove(e: TouchEvent) {
     if (this.inputLocked) return
     e.preventDefault()
+    if (this.vmdDriven) return // VMD owns the camera; pinch/pan/rotate is inert
 
     if (this.isPinching && e.touches.length === 2) {
       // Two-finger gesture: can be pinch zoom or pan

package/src/engine.ts

@@ -3,6 +3,8 @@ import { Mat4, Quat, Vec3 } from "./math"
 import { Model, type Material } from "./model"
 import { MORPH_COMPUTE_WGSL } from "./shaders/passes/morph"
 import { decodeTga } from "./tga-loader"
+import { VMDLoader } from "./vmd-loader"
+import { CameraAnimation } from "./camera-animation"
 import { PmxLoader } from "./pmx-loader"
 import { RezePhysics } from "./physics"
 import {
@@ -82,7 +84,7 @@ const PRESET_NAME_HINTS: Array<[MaterialPreset, string[]]> = [
   ],
   ["hair", ["前髪", "後髪", "髪", "髮", "头发", "頭髪", "もみあげ", "アホ毛", "ヘア", "hair", "ahoge", "bang"]],
   ["body", ["肌", "皮肤", "skin"]],
-  ["metal", ["金属", "メタル", "metal"]],
+  ["metal", ["金属", "メタル", "metal", "earring", "耳环", "耳環"]],
   [
     "cloth_smooth",
     [
@@ -101,15 +103,20 @@ const PRESET_NAME_HINTS: Array<[MaterialPreset, string[]]> = [
       "飾",
       "饰",
       "尾",
+      "套", // 外套 (coat), 手套 (gloves)
+      "腿", // 腿环 (leg ring/garter) and other leg-wear accessories
       "skirt",
       "dress",
       "ribbon",
       "sleeve",
       "shoes",
+      "shirt",
+      "short", // shorts
       "boot",
       "hat",
       "cloth",
       "accessor",
+      "trigger",
     ],
   ],
 ]
@@ -197,6 +204,16 @@ export type WorldOptions = {
   strength?: number
 }
 
+/** A model's scene placement — root offset baked into skinning + visibility. Serializable
+ *  into a scene descriptor via getModelTransform. */
+export type ModelTransform = {
+  position: Vec3
+  rotation: Quat
+  /** Uniform scale (default 1). */
+  scale: number
+  visible: boolean
+}
+
 export type SunOptions = {
   /** Linear color of the sun lamp (Blender: Light > Color). */
   color?: Vec3
@@ -606,6 +623,10 @@ export class Engine {
   // GPU vertex-morph path. Set false BEFORE loadModel to fall back to the CPU path (A/B).
   private useGpuMorphs = true
 
+  // VMD camera track (a dedicated camera VMD). When loaded + enabled it drives the shot,
+  // sampled off the animated model's clock so it stays synced to the dance.
+  private cameraAnimation: CameraAnimation | null = null
+
   // Camera target binding (Babylon/Three style: camera follows model)
   private cameraTargetModel: Model | null = null
   private cameraTargetBoneName = "全ての親"
@@ -2161,6 +2182,59 @@ export class Engine {
     this.cameraTargetOffset.z = offset?.z ?? 0
   }
 
+  // ── VMD camera track ──
+  // A dedicated camera VMD (target / rotation / distance / fov animated). Motion VMDs loaded
+  // via model.loadVmd never touch the camera — the camera shot is opt-in through here.
+
+  /** Load a camera VMD (dedicated camera file, or any VMD's camera block) and drive the shot
+   *  from it. Default-on once a non-empty track loads; toggle with setCameraVmdEnabled. */
+  async loadCameraVmd(url: string): Promise<void> {
+    const frames = await VMDLoader.loadCamera(url)
+    this.cameraAnimation = frames.length ? new CameraAnimation(frames) : null
+    this.camera.setVmdDriven(this.cameraAnimation !== null)
+  }
+
+  /** Load a camera VMD from an already-fetched buffer (e.g. a File the user dropped). */
+  loadCameraVmdFromBuffer(buffer: ArrayBuffer): void {
+    const frames = VMDLoader.loadCameraFromBuffer(buffer)
+    this.cameraAnimation = frames.length ? new CameraAnimation(frames) : null
+    this.camera.setVmdDriven(this.cameraAnimation !== null)
+  }
+
+  /** Turn the loaded camera VMD on/off (falls back to orbit when off). No-op if none loaded. */
+  setCameraVmdEnabled(enabled: boolean): void {
+    this.camera.setVmdDriven(enabled && this.cameraAnimation !== null)
+  }
+
+  /** True while the loaded camera VMD is actively driving the shot. */
+  isCameraVmdEnabled(): boolean {
+    return this.camera.vmdDriven
+  }
+
+  /** True if a (non-empty) camera VMD is loaded, regardless of enabled state. */
+  hasCameraVmd(): boolean {
+    return this.cameraAnimation !== null
+  }
+
+  /** Drop the loaded camera VMD and return to orbit control. */
+  clearCameraVmd(): void {
+    this.cameraAnimation = null
+    this.camera.setVmdDriven(false)
+  }
+
+  // Clock the camera VMD runs on: the first model with an active clip (playing or scrubbed),
+  // so a static stage in the scene never freezes the shot at frame 0. Falls back to the first
+  // model, then to 0 (empty scene).
+  private cameraClockTime(): number {
+    let first: number | null = null
+    for (const inst of this.modelInstances.values()) {
+      const p = inst.model.getAnimationProgress()
+      if (first === null) first = p.current
+      if (p.playing || p.paused) return p.current
+    }
+    return first ?? 0
+  }
+
   getCameraDistance(): number {
     return this.camera.radius
   }
@@ -2421,6 +2495,35 @@ export class Engine {
     return this.modelInstances.get(name)?.model ?? null
   }
 
+  /**
+   * Place a model in the scene — position, rotation, uniform scale, visibility. The
+   * transform is a root offset baked into skinning (moves the whole rig), so it composes
+   * with animation. Use it to sit a `stage.pmx` with a character, or to fit/hide either.
+   * Scale is **uniform** (normals renormalize in-shader). Don't scale a physics-driven
+   * character — its colliders won't scale; scale stages (which are typically physics-free).
+   */
+  setModelTransform(name: string, transform: Partial<ModelTransform>): void {
+    const model = this.modelInstances.get(name)?.model
+    if (!model) return
+    if (transform.position) model.setPosition(transform.position)
+    if (transform.rotation) model.setRotation(transform.rotation)
+    if (transform.scale !== undefined) model.setScale(transform.scale)
+    if (transform.visible !== undefined) model.setVisible(transform.visible)
+  }
+
+  /** Read a model's scene transform (for serialization into a scene descriptor). */
+  getModelTransform(name: string): ModelTransform | null {
+    const model = this.modelInstances.get(name)?.model
+    if (!model) return null
+    const p = model.position
+    return {
+      position: new Vec3(p.x, p.y, p.z),
+      rotation: model.rotation.clone(),
+      scale: model.scale,
+      visible: model.visible,
+    }
+  }
+
   markVertexBufferDirty(modelNameOrModel?: string | Model): void {
     if (modelNameOrModel === undefined) return
     if (typeof modelNameOrModel === "string") {
@@ -3727,6 +3830,7 @@ export class Engine {
     pass.setBindGroup(0, this.pickPerFrameBindGroup)
 
     this.forEachInstance((inst) => {
+      if (!inst.model.visible) return // hidden models aren't pickable
       pass.setVertexBuffer(0, inst.vertexBuffer)
       pass.setVertexBuffer(1, inst.jointsBuffer)
       pass.setVertexBuffer(2, inst.weightsBuffer)
@@ -3828,6 +3932,12 @@ export class Engine {
       }
     }
 
+    // Drive the shot from the camera VMD (synced to the animated model's clock).
+    if (this.camera.vmdDriven && this.cameraAnimation) {
+      const pose = this.cameraAnimation.sample(this.cameraClockTime())
+      if (pose) this.camera.setVmdPose(pose)
+    }
+
     this.updateCameraUniforms()
     this.updateShadowLightVP()
 
@@ -3848,12 +3958,17 @@ export class Engine {
         },
       })
       sp.setPipeline(this.shadowDepthPipeline)
-      this.forEachInstance((inst) => this.drawInstanceShadow(sp, inst))
+      this.forEachInstance((inst) => {
+        if (inst.model.visible) this.drawInstanceShadow(sp, inst)
+      })
       sp.end()
     }
 
     const pass = encoder.beginRenderPass(this.renderPassDescriptor)
-    if (hasModels) this.forEachInstance((inst) => this.renderOneModel(pass, inst))
+    if (hasModels)
+      this.forEachInstance((inst) => {
+        if (inst.model.visible) this.renderOneModel(pass, inst)
+      })
     if (this.hasGround) this.renderGround(pass)
     pass.end()
 

package/src/index.ts

@@ -9,6 +9,7 @@ export {
   type LoadModelFromFilesOptions,
   type MaterialPreset,
   type MaterialPresetMap,
+  type ModelTransform,
   type GizmoDragEvent,
   type GizmoDragCallback,
   type GizmoDragKind,
@@ -59,4 +60,6 @@ export type {
   ControlPoint,
 } from "./animation"
 export { FPS } from "./animation"
+export { VMDLoader, type CameraKeyframe } from "./vmd-loader"
+export { CameraAnimation, type CameraPose } from "./camera-animation"
 export { RezePhysics } from "./physics"

package/src/math.ts

@@ -535,6 +535,23 @@ export class Mat4 {
     out[14] = pz
   }
 
+  // Position + rotation + UNIFORM scale. Scales the rotation basis (columns 0–2) by s;
+  // uniform scale keeps normals correct after the shader's normalize() (no inverse-transpose).
+  static fromPositionRotationScaleInto(
+    px: number, py: number, pz: number,
+    qx: number, qy: number, qz: number, qw: number,
+    s: number,
+    out: Float32Array
+  ): void {
+    Mat4.fromQuatInto(qx, qy, qz, qw, out, 0)
+    out[0] *= s; out[1] *= s; out[2] *= s
+    out[4] *= s; out[5] *= s; out[6] *= s
+    out[8] *= s; out[9] *= s; out[10] *= s
+    out[12] = px
+    out[13] = py
+    out[14] = pz
+  }
+
   // In-place 4x4 inverse into out array. Returns true on success, false if singular (out untouched).
   static inverseInto(m: Float32Array, out: Float32Array): boolean {
     const a00 = m[0], a01 = m[1], a02 = m[2], a03 = m[3]

package/src/model.ts

@@ -199,6 +199,17 @@ export class Model {
     return this._rotation
   }
 
+  /** Uniform scale (default 1). Used to fit a stage.pmx to the character. */
+  get scale(): number {
+    return this._scale
+  }
+
+  /** Whether this model renders. Hidden models skip the main, shadow, and pick passes;
+   *  physics keeps running so they resume consistently. */
+  get visible(): boolean {
+    return this._visible
+  }
+
   setPosition(position: Vec3): void {
     this._position.set(position)
     this.rootMatrixDirty = true
@@ -209,6 +220,15 @@ export class Model {
     this.rootMatrixDirty = true
   }
 
+  setScale(scale: number): void {
+    this._scale = scale
+    this.rootMatrixDirty = true
+  }
+
+  setVisible(visible: boolean): void {
+    this._visible = visible
+  }
+
   private vertexData: Float32Array<ArrayBuffer>
   private baseVertexData: Float32Array<ArrayBuffer> // Original vertex data before morphing
   private vertexCount: number
@@ -254,6 +274,8 @@ export class Model {
   // Skip-when-identity flag avoids the extra mat mul per bone when unused.
   private _position: Vec3 = Vec3.zeros()
   private _rotation: Quat = Quat.identity()
+  private _scale: number = 1
+  private _visible: boolean = true
   private rootMatrixValues: Float32Array = new Float32Array([1,0,0,0, 0,1,0,0, 0,0,1,0, 0,0,0,1])
   private rootMatrixDirty: boolean = false
   private rootIsIdentity: boolean = true
@@ -851,11 +873,11 @@ export class Model {
 
     // Rebuild root matrix + cache identity-shortcut flag only when pos/rot changed.
     if (this.rootMatrixDirty) {
-      const p = this._position, r = this._rotation
-      Mat4.fromPositionRotationInto(p.x, p.y, p.z, r.x, r.y, r.z, r.w, this.rootMatrixValues)
+      const p = this._position, r = this._rotation, s = this._scale
+      Mat4.fromPositionRotationScaleInto(p.x, p.y, p.z, r.x, r.y, r.z, r.w, s, this.rootMatrixValues)
       this.rootIsIdentity =
         p.x === 0 && p.y === 0 && p.z === 0 &&
-        r.x === 0 && r.y === 0 && r.z === 0 && r.w === 1
+        r.x === 0 && r.y === 0 && r.z === 0 && r.w === 1 && s === 1
       this.rootMatrixDirty = false
     }
 

package/src/vmd-loader.ts

@@ -20,6 +20,18 @@ export interface VMDKeyFrame {
   morphFrames: MorphFrame[]
 }
 
+/** One MMD camera keyframe. The camera looks at `target` from `distance` away, oriented by
+ *  `rotation` (euler radians), with `fov` degrees. `interpolation` is 24 bytes: 6 channels
+ *  (posX, posY, posZ, rotation, distance, fov) × 4 bezier bytes (x1, x2, y1, y2). */
+export interface CameraKeyframe {
+  frame: number
+  distance: number
+  target: Vec3
+  rotation: Vec3 // euler radians
+  fov: number // degrees
+  interpolation: Uint8Array // 24 bytes
+}
+
 export class VMDLoader {
   private view: DataView
   private offset = 0
@@ -46,6 +58,48 @@ export class VMDLoader {
     return loader.parse()
   }
 
+  /** Parse only the camera track (a dedicated camera VMD, or the camera block of any VMD).
+   *  Returns [] if the file has no camera block (motion-only VMDs end after the morph block). */
+  static async loadCamera(url: string): Promise<CameraKeyframe[]> {
+    const loader = new VMDLoader(await fetch(url).then((r) => r.arrayBuffer()))
+    return loader.parseCamera()
+  }
+
+  static loadCameraFromBuffer(buffer: ArrayBuffer): CameraKeyframe[] {
+    return new VMDLoader(buffer).parseCamera()
+  }
+
+  // Seek past the bone + morph blocks (fixed record sizes) to the camera block, then read it.
+  // bone frame = 111 B (15 name + 4 frame + 12 pos + 16 rot + 64 interp); morph = 23 B
+  // (15 name + 4 frame + 4 weight); camera = 61 B.
+  private parseCamera(): CameraKeyframe[] {
+    this.offset = 0
+    const header = this.getString(30)
+    if (!header.startsWith("Vocaloid Motion Data")) throw new Error("Invalid VMD file header")
+    this.skip(20)
+    const boneCount = this.getUint32()
+    this.skip(boneCount * 111)
+    const morphCount = this.getUint32()
+    this.skip(morphCount * 23)
+    if (this.offset + 4 > this.view.buffer.byteLength) return [] // no camera block
+    const cameraCount = this.getUint32()
+
+    const frames: CameraKeyframe[] = []
+    for (let i = 0; i < cameraCount; i++) {
+      const frame = this.getUint32()
+      const distance = this.getFloat32()
+      const target = new Vec3(this.getFloat32(), this.getFloat32(), this.getFloat32())
+      const rotation = new Vec3(this.getFloat32(), this.getFloat32(), this.getFloat32())
+      const interpolation = new Uint8Array(24)
+      for (let j = 0; j < 24; j++) interpolation[j] = this.getUint8()
+      const fov = this.getUint32()
+      this.skip(1) // perspective flag (0 = perspective) — unused
+      frames.push({ frame, distance, target, rotation, fov, interpolation })
+    }
+    frames.sort((a, b) => a.frame - b.frame)
+    return frames
+  }
+
   private parse(): VMDKeyFrame[] {
     // Read header (30 bytes)
     const header = this.getString(30)