tracky-mouse 2.6.0 → 2.7.0

package/tracky-mouse.js

@@ -52,24 +52,30 @@ const isSelectorValid = ((dummyElement) =>
 
 const dwellClickers = [];
 
+let playSound = () => { console.log("audio module not loaded yet; can't play sound effect"); };
+let initialAudioEnabled = false;
+let setAudioEnabled = (enabled) => { initialAudioEnabled = enabled; };
+
+/**
+ * @param {Object} config
+ * @param {string} config.targets - a CSS selector for the elements to click. Anything else will be ignored (except as an occluder).
+ * @param {(el: Element) => boolean} [config.shouldDrag] - a function that returns true if the element should be dragged rather than simply clicked.
+ * @param {(el: Element) => boolean} [config.noCenter] - a function that returns true if the element should be clicked anywhere on the element, rather than always at the center.
+ * @param {Array<{
+ *   from: string | Element | ((el: Element) => boolean), // - an array of `{ from, to, withinMargin }` objects, which define rules for dynamically changing what is hovered/clicked when the mouse is over a different element.
+ *   to: string | Element | ((el: Element) => Element | null), // - the element to retarget from. Can be a CSS selector, an element, or a function taking the element under the mouse and returning whether it should be retargeted.
+ *   withinMargin?: number // - the element to retarget to. Can be a CSS selector for an element which is an ancestor or descendant of the `from` element, or an element, or a function taking the element under the mouse and returning an element to retarget to, or null to ignore the element.
+ * }>} [config.retarget] - a number of pixels within which to consider the mouse over the `to` element. Default to infinity.
+ * @param {(el1: Element, el2: Element) => boolean} [config.isEquivalentTarget] - a function that returns true if two elements should be considered part of the same control, i.e. if clicking either should do the same thing. Elements that are equal are always considered equivalent even if you return false. This option is used for preventing the system from detecting occluding elements as separate controls, and rejecting the click. (When an occlusion is detected, it flashes a red box.)
+ * @param {(el: Element) => boolean} [config.dwellClickEvenIfPaused] - a function that returns true if the element should be clicked even while dwell clicking is otherwise paused. Use this for a dwell clicking toggle button, so it's possible to resume dwell clicking. With dwell clicking it's important to let users take a break, since otherwise you have to constantly move the cursor in order to not click on things!
+ * @param {(el: Element) => boolean} [config.shouldClickThrough] - a function that returns true if the element should be totally ignored, allowing clicking on content behind it. Prefer `pointer-events: none` when possible, which will work for all input methods. Use this only if you need to differentiate input methods. Default: `(el) => el.matches(".tracky-mouse-click-through, .tracky-mouse-click-through *")`
+ * @param {(args: {x: number, y: number, target: Element}) => void} config.click - a function to trigger a click on the given target element.
+ * @param {() => void} [config.beforeDispatch] - a function to call before a pointer event is dispatched. For detecting un-trusted user gestures, outside of an event handler.
+ * @param {() => void} [config.afterDispatch] - a function to call after a pointer event is dispatched. For detecting un-trusted user gestures, outside of an event handler.
+ * @param {() => void} [config.beforePointerDownDispatch] - a function to call before a `pointerdown` event is dispatched. Likely to be merged with `config.beforeDispatch()` in the future.
+ * @param {() => boolean} [config.isHeld] - a function that returns true if the next dwell should be a release (triggering `pointerup`).
+ */
 const initDwellClicking = (config) => {
-	/*
-		Arguments:
-		- `config.targets` (required): a CSS selector for the elements to click. Anything else will be ignored.
-		- `config.shouldDrag(el)` (optional): a function that returns true if the element should be dragged rather than simply clicked.
-		- `config.noCenter(el)` (optional): a function that returns true if the element should be clicked anywhere on the element, rather than always at the center.
-		- `config.retarget` (optional): an array of `{ from, to, withinMargin }` objects, which define rules for dynamically changing what is hovered/clicked when the mouse is over a different element.
-			- `from` (required): the element to retarget from. Can be a CSS selector, an element, or a function taking the element under the mouse and returning whether it should be retargeted.
-			- `to` (required): the element to retarget to. Can be a CSS selector for an element which is an ancestor or descendant of the `from` element, or an element, or a function taking the element under the mouse and returning an element to retarget to, or null to ignore the element.
-			- `withinMargin` (optional): a number of pixels within which to consider the mouse over the `to` element. Default to infinity.
-		- `config.isEquivalentTarget(el1, el2)` (optional): a function that returns true if two elements should be considered part of the same control, i.e. if clicking either should do the same thing. Elements that are equal are always considered equivalent even if you return false. This option is used for preventing the system from detecting occluding elements as separate controls, and rejecting the click. (When an occlusion is detected, it flashes a red box.)
-		- `config.dwellClickEvenIfPaused(el)` (optional): a function that returns true if the element should be clicked even while dwell clicking is otherwise paused. Use this for a dwell clicking toggle button, so it's possible to resume dwell clicking. With dwell clicking it's important to let users take a break, since otherwise you have to constantly move the cursor in order to not click on things!
-		- `config.click({x, y, target})` (required): a function to trigger a click on the given target element.
-		- `config.beforeDispatch()` (optional): a function to call before a pointer event is dispatched. For detecting un-trusted user gestures, outside of an event handler.
-		- `config.afterDispatch()` (optional): a function to call after a pointer event is dispatched. For detecting un-trusted user gestures, outside of an event handler.
-		- `config.beforePointerDownDispatch()` (optional): a function to call before a `pointerdown` event is dispatched. Likely to be merged with `config.beforeDispatch()` in the future.
-		- `config.isHeld()` (optional): a function that returns true if the next dwell should be a release (triggering `pointerup`).
-	*/
 
 	/** translation placeholder */
 	const t = (key, options = {}) => options.defaultValue ?? key;
@@ -104,6 +110,9 @@ const initDwellClicking = (config) => {
 	if (config.dwellClickEvenIfPaused !== undefined && typeof config.dwellClickEvenIfPaused !== "function") {
 		throw new Error(t("api.errors.functionRequired", { defaultValue: "%0 must be a function" }).replace("%0", "config.dwellClickEvenIfPaused"));
 	}
+	if (config.shouldClickThrough !== undefined && typeof config.shouldClickThrough !== "function") {
+		throw new Error(t("api.errors.functionRequired", { defaultValue: "%0 must be a function" }).replace("%0", "config.shouldClickThrough"));
+	}
 	if (config.beforeDispatch !== undefined && typeof config.beforeDispatch !== "function") {
 		throw new Error(t("api.errors.functionRequired", { defaultValue: "%0 must be a function" }).replace("%0", "config.beforeDispatch"));
 	}
@@ -149,6 +158,8 @@ const initDwellClicking = (config) => {
 		}
 	}
 
+	const shouldClickThrough = config.shouldClickThrough ?? ((el) => el.matches(".tracky-mouse-click-through, .tracky-mouse-click-through *"));
+
 	// trackyMouseContainer.querySelector(".tracky-mouse-canvas").classList.add("inset-deep");
 
 	const circleRadiusMax = 50; // dwell indicator size in pixels
@@ -222,6 +233,14 @@ const initDwellClicking = (config) => {
 			return null;
 		}
 
+		if (shouldClickThrough(target)) {
+			const elements = document.elementsFromPoint(clientX, clientY);
+			target = elements.find(el => !shouldClickThrough(el));
+			if (!target) {
+				return null;
+			}
+		}
+
 		let hoverCandidate = {
 			x: clientX,
 			y: clientY,
@@ -365,6 +384,9 @@ const initDwellClicking = (config) => {
 						showOccluderIndicator(apparentHoverCandidate.target);
 					}
 				} else {
+					// TODO: ignore .tracky-mouse-click-through elements here as well
+					// TODO: distinguish occlusion vs moved element (i.e. element is no longer in the elementsFromPoint list)
+					// for example for the archery targets in the demo on the website, which animate
 					let occluder = document.elementFromPoint(hoverCandidate.x, hoverCandidate.y);
 					hoverCandidate = null;
 					deactivateForAtLeast(inactiveAfterInvalidTimespan);
@@ -391,6 +413,7 @@ const initDwellClicking = (config) => {
 							})
 						));
 						config.afterDispatch?.();
+						playSound("clickRelease");
 					} else {
 						config.beforePointerDownDispatch?.();
 						config.beforeDispatch?.();
@@ -403,6 +426,7 @@ const initDwellClicking = (config) => {
 						config.afterDispatch?.();
 						if (config.shouldDrag?.(hoverCandidate.target)) {
 							dwellDragging = hoverCandidate.target;
+							playSound("clickPress");
 						} else {
 							config.beforeDispatch?.();
 							hoverCandidate.target.dispatchEvent(new PointerEvent("pointerup",
@@ -413,6 +437,8 @@ const initDwellClicking = (config) => {
 							));
 							config.click(hoverCandidate);
 							config.afterDispatch?.();
+							playSound("clickPress");
+							playSound("clickRelease", { delay: 0.03 }); // fully separating the sounds sounded worse
 						}
 					}
 					hoverCandidate = null;
@@ -571,6 +597,28 @@ TrackyMouse.cleanupDwellClicking = function () {
 	}
 };
 
+TrackyMouse._initAudio = async function () {
+	let module;
+	try {
+		// console.log("Loading audio support...");
+		module = await import("./audio.js");
+	} catch (e) {
+		console.warn("Failed to load audio module, click sounds will be disabled:", e);
+	}
+	// console.log("Audio module loaded.");
+	try {
+		const { initAudio } = module;
+		initAudio();
+		playSound = module.playSound;
+		setAudioEnabled = module.setAudioEnabled;
+		setAudioEnabled(initialAudioEnabled);
+		// console.log("Audio is initially " + (initialAudioEnabled ? "enabled" : "disabled"));
+	} catch (e) {
+		console.warn("Failed to initialize audio support, click sounds will be disabled:", e);
+	}
+	return module;
+};
+
 TrackyMouse._initInner = function (div, initOptions, reinit) {
 
 	const {
@@ -590,6 +638,17 @@ TrackyMouse._initInner = function (div, initOptions, reinit) {
 		// Could group things under an "unstable" object, or ideally, design nice APIs for everything.
 	} = initOptions;
 
+	/** @type {SleepSweep | null} */
+	let sleepSweep = null;
+
+	TrackyMouse._initAudio().then((module) => {
+		// _initAudio warns in the console and resolves to undefined if it fails to load audio support
+		if (module) {
+			const { SleepSweep } = module;
+			sleepSweep = new SleepSweep();
+		}
+	});
+
 	const isDesktopApp = !!window.electronAPI;
 
 	let translations = {};
@@ -843,7 +902,7 @@ TrackyMouse._initInner = function (div, initOptions, reinit) {
 	};
 
 
-	var languageToDefaultRegion = {
+	let languageToDefaultRegion = {
 		aa: "ET",
 		ab: "GE",
 		abr: "GH",
@@ -1588,9 +1647,9 @@ TrackyMouse._initInner = function (div, initOptions, reinit) {
 			// </svg>`;
 		}
 
-		var split = locale.toUpperCase().split(/-|_/);
-		var lang = split.shift();
-		var code = split.pop();
+		let split = locale.toUpperCase().split(/-|_/);
+		let lang = split.shift();
+		let code = split.pop();
 
 		if (!/^[A-Z]{2}$/.test(code)) {
 			code = languageToDefaultRegion[lang.toLowerCase()];
@@ -1605,7 +1664,7 @@ TrackyMouse._initInner = function (div, initOptions, reinit) {
 		return a + b;
 	}
 
-	var uiContainer = div || document.createElement("div");
+	let uiContainer = div || document.createElement("div");
 	uiContainer.classList.add("tracky-mouse-ui");
 	uiContainer.classList.toggle("tracky-mouse-rtl", isRTL);
 	uiContainer.dir = isRTL ? "rtl" : "ltr";
@@ -1631,12 +1690,12 @@ TrackyMouse._initInner = function (div, initOptions, reinit) {
 	if (!div) {
 		document.body.appendChild(uiContainer);
 	}
-	var startStopButton = uiContainer.querySelector(".tracky-mouse-start-stop-button");
-	var useCameraButton = uiContainer.querySelector(".tracky-mouse-use-camera-button");
-	var useDemoFootageButton = uiContainer.querySelector(".tracky-mouse-use-demo-footage-button");
-	var errorMessage = uiContainer.querySelector(".tracky-mouse-error-message");
-	var canvasContainer = uiContainer.querySelector('.tracky-mouse-canvas-container');
-	var desktopAppDownloadMessage = uiContainer.querySelector('.tracky-mouse-desktop-app-download-message');
+	let startStopButton = uiContainer.querySelector(".tracky-mouse-start-stop-button");
+	let useCameraButton = uiContainer.querySelector(".tracky-mouse-use-camera-button");
+	let useDemoFootageButton = uiContainer.querySelector(".tracky-mouse-use-demo-footage-button");
+	let errorMessage = uiContainer.querySelector(".tracky-mouse-error-message");
+	let canvasContainer = uiContainer.querySelector('.tracky-mouse-canvas-container');
+	let desktopAppDownloadMessage = uiContainer.querySelector('.tracky-mouse-desktop-app-download-message');
 
 	// Settings (initialized later; defaults are defined in settingsCategories)
 	const s = {};
@@ -2015,6 +2074,20 @@ You may want to turn this off if you're drawing on a canvas, or increase it if y
 			type: "group",
 			label: t("settings.sections.general.label", { defaultValue: "General" }),
 			settings: [
+				{
+					label: t("settings.soundEffects.label", { defaultValue: "Sound effects" }),
+					className: "tracky-mouse-sound-effects",
+					key: "soundEffects",
+					type: "checkbox",
+					default: true,
+					afterInitialLoad: () => {
+						setAudioEnabled(s.soundEffects);
+					},
+					handleSettingChange: () => {
+						setAudioEnabled(s.soundEffects);
+					},
+					description: t("settings.soundEffects.description", { defaultValue: "Plays sounds when you click." }),
+				},
 				// opposite, "Start paused", might be clearer, especially if I add a "pause" button
 				{
 					label: t("settings.startEnabled.label", { defaultValue: "Start enabled" }),
@@ -2036,7 +2109,8 @@ You may want to turn this off if you're drawing on a canvas, or increase it if y
 					// - I considered adding "⚠︎" but it feels a little too alarming
 					// label: "Close eyes to start/stop (<span style=\"border-bottom: 1px dotted;\" title=\"Planned refinements include: visual and auditory feedback, improved detection accuracy, and separate settings for durations to toggle on and off.\">experimental</span>)",
 					// label: "Close eyes to start/stop (<span style=\"border-bottom: 1px dotted;\" title=\"• Missing visual and auditory feedback.\n• Missing settings for duration(s) to toggle on and off.\n• Affected by false positive blink detections, especially when looking downward.\">Experimental</span>)",
-					label: t("settings.closeEyesToToggle.label", { defaultValue: "Close eyes to start/stop (<span style=\"border-bottom: 1px dotted;\" title=\"• There is currently no visual or auditory feedback.\n• There are no settings for duration(s) to toggle on and off.\n• It is affected by false positive blink detections, especially when looking downward.\">Experimental</span>)" }),
+					// label: t("settings.closeEyesToToggle.label", { defaultValue: "Close eyes to start/stop (<span style=\"border-bottom: 1px dotted;\" title=\"• There is currently no visual or auditory feedback.\n• There are no settings for duration(s) to toggle on and off.\n• It is affected by false positive blink detections, especially when looking downward.\">Experimental</span>)" }),
+					label: t("settings.closeEyesToToggle.label", { defaultValue: "Close eyes to start/stop" }),
 					className: "tracky-mouse-close-eyes-to-toggle",
 					key: "closeEyesToToggle",
 					type: "checkbox",
@@ -2309,26 +2383,27 @@ You may want to turn this off if you're drawing on a canvas, or increase it if y
 		});
 	}
 
-	var canvas = uiContainer.querySelector(".tracky-mouse-canvas");
-	var ctx = canvas.getContext('2d', { willReadFrequently: true });
+	let canvas = uiContainer.querySelector(".tracky-mouse-canvas");
+	let ctx = canvas.getContext('2d', { willReadFrequently: true });
 
-	var debugEyeCanvas = document.createElement("canvas");
+	let debugEyeCanvas = document.createElement("canvas");
 	debugEyeCanvas.className = "tracky-mouse-debug-eye-canvas";
 	debugEyeCanvas.style.display = "none";
 	uiContainer.querySelector(".tracky-mouse-canvas-container-container").appendChild(debugEyeCanvas);
-	var debugEyeCtx = debugEyeCanvas.getContext('2d');
+	let debugEyeCtx = debugEyeCanvas.getContext('2d');
 
-	var pointerEl = document.createElement('div');
+	let pointerEl = document.createElement('div');
 	pointerEl.className = "tracky-mouse-pointer";
 	pointerEl.style.display = "none";
 	document.body.appendChild(pointerEl);
 
-	var cameraVideo = document.createElement('video');
+	let cameraVideo = document.createElement('video');
 	// required to work in iOS 11 & up:
 	cameraVideo.setAttribute('playsinline', '');
 
+	let stats;
 	if (statsJs) {
-		var stats = new Stats();
+		stats = new Stats();
 		stats.domElement.style.position = 'fixed';
 		stats.domElement.style.top = '0px';
 		stats.domElement.style.right = '0px';
@@ -2337,72 +2412,74 @@ You may want to turn this off if you're drawing on a canvas, or increase it if y
 	}
 
 	// Debug flags (not shown in the UI; could become Advanced Settings in the future)
-	var debugAcceleration = false;
-	var showDebugText = false;
-	var showDebugEyeZoom = false;
-	var showDebugHeadTilt = false;
+	let debugAcceleration = false;
+	let showDebugText = false;
+	let showDebugEyeZoom = false;
+	let showDebugHeadTilt = false;
+	let showDebugRegionFilter = false;
 
 	// Constants (could become Advanced Settings in the future)
-	var defaultWidth = 640;
-	var defaultHeight = 480;
-	var maxPoints = 1000;
-	var faceScoreThreshold = 0.5;
-	var facemeshOptions = {
+	let defaultWidth = 640;
+	let defaultHeight = 480;
+	let maxPoints = 1000;
+	let faceScoreThreshold = 0.5;
+	let facemeshOptions = {
 		maxContinuousChecks: 5,
 		detectionConfidence: 0.9,
 		maxFaces: 1,
 		iouThreshold: 0.3,
 		scoreThreshold: 0.75
 	};
-	var useFacemesh = true;
+	let useFacemesh = true;
+	let sleepGestureEyesClosedDuration = 2000;
 	// maybe should be based on size of head in view?
 	const pruningGridSize = 5;
 	const minDistanceToAddPoint = pruningGridSize * 1.5;
 
 	// Head tracking and facial gesture state
 	// ## Clmtrackr state
-	var face;
-	var faceScore = 0;
-	var faceConvergence = 0;
-	// var faceConvergenceThreshold = 50;
-	var pointsBasedOnFaceScore = 0;
+	let face;
+	let faceScore = 0;
+	let faceConvergence = 0;
+	// let faceConvergenceThreshold = 50;
+	let pointsBasedOnFaceScore = 0;
 	// ## Facemesh state
 	let detector;
 	let currentCameraImageData;
-	var facemeshLoaded = false;
-	var facemeshFirstEstimation = true;
-	var facemeshEstimating = false;
-	var facemeshRejectNext = 0;
-	var facemeshPrediction;
-	var facemeshEstimateFaces;
-	var faceInViewConfidenceThreshold = 0.7;
-	var pointsBasedOnFaceInViewConfidence = 0;
-	var cameraFramesSinceFacemeshUpdate = [];
-	var blinkInfo;
-	var mouthInfo;
-	var headTilt = { pitch: 0, yaw: 0, roll: 0 };
-	var headTiltFilters = { pitch: null, yaw: null, roll: null };
-	var lastTimeWhenAnEyeWasOpen = Infinity; // far future rather than far past so that sleep gesture doesn't trigger initially, skipping the delay
+	let facemeshLoaded = false;
+	let facemeshFirstEstimation = true;
+	let facemeshEstimating = false;
+	let facemeshRejectNext = 0;
+	let facemeshPrediction;
+	let facemeshEstimateFaces;
+	let faceInViewConfidenceThreshold = 0.7;
+	let pointsBasedOnFaceInViewConfidence = 0;
+	let cameraFramesSinceFacemeshUpdate = [];
+	let blinkInfo;
+	let mouthInfo;
+	let headTilt = { pitch: 0, yaw: 0, roll: 0 };
+	let headTiltFilters = { pitch: null, yaw: null, roll: null };
+	let sleepGestureProgress = 0;
 	// ## State related to switching between head trackers
-	var useClmTracking = true;
-	var showClmTracking = useClmTracking;
-	var fallbackTimeoutID;
+	let useClmTracking = true;
+	let showClmTracking = useClmTracking;
+	let fallbackTimeoutID;
 
 	// Mouse state
-	var mouseX = 0;
-	var mouseY = 0;
-	var buttonStates = {
+	let mouseX = 0;
+	let mouseY = 0;
+	let buttonStates = {
 		left: false,
 		right: false,
 		middle: false,
 	};
-	var mouseButtonUntilMouthCloses = -1;
-	var lastMouseDownTime = -Infinity;
-	var mouseNeedsInitPos = true;
+	let mouseButtonUntilMouthCloses = -1;
+	let lastMouseDownTime = -Infinity;
+	let mouseNeedsInitPos = true;
 
 	// Other state
-	var paused = true;
-	var pointTracker;
+	let paused = true;
+	let pointTracker;
 
 	// Named lists of facemesh landmark indices
 	const MESH_ANNOTATIONS = {
@@ -2514,6 +2591,7 @@ You may want to turn this off if you're drawing on a canvas, or increase it if y
 				setting._load?.(settings, initialLoad);
 			});
 		}
+		setAudioEnabled(s.soundEffects);
 
 		// Now that all settings are loaded, update disabled states
 		for (const func of functionsToUpdateDisabledStates) {
@@ -2579,6 +2657,12 @@ You may want to turn this off if you're drawing on a canvas, or increase it if y
 		}
 		if (stored) {
 			deserializeSettings(stored, initialLoad);
+		} else {
+			// HACK: ensure handleInitialLoad is called even for first run
+			// Combined with the below, this feels very redundant, and I'd like to
+			// move to a subscription-based pattern, more of a formal "settings store", something like that.
+			// This is currently necessary for sound effects to work on the first run of the web demo.
+			deserializeSettings(serializeSettings(), initialLoad);
 		}
 		if (initialLoad && (!stored || !stored.globalSettings || Object.keys(stored.globalSettings).length === 0)) {
 			// We could just call setOptions in both cases,
@@ -2692,9 +2776,9 @@ You may want to turn this off if you're drawing on a canvas, or increase it if y
 	const settingsLoadedPromise = loadOptions(true);
 
 	// Don't use WebGL because clmTracker is our fallback! It's also not much slower than with WebGL.
-	var clmTracker = new clm.tracker({ useWebGL: false });
+	let clmTracker = new clm.tracker({ useWebGL: false });
 	clmTracker.init();
-	var clmTrackingStarted = false;
+	let clmTrackingStarted = false;
 
 	const stopCameraStream = () => {
 		if (cameraVideo.srcObject) {
@@ -2720,7 +2804,7 @@ You may want to turn this off if you're drawing on a canvas, or increase it if y
 		pointsBasedOnFaceScore = 0;
 		faceScore = 0;
 		faceConvergence = 0;
-		lastTimeWhenAnEyeWasOpen = Infinity; // far future rather than far past so that sleep gesture doesn't trigger initially, skipping the delay
+		sleepGestureProgress = 0;
 		updateStartStopButton();
 	};
 
@@ -3008,7 +3092,7 @@ You may want to turn this off if you're drawing on a canvas, or increase it if y
 		}
 		addPoint(x, y) {
 			if (this.pointCount < maxPoints) {
-				var pointIndex = this.pointCount * 2;
+				let pointIndex = this.pointCount * 2;
 				this.curXY[pointIndex] = x;
 				this.curXY[pointIndex + 1] = y;
 				this.prevXY[pointIndex] = x;
@@ -3017,8 +3101,8 @@ You may want to turn this off if you're drawing on a canvas, or increase it if y
 			}
 		}
 		filterPoints(condition) {
-			var outputPointIndex = 0;
-			for (var inputPointIndex = 0; inputPointIndex < this.pointCount; inputPointIndex++) {
+			let outputPointIndex = 0;
+			for (let inputPointIndex = 0; inputPointIndex < this.pointCount; inputPointIndex++) {
 				if (condition(inputPointIndex)) {
 					if (outputPointIndex < inputPointIndex) {
 						const inputOffset = inputPointIndex * 2;
@@ -3066,10 +3150,10 @@ You may want to turn this off if you're drawing on a canvas, or increase it if y
 			[this.prevPyramid, this.curPyramid] = [this.curPyramid, this.prevPyramid];
 
 			// these are options worth breaking out and exploring
-			var winSize = 20;
-			var maxIterations = 30;
-			var epsilon = 0.01;
-			var minEigen = 0.001;
+			let winSize = 20;
+			let maxIterations = 30;
+			let epsilon = 0.01;
+			let minEigen = 0.001;
 
 			jsfeat.imgproc.grayscale(imageData.data, imageData.width, imageData.height, this.curPyramid.data[0]);
 			this.curPyramid.build(this.curPyramid.data[0], true);
@@ -3083,9 +3167,9 @@ You may want to turn this off if you're drawing on a canvas, or increase it if y
 			this.prunePoints();
 		}
 		draw(ctx) {
-			for (var i = 0; i < this.pointCount; i++) {
-				var pointOffset = i * 2;
-				// var distMoved = Math.hypot(
+			for (let i = 0; i < this.pointCount; i++) {
+				let pointOffset = i * 2;
+				// let distMoved = Math.hypot(
 				// 	this.prevXY[pointOffset] - this.curXY[pointOffset],
 				// 	this.prevXY[pointOffset + 1] - this.curXY[pointOffset + 1]
 				// );
@@ -3103,11 +3187,11 @@ You may want to turn this off if you're drawing on a canvas, or increase it if y
 			}
 		}
 		getMovement() {
-			var movementX = 0;
-			var movementY = 0;
-			var numMovements = 0;
-			for (var i = 0; i < this.pointCount; i++) {
-				var pointOffset = i * 2;
+			let movementX = 0;
+			let movementY = 0;
+			let numMovements = 0;
+			for (let i = 0; i < this.pointCount; i++) {
+				let pointOffset = i * 2;
 				movementX += this.curXY[pointOffset] - this.prevXY[pointOffset];
 				movementY += this.curXY[pointOffset + 1] - this.prevXY[pointOffset + 1];
 				numMovements += 1;
@@ -3144,9 +3228,9 @@ You may want to turn this off if you're drawing on a canvas, or increase it if y
 		// in order to keep a smooth overall tracking calculation,
 		// don't add points if they're close to an existing point.
 		// Otherwise, it would not just be redundant, but often remove the older points, in the pruning.
-		for (var pointIndex = 0; pointIndex < oops.pointCount; pointIndex++) {
-			var pointOffset = pointIndex * 2;
-			// var distance = Math.hypot(
+		for (let pointIndex = 0; pointIndex < oops.pointCount; pointIndex++) {
+			let pointOffset = pointIndex * 2;
+			// let distance = Math.hypot(
 			// 	x - oops.curXY[pointOffset],
 			// 	y - oops.curXY[pointOffset + 1]
 			// );
@@ -3183,6 +3267,7 @@ You may want to turn this off if you're drawing on a canvas, or increase it if y
 		return ((px - x1) * nx + (py - y1) * ny) / Math.hypot(nx, ny);
 	}
 
+	let lastTimestamp = -Infinity;
 	function draw(update = true) {
 		ctx.resetTransform(); // in case there is an error, don't flip constantly back and forth due to mirroring
 		ctx.clearRect(0, 0, canvas.width, canvas.height); // in case there's no footage
@@ -3197,6 +3282,13 @@ You may want to turn this off if you're drawing on a canvas, or increase it if y
 			ctx.drawImage(cameraVideo, 0, 0, canvas.width, canvas.height);
 		}
 
+		const timestamp = performance.now();
+		const deltaTime = Math.min(timestamp - lastTimestamp, 100);
+		lastTimestamp = timestamp;
+
+		sleepSweep?.setEnabled(s.closeEyesToToggle);
+		sleepSweep?.update(sleepGestureProgress);
+
 		if (!pointTracker) {
 			return;
 		}
@@ -3332,39 +3424,77 @@ You may want to turn this off if you're drawing on a canvas, or increase it if y
 
 						// TODO: separate confidence threshold for removing vs adding points?
 
+
 						// cull points to those within useful facial region
-						pointTracker.filterPoints((pointIndex) => {
-							var pointOffset = pointIndex * 2;
+						function regionFilter([x, y]) {
+
 							// distance from tip of nose (stretched so make an ellipse taller than wide)
-							var distance = Math.hypot(
-								(annotations.noseTip[0][0] - pointTracker.curXY[pointOffset]) * 1.4,
-								annotations.noseTip[0][1] - pointTracker.curXY[pointOffset + 1]
+							let distance = Math.hypot(
+								(annotations.noseTip[0][0] - x) * 1.4,
+								annotations.noseTip[0][1] - y
 							);
-							var headSize = Math.hypot(
+							let headSize = Math.hypot(
 								annotations.leftCheek[0][0] - annotations.rightCheek[0][0],
 								annotations.leftCheek[0][1] - annotations.rightCheek[0][1]
 							);
 							if (distance > headSize) {
 								return false;
 							}
+							// Avoid mouth affecting pointer position.
+							distance = annotations.lipsLowerInner.map((lipPoint) =>
+								Math.min(
+									Math.hypot(lipPoint[0] - x, lipPoint[1] - y),
+									Math.hypot(lipPoint[0] - x, lipPoint[1] + headSize * 0.1 - y), // a bit below too
+									Math.hypot(lipPoint[0] - x, lipPoint[1] + headSize * 0.2 - y), // a bit below too
+									Math.hypot(lipPoint[0] - x, lipPoint[1] + headSize * 0.3 - y), // a bit below too
+									Math.hypot(lipPoint[0] - x, lipPoint[1] + headSize * 0.4 - y), // a bit below too (yeah I'm being a little lazy here)
+								)
+							).reduce((a, b) => Math.min(a, b), Infinity);
+							if (distance < headSize * 0.1) {
+								return false;
+							}
 							// Avoid blinking eyes affecting pointer position.
 							// distance to outer corners of eyes
 							distance = Math.min(
 								Math.hypot(
-									annotations.leftEyeLower0[0][0] - pointTracker.curXY[pointOffset],
-									annotations.leftEyeLower0[0][1] - pointTracker.curXY[pointOffset + 1]
+									annotations.leftEyeLower0[0][0] - x,
+									annotations.leftEyeLower0[0][1] - y
 								),
 								Math.hypot(
-									annotations.rightEyeLower0[0][0] - pointTracker.curXY[pointOffset],
-									annotations.rightEyeLower0[0][1] - pointTracker.curXY[pointOffset + 1]
+									annotations.rightEyeLower0[0][0] - x,
+									annotations.rightEyeLower0[0][1] - y
 								),
 							);
 							if (distance < headSize * 0.42) {
 								return false;
 							}
 							return true;
+						}
+						pointTracker.filterPoints((pointIndex) => {
+							let pointOffset = pointIndex * 2;
+							const point = [pointTracker.curXY[pointOffset], pointTracker.curXY[pointOffset + 1]];
+							return regionFilter(point);
 						});
 
+						// Debug visualization for region filter (a sort of heatmap of where points will be culled)
+						if (showDebugRegionFilter) {
+							ctx.save();
+							if (s.mirror) {
+								ctx.translate(canvas.width, 0);
+								ctx.scale(-1, 1);
+							}
+							ctx.fillStyle = "rgba(255, 0, 0, 0.5)";
+							const vizStep = 4;
+							for (let x = 0; x < canvas.width; x += vizStep) {
+								for (let y = 0; y < canvas.height; y += vizStep) {
+									if (!regionFilter([x, y])) {
+										ctx.fillRect(x - 5, y - 5, vizStep, vizStep);
+									}
+								}
+							}
+							ctx.restore();
+						}
+
 						const keypoints = facemeshPrediction.keypoints;
 						if (keypoints) {
 							const top = keypoints[10]; // Top of forehead
@@ -3546,16 +3676,19 @@ You may want to turn this off if you're drawing on a canvas, or increase it if y
 
 						blinkInfo = detectBlinks();
 						mouthInfo = detectMouthOpen();
-						if (blinkInfo.rightEye.open || blinkInfo.leftEye.open) {
-							lastTimeWhenAnEyeWasOpen = performance.now();
+						if (!blinkInfo.rightEye.open && !blinkInfo.leftEye.open) {
+							sleepGestureProgress += deltaTime / sleepGestureEyesClosedDuration;
+							sleepGestureProgress = Math.min(sleepGestureProgress, 1);
+						} else {
+							sleepGestureProgress -= deltaTime / sleepGestureEyesClosedDuration;
+							sleepGestureProgress = Math.max(sleepGestureProgress, 0);
 						}
-						if (performance.now() - lastTimeWhenAnEyeWasOpen > 2000) {
+						if (sleepGestureProgress >= 1) {
+							sleepGestureProgress = 0;
 							if (s.closeEyesToToggle) {
 								paused = !paused;
 								updatePaused();
-								// TODO: handle edge cases
-								// TODO: try to keep variable names meaningful
-								lastTimeWhenAnEyeWasOpen = Infinity;
+								sleepSweep?.sleepModeWasToggled(paused);
 							}
 						}
 
@@ -3617,10 +3750,41 @@ You may want to turn this off if you're drawing on a canvas, or increase it if y
 
 						const buttonNames = ["left", "middle", "right"];
 						for (let buttonIndex = 0; buttonIndex < 3; buttonIndex++) {
-							if ((clickButton === buttonIndex) !== buttonStates[buttonNames[buttonIndex]]) {
-								setMouseButtonState(buttonIndex, clickButton === buttonIndex);
-								buttonStates[buttonNames[buttonIndex]] = clickButton === buttonIndex;
-								if ((clickButton === buttonIndex)) {
+							const buttonIsActive = clickButton === buttonIndex;
+							if (buttonIsActive !== buttonStates[buttonNames[buttonIndex]]) {
+								// Wait for confirmation of the button state change before playing SFX
+								// but not before updating buttonStates, since we check that in this loop
+								// to decide whether to call setMouseButtonState.
+								// We don't want to send extraneous mouse button changes to the main process,
+								// even if it does track button states itself. If nothing else it's wasted IPC.
+								// That said, an argument could be made for updating lastMouseDownTime later
+								// if the IPC is slow, to extend the time frame for making a simple click
+								// rather than a drag.
+								if (!setMouseButtonState) {
+									console.warn("setMouseButtonState function not provided");
+								} else {
+									const maybeAPromise = setMouseButtonState(buttonIndex, buttonIsActive);
+									const playSoundForButton = (changedButtonState) => {
+										if (changedButtonState) {
+											if (buttonIndex === 1) {
+												playSound(buttonIsActive ? "middleClickPress" : "middleClickRelease", {
+													volume: 4,
+												});
+											} else {
+												playSound(buttonIsActive ? "clickPress" : "clickRelease", {
+													playbackRate: buttonIndex === 0 ? 1 : buttonIndex === 2 ? 1.2 : 1.5,
+												});
+											}
+										}
+									};
+									if (maybeAPromise instanceof Promise) {
+										maybeAPromise.then(playSoundForButton);
+									} else {
+										playSoundForButton(maybeAPromise);
+									}
+								}
+								buttonStates[buttonNames[buttonIndex]] = buttonIsActive;
+								if (buttonIsActive) {
 									lastMouseDownTime = performance.now();
 								} else {
 									// Limit "Delay Before Dragging" effect to the duration of a click.
@@ -3893,14 +4057,14 @@ You may want to turn this off if you're drawing on a canvas, or increase it if y
 
 					// cull points to those within useful facial region
 					pointTracker.filterPoints((pointIndex) => {
-						var pointOffset = pointIndex * 2;
+						let pointOffset = pointIndex * 2;
 						// distance from tip of nose (stretched so make an ellipse taller than wide)
-						var distance = Math.hypot(
+						let distance = Math.hypot(
 							(face[62][0] - pointTracker.curXY[pointOffset]) * 1.4,
 							face[62][1] - pointTracker.curXY[pointOffset + 1]
 						);
 						// distance based on outer eye corners
-						var headSize = Math.hypot(
+						let headSize = Math.hypot(
 							face[23][0] - face[28][0],
 							face[23][1] - face[28][1]
 						);
@@ -3928,18 +4092,18 @@ You may want to turn this off if you're drawing on a canvas, or increase it if y
 			const screenWidth = window.electronAPI ? screen.width : innerWidth;
 			const screenHeight = window.electronAPI ? screen.height : innerHeight;
 
-			var [movementX, movementY] = pointTracker.getMovement();
+			let [movementX, movementY] = pointTracker.getMovement();
 
 			// Acceleration curves add a lot of stability,
 			// letting you focus on a specific point without jitter, but still move quickly.
 
-			// var accelerate = (delta, distance) => (delta / 10) * (distance ** 0.8);
-			// var accelerate = (delta, distance) => (delta / 1) * (Math.abs(delta) ** 0.8);
-			var accelerate = (delta, _distance) => (delta / 1) * (Math.abs(delta * 5) ** s.headTrackingAcceleration);
+			// let accelerate = (delta, distance) => (delta / 10) * (distance ** 0.8);
+			// let accelerate = (delta, distance) => (delta / 1) * (Math.abs(delta) ** 0.8);
+			let accelerate = (delta, _distance) => (delta / 1) * (Math.abs(delta * 5) ** s.headTrackingAcceleration);
 
-			var distance = Math.hypot(movementX, movementY);
-			var deltaX = accelerate(movementX * s.headTrackingSensitivityX, distance);
-			var deltaY = accelerate(movementY * s.headTrackingSensitivityY, distance);
+			let distance = Math.hypot(movementX, movementY);
+			let deltaX = accelerate(movementX * s.headTrackingSensitivityX, distance);
+			let deltaY = accelerate(movementY * s.headTrackingSensitivityY, distance);
 
 			if (s.headTrackingTiltInfluence > 0) {
 				const yawRange = [
@@ -4112,7 +4276,7 @@ You may want to turn this off if you're drawing on a canvas, or increase it if y
 
 	// Can't use requestAnimationFrame, doesn't work with webPreferences.backgroundThrottling: false (at least in some version of Electron (v12 I think, when I tested it), on Ubuntu, with XFCE)
 	const iid = setInterval(function animationLoop() {
-		draw(!paused || document.visibilityState === "visible");
+		draw(!paused || document.visibilityState === "visible" || isDesktopApp);
 	}, 15);
 
 	let autoDemo = false;