substrate-lang 0.10.0 → 0.10.2

package/substrate/host/web/gpu.mjs

@@ -39,6 +39,8 @@ function unavailable(reason) {
     source: SOURCE,
     apply() { caller.trap(`gpu: WebGPU is not available: ${reason}`); },
     loading() { return 0; },
+    listen() { return 0; },
+    unlisten() {},
   });
   module.available = false;
   module.adapter = {};
@@ -71,17 +73,31 @@ export async function gpu(options = {}) {
   const layered = new Map();
   const bindGroups = new Map();
   const tables = { 1: surfaces, 2: shaders, 3: pipelines, 4: buffers, 5: textures, 6: samplers, 7: layouts, 8: bindGroups };
-  let loading = 0;
+  const loading = new Map();
+  const counted = (t, n) => {
+    const k = (loading.get(t) || 0) + n;
+    if (k === 0) loading.delete(t);
+    else loading.set(t, k);
+  };
 
   let app = null;
   let caller = null;
-  function event(kind, id, data) {
-    if (caller) caller.event(kind, id, 0, kind === ERROR ? errors.invalid : 0, data);
+  function event(kind, id, data, to = 0) {
+    if (caller) caller.event(kind, id, to, kind === ERROR ? errors.invalid : 0, data);
+  }
+  const error = (id, message, to = 0) => event(ERROR, id, utf8.encode(message), to);
+
+  let tag = 0;
+  const listeners = new Map();
+  let nextListener = 1;
+
+  function deviceError(message) {
+    if (listeners.size === 0) error(0, message);
+    for (const t of listeners.values()) error(0, message, t);
   }
-  const error = (id, message) => event(ERROR, id, utf8.encode(message));
 
-  device.addEventListener('uncapturederror', (e) => error(0, e.error.message));
-  device.lost.then((info) => error(0, `the device was lost (${info.reason}): ${info.message}`));
+  device.addEventListener('uncapturederror', (e) => deviceError(e.error.message));
+  device.lost.then((info) => deviceError(`the device was lost (${info.reason}): ${info.message}`));
 
   function get(kind, id) {
     const r = tables[kind].get(id);
@@ -100,7 +116,18 @@ export async function gpu(options = {}) {
     return v;
   };
 
-  function run(memory, w, f, string) {
+  function run(memory, w, f, string, batchTag) {
+    tag = batchTag;
+    device.pushErrorScope('validation');
+    try {
+      commands(memory, w, f, string);
+    } finally {
+      tag = 0;
+      device.popErrorScope().then((e) => { if (e) error(0, e.message, batchTag); });
+    }
+  }
+
+  function commands(memory, w, f, string) {
     const encoder = device.createCommandEncoder();
     let pass = null;
     let computing = false;
@@ -117,7 +144,7 @@ export async function gpu(options = {}) {
         case 1: {
           const id = w[i + 1];
           const node = theDom && theDom.nodeOf ? theDom.nodeOf(w[i + 2]) : null;
-          if (!node) caller.trap(`gpu: surface ${id}: the page gave this module no dom, or node ${w[i + 2]} is not on the page`);
+          if (!node) caller.trap(`gpu: surface ${id}: this module has no dom, or node ${w[i + 2]} is not in the document`);
           if (!(node instanceof HTMLCanvasElement)) caller.trap(`gpu: surface ${id}: node ${w[i + 2]} is not a canvas`);
           if (options.offscreen) {
             const texture = device.createTexture({
@@ -137,6 +164,7 @@ export async function gpu(options = {}) {
         }
         case 2: {
           const id = w[i + 1];
+          const to = tag;
           device.pushErrorScope('validation');
           const module = device.createShaderModule({ label: `shader ${id}`, code: string(i + 2) });
           shaders.set(id, module);
@@ -144,7 +172,7 @@ export async function gpu(options = {}) {
             if (!e) return;
             const errors = (await module.getCompilationInfo()).messages.filter((m) => m.type === 'error');
             error(id, errors.length === 0 ? e.message : `shader ${id} does not compile:\n` +
-              errors.map((m) => `  line ${m.lineNum}, column ${m.linePos}: ${m.message}`).join('\n'));
+              errors.map((m) => `  line ${m.lineNum}, column ${m.linePos}: ${m.message}`).join('\n'), to);
           });
           i += 4;
           break;
@@ -246,7 +274,7 @@ export async function gpu(options = {}) {
             usage: GPUTextureUsage.TEXTURE_BINDING | GPUTextureUsage.COPY_DST | GPUTextureUsage.COPY_SRC,
           });
           textures.set(id, { texture: placeholder, format: 'rgba8unorm', width: 1, height: 1, layers: 1 });
-          loadTexture(id, name);
+          loadTexture(id, name, tag);
           i += 4;
           break;
         }
@@ -311,7 +339,7 @@ export async function gpu(options = {}) {
             entries.push({ binding: w[i], kind, id: w[i + 2], offset: w[i + 3], size: w[i + 4] });
             i += 5;
           }
-          const description = { id, layout, entries, layers: layoutId !== 0 ? layered.get(layoutId) : new Set() };
+          const description = { id, layout, entries, layers: layoutId !== 0 ? layered.get(layoutId) : new Set(), tag };
           description.group = makeBindGroup(description);
           bindGroups.set(id, description);
           break;
@@ -347,17 +375,19 @@ export async function gpu(options = {}) {
         }
         case 14: {
           const id = w[i + 1];
+          const to = tag;
           const read = readTexture(encoder, id, w[i + 2], w[i + 3], w[i + 4], w[i + 5], w[i + 6], w[i + 7]);
-          reads.push(() => read().then((bytes) => event(PIXELS, id, bytes), (e) => error(id, e.message)));
+          reads.push(() => read().then((bytes) => event(PIXELS, id, bytes, to), (e) => error(id, e.message, to)));
           i += 8;
           break;
         }
         case 15: {
           const id = w[i + 1];
+          const to = tag;
           const len = w[i + 3];
           const staging = device.createBuffer({ size: (len + 3) & ~3, usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ });
           encoder.copyBufferToBuffer(get(4, id), w[i + 2], staging, 0, (len + 3) & ~3);
-          reads.push(() => readStaging(staging, len).then((bytes) => event(BYTES, id, bytes), (e) => error(id, e.message)));
+          reads.push(() => readStaging(staging, len).then((bytes) => event(BYTES, id, bytes, to), (e) => error(id, e.message, to)));
           i += 4;
           break;
         }
@@ -499,10 +529,10 @@ export async function gpu(options = {}) {
     for (const read of reads) read();
   }
 
-  function checked(what, id, make) {
+  function checked(what, id, make, to = tag) {
     device.pushErrorScope('validation');
     const resource = make();
-    device.popErrorScope().then((e) => { if (e) error(id, `${what} is invalid: ${e.message}`); });
+    device.popErrorScope().then((e) => { if (e) error(id, `${what} is invalid: ${e.message}`, to); });
     return resource;
   }
 
@@ -529,11 +559,11 @@ export async function gpu(options = {}) {
     });
   }
 
-  async function loadTexture(id, name) {
-    loading++;
+  async function loadTexture(id, name, to) {
+    counted(to, 1);
     try {
       const bytes = app && app.files.get(resolvePath(app.cwd, name));
-      const blob = bytes ? new Blob([bytes]) : await fetch(name).then((r) => {
+      const blob = bytes ? new Blob([bytes]) : await fetch(app ? app.url(name) : name).then((r) => {
         if (!r.ok) throw new Error(`the server answered ${r.status}`);
         return r.blob();
       });
@@ -564,15 +594,15 @@ export async function gpu(options = {}) {
         if (description.entries.some((e) => e.kind === 3 && e.id === id)) description.group = makeBindGroup(description);
       }
       t.texture.destroy();
-      event(LOADED, id, new Uint8Array(0));
+      event(LOADED, id, new Uint8Array(0), to);
     } catch (e) {
-      error(id, `cannot load the texture ${name}: ${e.message}`);
+      error(id, `cannot load the texture ${name}: ${e.message}`, to);
     } finally {
-      loading--;
+      counted(to, -1);
     }
   }
 
-  function makeBindGroup({ id, layout, entries, layers }) {
+  function makeBindGroup({ id, layout, entries, layers, tag: to }) {
     return checked(`bind group ${id}`, id, () => device.createBindGroup({
       label: `bind group ${id}`,
       layout,
@@ -590,7 +620,7 @@ export async function gpu(options = {}) {
         }
         return { binding: e.binding, resource };
       }),
-    }));
+    }), to);
   }
 
   function readTexture(encoder, id, mip, layer, x, y, width, height) {
@@ -624,17 +654,25 @@ export async function gpu(options = {}) {
     app = theApp;
     return {
       source: SOURCE,
-      apply(ptr, len) {
+      apply(ptr, len, t) {
         const memory = caller.memory();
         const w = ptr % 4 === 0
           ? new Uint32Array(memory.buffer, memory.byteOffset + ptr, len >>> 2)
           : new Uint32Array(memory.slice(ptr, ptr + len).buffer);
         const f = new Float32Array(w.buffer, w.byteOffset, w.length);
         const string = (at) => text.decode(memory.subarray(w[at], w[at] + w[at + 1]));
-        run(memory, w, f, string);
+        run(memory, w, f, string, t);
+      },
+      loading(t) {
+        return loading.get(t) || 0;
+      },
+      listen(t) {
+        const id = nextListener++;
+        listeners.set(id, t);
+        return id;
       },
-      loading() {
-        return loading;
+      unlisten(id) {
+        listeners.delete(id);
       },
     };
   };

package/substrate/host/web/gpu.sub

@@ -1,5 +1,8 @@
-;; WebGPU: the host module "gpu".
-(provide gpu-source gpu-error gpu-loaded gpu-pixels gpu-bytes gpu-apply gpu-loading)
+;; WebGPU: the host module "gpu". gpu-apply runs a batch of commands, and the events it causes go to its tag: errors,
+;; loaded texture files and finished reads. gpu-loading counts the texture files loading for a tag. Each listener's tag
+;; hears the errors about the whole device, such as its loss. gpu-listen gives the listener's number, and gpu-unlisten
+;; takes it.
+(provide gpu-source gpu-error gpu-loaded gpu-pixels gpu-bytes gpu-apply gpu-loading gpu-listen gpu-unlisten)
 
 (const gpu-source u32 2)
 (const gpu-error u32 1)
@@ -7,5 +10,16 @@
 (const gpu-pixels u32 3)
 (const gpu-bytes u32 4)
 
-(import gpu-apply "gpu" "apply" (i32 i32 ->))
-(import gpu-loading "gpu" "loading" (-> i32))
+(import apply "gpu" "apply" (i32 i32 i32 ->))
+(import loading "gpu" "loading" (i32 -> i32))
+(import listen "gpu" "listen" (i32 -> i32))
+(import unlisten "gpu" "unlisten" (i32 ->))
+
+(define (gpu-apply [commands (slice u8)] [tag (ptr void)])
+  (apply (cast i32 (-> commands items)) (cast i32 (-> commands length)) (cast i32 tag)))
+
+(define (gpu-loading [tag (ptr void)]) -> u32 (cast u32 (loading (cast i32 tag))))
+
+(define (gpu-listen [tag (ptr void)]) -> u32 (cast u32 (listen (cast i32 tag))))
+
+(define (gpu-unlisten [id u32]) (unlisten (cast i32 id)))

package/substrate/host/web/location.mjs

@@ -8,7 +8,12 @@ const CHANGED = 1;
 
 export function locationModule() {
   return (caller) => {
-    const popped = () => caller.event(CHANGED, 0, 0, 0, utf8.encode(globalThis.location.href));
+    const listeners = new Map();
+    let next = 1;
+    const popped = () => {
+      const url = utf8.encode(globalThis.location.href);
+      for (const [id, tag] of listeners) caller.event(CHANGED, id, tag, 0, url);
+    };
     addEventListener('popstate', popped);
     const move = (go, ptr, len, err) => {
       try {
@@ -30,6 +35,15 @@ export function locationModule() {
       back() {
         history.back();
       },
+      listen(tag) {
+        const id = next++;
+        listeners.set(id, tag);
+        return id;
+      },
+      unlisten(id) {
+        listeners.delete(id);
+        caller.forget(id);
+      },
     };
   };
 }

package/substrate/host/web/location.sub

@@ -1,11 +1,15 @@
-;; The page's URL and history: the host module "location". When the user goes back or forward, an untagged changed
-;; event brings the URL as its bytes.
-(provide location-source location-changed location-get location-push location-replace location-back)
+;; The document's URL and history: the host module "location". When the user goes back or forward, each listener's tag
+;; gets a changed event with the URL as its bytes. location-listen gives the listener's number, and location-unlisten
+;; takes it.
+(provide location-source location-changed location-get location-push location-replace location-back location-listen
+         location-unlisten)
 
 (import get "location" "get" (i32 -> i32))
 (import push "location" "push" (i32 i32 i32 -> i32))
 (import replace "location" "replace" (i32 i32 i32 -> i32))
 (import back "location" "back" (->))
+(import listen "location" "listen" (i32 -> i32))
+(import unlisten "location" "unlisten" (i32 ->))
 
 (const location-source u32 12)
 (const location-changed u32 1)
@@ -23,3 +27,7 @@
         (host-failed! (& why) err))))
 
 (define (location-back) (back))
+
+(define (location-listen [tag (ptr void)]) -> u32 (cast u32 (listen (cast i32 tag))))
+
+(define (location-unlisten [id u32]) (unlisten (cast i32 id)))

package/substrate/host/web/start.mjs

@@ -1,5 +1,6 @@
-// Starts a site's program with the files in assets.list and files.pack, the URL's arg parameters as its arguments, and
-// the host modules it imports. It sets window.modules, sets window.ready once the program has started, and sets
+// Starts a site's program with the files in files.pack, the URL's arg parameters as its arguments, and the host modules
+// it imports. The program, files.pack and assets.list sit beside the host directory. The assets load from the site
+// when the program asks for them. It sets window.modules, sets window.ready once the program has started, and sets
 // window.ended to the program's exit status and window.endReason to why it ended.
 import { boot } from './subhost-web.mjs';
 import { allGrants } from '../grants.mjs';
@@ -7,6 +8,7 @@ import { allGrants } from '../grants.mjs';
 const query = new URLSearchParams(location.search);
 const grants = allGrants();
 const root = document.getElementById('root');
+const built = new URL('../../', import.meta.url);
 const made = {};
 const makers = {
   dom: async () => (await import('./dom.mjs')).dom(root),
@@ -26,9 +28,10 @@ const makers = {
 const make = (name) => (made[name] ||= makers[name]());
 
 window.app = await boot({
-  program: 'program/',
-  lists: [['assets.list', '/']],
-  packs: [['files.pack', '/']],
+  program: new URL('program/', built).href,
+  packs: [[new URL('files.pack', built).href, '/']],
+  assets: new URL('assets.list', built).href,
+  site: document.baseURI,
   args: query.getAll('arg'),
   grants,
   onEnd: (code, reason) => {
@@ -36,8 +39,9 @@ window.app = await boot({
     window.endReason = reason;
   },
   async modules(wanted) {
-    const modules = {};
-    for (const name of wanted) if (Object.hasOwn(makers, name)) modules[name] = await make(name);
+    const names = [...wanted].filter((name) => Object.hasOwn(makers, name));
+    const given = await Promise.all(names.map(make));
+    const modules = Object.fromEntries(names.map((name, i) => [name, given[i]]));
     window.modules = modules;
     return modules;
   },

package/substrate/host/web/subhost-web.mjs

@@ -1,9 +1,9 @@
-// The host for a web page: runs a program in a browser, with its files kept in memory. A page has every grant and
+// The web host: runs a program in a browser, with its files kept in memory. A web program has every grant and
 // no fakes. Its workers can be given fewer grants and some fakes (worker-script.mjs).
 import { createHost } from '../subhost-core.mjs';
 import { fsModule } from '../fs.mjs';
 import { allGrants, pairsFrom, requestOf } from '../grants.mjs';
-import { pageFiles, resolvePath } from './fs.mjs';
+import { memoryFiles, resolvePath } from './fs.mjs';
 
 const text = new TextDecoder();
 const utf8 = new TextEncoder();
@@ -26,13 +26,13 @@ async function fetchBytes(url) {
   return new Uint8Array(await response.arrayBuffer());
 }
 
-async function fetchListed(url, dir, files) {
-  const list = new URL(url, globalThis.document ? document.baseURI : globalThis.location.href);
-  const paths = text.decode(await fetchBytes(list)).split('\n').filter((p) => p !== '');
-  await Promise.all(paths.map(async (path) => {
-    const from = new URL(path.split('/').map(encodeURIComponent).join('/'), list);
-    files.set(resolvePath(dir, path), await fetchBytes(from));
-  }));
+async function fetchAssets(url) {
+  const urls = new Map();
+  for (const line of text.decode(await fetchBytes(url)).split('\n')) {
+    const tab = line.indexOf('\t');
+    if (tab >= 0) urls.set(resolvePath('/', line.slice(0, tab)), line.slice(tab + 1));
+  }
+  return urls;
 }
 
 function lines(line) {
@@ -92,36 +92,50 @@ export function importedModules(bytes) {
   return names;
 }
 
+async function fetchProgram(base, wasm) {
+  const names = text.decode(await fetchBytes(`${base}program`)).split('\n').filter((n) => n !== '');
+  await Promise.all(names.map(async (name) => {
+    wasm.set(name, await fetchBytes(`${base}${encodeURIComponent(name)}.wasm`));
+  }));
+  return names;
+}
+
+// The compiled modules for sessions: the compiled file lists them, each one is in <hash>.subc, and compiled.map holds
+// their module map.
+async function fetchCompiled(base) {
+  const compiled = new Map();
+  const listed = await fetch(`${base}compiled`);
+  if (!listed.ok) return compiled;
+  const list = new Uint8Array(await listed.arrayBuffer());
+  const hashes = text.decode(list).split('\n').filter((l) => l !== '').map((l) => l.slice(0, l.indexOf(' ')));
+  const fetched = async (name) => compiled.set(name, await fetchBytes(`${base}${name}`));
+  const names = hashes.map((h) => `${h}.subc`);
+  compiled.set('compiled', list);
+  await Promise.all((hashes.length > 0 ? ['compiled.map', ...names] : []).map(fetched));
+  return compiled;
+}
+
 // options.modules is the modules, or a function that takes the names the program imports and gives the modules.
+// options.assets is the URL of the site's assets.list.
 export async function boot(options) {
   const base = options.program.endsWith('/') ? options.program : `${options.program}/`;
   const files = options.files || new Map();
-  for (const [url, dir] of options.lists || []) await fetchListed(url, dir, files);
-  for (const [url, dir] of options.packs || []) unpack(await fetchBytes(url), dir, files);
-
-  const names = text.decode(await fetchBytes(`${base}program`)).split('\n').filter((n) => n !== '');
   const wasm = new Map();
-  await Promise.all(names.map(async (name) => {
-    wasm.set(name, await fetchBytes(`${base}${encodeURIComponent(name)}.wasm`));
-  }));
-  let { modules } = options;
-  if (typeof modules === 'function') {
+  const names = fetchProgram(base, wasm);
+  const ask = options.modules;
+  const modules = typeof ask !== 'function' ? ask : names.then(() => {
     const wanted = new Set();
     for (const bytes of wasm.values()) for (const name of importedModules(bytes)) wanted.add(name);
-    modules = await modules(wanted);
-  }
-  // The compiled modules for sessions: the compiled file lists them, each one is in <hash>.subc, and compiled.map holds
-  // their module map.
-  const compiled = new Map();
-  const listed = await fetch(`${base}compiled`);
-  if (listed.ok) {
-    const list = new Uint8Array(await listed.arrayBuffer());
-    const hashes = text.decode(list).split('\n').filter((l) => l !== '').map((l) => l.slice(0, l.indexOf(' ')));
-    compiled.set('compiled', list);
-    if (hashes.length > 0) compiled.set('compiled.map', await fetchBytes(`${base}compiled.map`));
-    await Promise.all(hashes.map(async (h) => compiled.set(`${h}.subc`, await fetchBytes(`${base}${h}.subc`))));
-  }
-  return run({ ...options, files, modules, compiled }, names, wasm);
+    return ask(wanted);
+  });
+  const [listed, given, compiled, urls] = await Promise.all([
+    names,
+    modules,
+    fetchCompiled(base),
+    options.assets ? fetchAssets(options.assets) : new Map(),
+    ...(options.packs || []).map(async ([url, dir]) => unpack(await fetchBytes(url), dir, files)),
+  ]);
+  return run({ ...options, files, modules: given, compiled, urls }, listed, wasm);
 }
 
 export function run(options, names, wasm) {
@@ -136,9 +150,16 @@ export function run(options, names, wasm) {
   const output = options.output || ((fd, bytes) => write(fd, decoders[fd].decode(bytes, { stream: true })));
 
   const app = { files, cwd, entry: null, ended: false, realTime: [] };
+  // app.url gives the URL to load a file from when it is not in memory.
+  const site = options.site || (globalThis.document ? document.baseURI : globalThis.location.href);
+  const urls = options.urls || new Map();
+  app.url = (name) => {
+    const path = resolvePath(cwd, name);
+    return new URL((urls.get(path) || path.slice(1)).split('/').map(encodeURIComponent).join('/'), site).href;
+  };
   const modules = new Map();
   for (const [name, make] of Object.entries(options.modules || {})) modules.set(name, (caller) => make(caller, app));
-  if (!modules.has('fs')) modules.set('fs', fsModule(pageFiles(files, cwd, grants)));
+  if (!modules.has('fs')) modules.set('fs', fsModule(memoryFiles(files, cwd, grants)));
   const report = (message) => output(2, utf8.encode(`error: ${message}\n`));
   const ended = (code, reason) => {
     app.ended = true;

package/substrate/host/web/worker-script.mjs

@@ -1,9 +1,9 @@
-// Runs a worker in a Web Worker. The worker can use its side of the worker module and the page's modules that a
+// Runs a worker in a Web Worker. The worker can use its side of the worker module and the web program's modules that a
 // worker may have, within its grants, with the fakes it was given in place of the modules they fake. Its files are a
-// read-only copy of the page's files.
+// read-only copy of the web program's files.
 import { run } from './subhost-web.mjs';
 import { fsModule } from '../fs.mjs';
-import { pageFiles, resolvePath } from './fs.mjs';
+import { memoryFiles, resolvePath } from './fs.mjs';
 import { workerSide } from '../worker-side.mjs';
 
 const loaders = {
@@ -51,7 +51,7 @@ self.onmessage = async (e) => {
   if (m.type === 'start') {
     if (m.heard) side.limitOutput();
     const wasm = new Map(m.modules);
-    const modules = { worker: side.worker, fs: fsModule(readOnly(pageFiles(m.files, m.cwd, m.grants))) };
+    const modules = { worker: side.worker, fs: fsModule(readOnly(memoryFiles(m.files, m.cwd, m.grants))) };
     const absent = new Set();
     const fakes = await fakesOf(m);
     for (const name of m.names) {

package/substrate/host/web/worker.mjs

@@ -1,12 +1,12 @@
-// The host module "worker" in a browser. Each worker runs in a Web Worker, which gets a copy of the page's files and
-// the fakes the worker's grants ask for.
+// The host module "worker" in a browser. Each worker runs in a Web Worker, which gets a copy of the web program's files
+// and the fakes the worker's grants ask for.
 import { workerModule } from '../worker.mjs';
 
 export function workers() {
   return workerModule({
     spawn(plan, app) {
-      if (plan.tls) throw new Error('worker.start: a page\'s workers cannot have TLS files');
-      if (plan.ends) throw new Error('worker.start: a page\'s workers do not end on their own');
+      if (plan.tls) throw new Error('worker.start: a web program\'s workers cannot have TLS files');
+      if (plan.ends) throw new Error('worker.start: a web program\'s workers do not end on their own');
       const thread = new Worker(new URL('./worker-script.mjs', import.meta.url), { type: 'module' });
       thread.postMessage({
         type: 'start',

package/substrate/host/worker.mjs

@@ -86,7 +86,7 @@ export function workerModule({ spawn }) {
 
   return (theCaller, app) => {
     caller = theCaller;
-    // What workers have been sent and not yet started on, which browser tests wait for as they settle a page.
+    // What workers have been sent and not yet started on, which browser tests wait for as they settle a document.
     if (app && app.realTime) {
       app.realTime.push({ pending: () => [...alive.values()].reduce((n, x) => n + x.owed, 0), progress: () => started });
     }