rooklang 0.3.1 → 0.3.2

package/dist/walk.js

@@ -103,6 +103,14 @@ export function forEachChild(node, visit) {
             visit(node.element);
             optional(node.size);
             break;
+        case 'StreamChanExpression':
+            visit(node.stream);
+            optional(node.size);
+            break;
+        case 'QueueExpression':
+            visit(node.element);
+            optional(node.items);
+            break;
         case 'MakeExpression':
             visit(node.type);
             visit(node.length);
@@ -256,6 +264,7 @@ export function forEachChild(node, visit) {
         case 'TypeParameter':
         case 'ArrayType':
         case 'ChanType':
+        case 'QueueType':
         case 'NullableType':
         case 'FuncType':
         case 'ComponentType':

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rooklang",
-  "version": "0.3.1",
+  "version": "0.3.2",
   "description": "A language with JavaScript semantics, Go-style syntax, goroutines on real threads and static types that compiles to readable JavaScript",
   "keywords": [
     "rooklang",
@@ -47,10 +47,12 @@
       "types": "./dist/browser.d.ts",
       "default": "./dist/browser.js"
     },
+    "./runtime": "./src/runtime/index.js",
     "./package.json": "./package.json"
   },
   "files": [
-    "dist"
+    "dist",
+    "src/runtime/*.js"
   ],
   "scripts": {
     "rook": "node src/cli.ts",
@@ -59,7 +61,7 @@
     "site": "npm run site:build && node src/cli.ts run site/server.rk",
     "site:images": "node scripts/og-image.ts",
     "vscode:build": "node scripts/build-vscode.ts",
-    "vscode:package": "npm run vscode:build && cd editors/vscode && npx --yes @vscode/vsce@4 package --no-dependencies --allow-missing-repository --skip-license --out rooklang.vsix",
+    "vscode:package": "npm run vscode:build && cd editors/vscode && npx --yes @vscode/vsce@4 package --no-dependencies --baseContentUrl https://github.com/QWERN-9876r/RookLang/blob/HEAD/editors/vscode --baseImagesUrl https://github.com/QWERN-9876r/RookLang/raw/HEAD/editors/vscode --out rooklang.vsix",
     "vscode:install": "npm run vscode:package && code --install-extension editors/vscode/rooklang.vsix --force",
     "build": "node -e \"fs.rmSync('dist', { recursive: true, force: true })\" && tsc -p tsconfig.build.json",
     "prepack": "npm run build",
@@ -68,7 +70,7 @@
     "test": "vitest run",
     "test:watch": "vitest",
     "lint": "eslint .",
-    "format": "prettier --write . && eslint --fix --fix-type layout .",
+    "format": "prettier --write . && eslint --fix --fix-type layout . && node scripts/runtime-sources.ts",
     "format:check": "prettier --check .",
     "check": "npm run typecheck && npm run lint && npm run format:check && npm test",
     "prepare": "husky"

package/src/runtime/chan.js

@@ -0,0 +1,277 @@
+// Channels: sending, receiving and closing, here or in the thread that made the channel
+// (threads.js), and the queues of a channel. Receiving from a closed channel gives
+// the zero value of its type, as in Go.
+
+import { $$Queue } from './queue.js';
+import { $$commit, $$lock, $$select, $$try, $$unlock, $$unwait } from './select.js';
+import { $$adopt, $$ask, $$ownerOf, $$post, $$registry, $$stopped } from './threads.js';
+
+$$registry().handle ??= { message: $$message, pump: $$pump };
+
+export function $$chan(size, zero) {
+  const registry = $$registry();
+  const id = `${registry.key}:${registry.channels++}`;
+  const channel = { $$chan: id, size, zero };
+
+  registry.states.set(id, {
+    size,
+    zero,
+    buffer: new $$Queue(),
+    receivers: new $$Queue(),
+    senders: new $$Queue(),
+    closed: false,
+    holders: 1,
+    busy: false,
+  });
+  $$adopt(channel);
+
+  return channel;
+}
+
+// A local operation waits in the queue of its channel; one of another thread asks the thread
+// that made the channel.
+export function $$send(channel, value) {
+  const id = channel.$$chan;
+  const owner = $$ownerOf(id);
+
+  if (owner !== $$registry().key) return $$ask(owner, { op: 'send', channel: id, value });
+
+  return new Promise((resolve, reject) =>
+    $$wait(id, 'senders', { value, done: resolve, fail: reject }),
+  );
+}
+
+export function $$receive(channel, withOk = false) {
+  const id = channel.$$chan;
+  const owner = $$ownerOf(id);
+  const received =
+    owner !== $$registry().key
+      ? $$ask(owner, { op: 'receive', channel: id })
+      : new Promise((resolve, reject) => $$wait(id, 'receivers', { done: resolve, fail: reject }));
+
+  return withOk ? received : received.then(([value]) => value);
+}
+
+// Closing a channel of another thread fails this one if it was closed already, as a panic would.
+export function $$close(channel) {
+  const id = channel.$$chan;
+  const owner = $$ownerOf(id);
+
+  if (owner === $$registry().key) return $$closeHere(id);
+  $$ask(owner, { op: 'close', channel: id }).catch((error) => {
+    setTimeout(() => {
+      throw error;
+    });
+  });
+}
+
+export function $$closeHere(id) {
+  const state = $$registry().states.get(id);
+
+  if (state.closed) throw new Error('close of a closed channel');
+  state.closed = true;
+  $$pump(id);
+}
+
+// What a closed channel gives: a value of its own for each receiver, if it is an array or a queue.
+export function $$zero({ zero }) {
+  return Array.isArray(zero) ? [] : zero instanceof $$Queue ? new $$Queue() : zero;
+}
+
+export async function* $$range(channel) {
+  while (true) {
+    const [value, ok] = await $$receive(channel, true);
+
+    if (!ok) return;
+    yield value;
+  }
+}
+
+export function $$wait(id, queue, entry) {
+  const state = $$registry().states.get(id);
+
+  if (!state) return entry.fail?.($$stopped());
+  state[queue].push(entry);
+  $$pump(id);
+}
+
+// The operations of other threads on the channels of this one.
+export function $$message(from, data) {
+  const answer = (value) => $$post(from, { answer: data.ask, value });
+  const fail = (error) => $$post(from, { answer: data.ask, error });
+  const select = { key: from, sid: data.sid };
+
+  switch (data.op) {
+    case 'send':
+      return $$wait(data.channel, 'senders', { from, value: data.value, done: answer, fail });
+
+    case 'receive':
+      return $$wait(data.channel, 'receivers', { from, done: answer, fail });
+
+    case 'close':
+      try {
+        $$closeHere(data.channel);
+
+        return answer();
+      } catch (error) {
+        return fail(error);
+      }
+
+    case 'try':
+      try {
+        return answer($$try(data.channel, data.send, data.value));
+      } catch (error) {
+        return fail(error);
+      }
+
+    case 'select':
+      return $$select(data.cases, data.hasDefault).then(answer, fail);
+
+    case 'wait':
+      return $$wait(data.channel, data.send ? 'senders' : 'receivers', {
+        select,
+        case: data.case,
+        value: data.value,
+      });
+
+    case 'cancel':
+      return $$unwait(select, data.channels);
+
+    case 'lock':
+      return answer($$lock(data.sid));
+
+    case 'unlock':
+      return $$unlock(data.sid);
+
+    case 'commit':
+      return $$commit(data.sid, data.case, data.result, data.error);
+  }
+}
+
+// Matches what waits in a channel: a receiver with a value of the buffer, a sender or the zero value
+// after close; a sender with room in the buffer. A case of a select goes on only if its select
+// has not gone on with another case, which the thread of the select tells.
+export function $$pump(id) {
+  const state = $$registry().states.get(id);
+
+  while (state && !state.busy) {
+    const receiver = state.receivers.at(0);
+    const sender = state.senders.at(0);
+
+    if (receiver && (state.buffer.length > 0 || state.closed)) {
+      if (!$$pair(id, state, receiver, null)) return;
+    } else if (receiver && sender) {
+      if (!$$pair(id, state, receiver, sender)) return;
+    } else if (sender && (state.closed || state.buffer.length < state.size)) {
+      if (!$$pair(id, state, null, sender)) return;
+    } else {
+      return;
+    }
+  }
+}
+
+// Whether the pump can go on at once: a pair whose selects are all in this thread is decided here.
+export function $$pair(id, state, receiver, sender) {
+  const entries = [receiver, sender].filter((entry) => entry !== null);
+  const key = $$registry().key;
+
+  if (entries.some((entry) => entry.select && entry.select.key !== key)) {
+    $$pairLater(id, state, receiver, sender);
+
+    return false;
+  }
+
+  const locks = entries.map((entry) => (entry.select ? $$lock(entry.select.sid) : 'locked'));
+
+  if (locks.every((lock) => lock === 'locked')) {
+    $$complete(state, receiver, sender);
+
+    return true;
+  }
+
+  entries.forEach((entry, i) => locks[i] === 'locked' && $$unlockAt(entry.select));
+
+  return $$settle(id, state, entries, locks);
+}
+
+// A pair with a select of another thread: the channel waits for the answers of that thread.
+export async function $$pairLater(id, state, receiver, sender) {
+  state.busy = true;
+
+  const entries = [receiver, sender].filter((entry) => entry !== null);
+  const locks = [];
+
+  for (const { select } of entries) {
+    const lock = !select
+      ? 'locked'
+      : select.key === $$registry().key
+        ? $$lock(select.sid)
+        : await $$ask(select.key, { op: 'lock', sid: select.sid }).catch(() => 'done');
+
+    locks.push(lock);
+    if (lock !== 'locked') break;
+  }
+
+  state.busy = false;
+  if (locks.length === entries.length && locks.every((lock) => lock === 'locked')) {
+    $$complete(state, receiver, sender);
+  } else {
+    entries.forEach((entry, i) => locks[i] === 'locked' && $$unlockAt(entry.select));
+    if (!$$settle(id, state, entries, locks)) return;
+  }
+
+  $$pump(id);
+}
+
+// After a pair that could not go on: a select that has gone on leaves the queue; one that another
+// channel is deciding is tried again a little later.
+export function $$settle(id, state, entries, locks) {
+  entries.forEach((entry, i) => {
+    if (locks[i] === 'done') $$remove(state, entry);
+  });
+  if (locks.includes('done')) return true;
+  setTimeout(() => $$pump(id), 1 + Math.random() * 4);
+
+  return false;
+}
+
+export function $$remove(state, entry) {
+  state.receivers.$$delete(entry);
+  state.senders.$$delete(entry);
+}
+
+// Takes the pair off the queues and passes the value.
+export function $$complete(state, receiver, sender) {
+  if (receiver) state.receivers.$$delete(receiver);
+  if (sender) state.senders.$$delete(sender);
+  if (receiver) {
+    const result =
+      state.buffer.length > 0
+        ? [state.buffer.shift(), true]
+        : sender
+          ? [sender.value, true]
+          : [$$zero(state), false];
+
+    $$finish(receiver, result);
+    if (sender) $$finish(sender);
+  } else if (state.closed) {
+    $$finish(sender, undefined, new Error('send on a closed channel'));
+  } else {
+    state.buffer.push(sender.value);
+    $$finish(sender);
+  }
+}
+
+export function $$finish(entry, result, error) {
+  const { select } = entry;
+
+  if (!select) return error ? entry.fail(error) : entry.done(result);
+  if (select.key === $$registry().key) return $$commit(select.sid, entry.case, result, error);
+  $$post(select.key, { op: 'commit', sid: select.sid, case: entry.case, result, error });
+}
+
+export function $$unlockAt(select) {
+  if (!select) return;
+  if (select.key === $$registry().key) $$unlock(select.sid);
+  else $$post(select.key, { op: 'unlock', sid: select.sid });
+}

package/src/runtime/core.js

@@ -0,0 +1,23 @@
+// What the code of any module may need: `for` over a value of unknown type, and `defer`.
+
+// `for v in x` over an untyped value: a Map gives its values, as a typed one does.
+export function $$values(iterable) {
+  if (typeof iterable === 'function') return iterable();
+
+  return iterable instanceof Map ? iterable.values() : iterable;
+}
+
+// Runs deferred calls in reverse order; every call runs even if an earlier one throws.
+export function $$runDeferred(deferred) {
+  let failure;
+
+  for (let i = deferred.length - 1; i >= 0; i--) {
+    try {
+      deferred[i]();
+    } catch (error) {
+      failure = { error };
+    }
+  }
+
+  if (failure) throw failure.error;
+}

package/src/runtime/go.js

@@ -0,0 +1,334 @@
+// `go`: the pool of workers of the program, and the side of the worker. They work
+// with Web Workers and with the worker threads of Node, whose API differs: `on("message")` instead
+// of `onmessage`, and `unref()`, so that idle workers do not keep the program running.
+
+import {
+  $$adopt,
+  $$connect,
+  $$copy,
+  $$gone,
+  $$incoming,
+  $$post,
+  $$registry,
+  $$transfer,
+} from './threads.js';
+
+// `go` in a module: a call in a worker of the pool. `start` starts a worker of the module, with its
+// worker module: it is $$startWorker of the module, whose URL a bundler needs to see there.
+export function $$goWorker(start, name, args) {
+  return (globalThis[Symbol.for('rook.goroutines')] ??= $$pool()).call(
+    start,
+    name,
+    $$copy(args, true),
+  );
+}
+
+// The workers of the program, shared by its modules: up to one per processor. Each runs the worker
+// module of one module, `start`. Calls wait in one queue and go to a worker of their module that is
+// free: one whose thread is not running code, though its earlier calls may wait in `await`. A
+// worker that is still busy keeps its calls for later; an idle worker of another module makes room
+// for a new one, unless it keeps channels.
+export function $$pool() {
+  const size = globalThis.navigator?.hardwareConcurrency ?? 4;
+  const registry = $$registry();
+  const workers = [];
+  const queue = [];
+  const calls = new Map();
+  let ids = 0;
+  // A worker without calls lets the program end, unless something here waits for it.
+  const unref = (worker) => {
+    if (worker.calls === 0 && !(registry.pins.get(worker.key) > 0)) worker.thread.unref?.();
+  };
+
+  registry.idle = (key) => {
+    const worker = workers.find((each) => each.key === key);
+
+    if (worker) unref(worker);
+  };
+
+  const send = (worker, call) => {
+    worker.busy = true;
+    worker.calls++;
+    worker.thread.ref?.();
+    call.worker = worker;
+    calls.set(call.id, call);
+    $$post(worker.key, { call: call.id, name: call.name, args: call.args });
+    call.args = null;
+  };
+  const dispatch = () => {
+    const starting = new Map();
+
+    for (const worker of workers) {
+      if (!worker.key) starting.set(worker.start, (starting.get(worker.start) ?? 0) + 1);
+    }
+
+    for (let i = 0; i < queue.length; i++) {
+      const call = queue[i];
+      let free = null;
+
+      for (const worker of workers) {
+        if (worker.start !== call.start || !worker.key || worker.busy) continue;
+        if (!free || worker.calls < free.calls) free = worker;
+      }
+
+      // A free worker without calls; else a new worker; else one whose calls wait in await.
+      if (free?.calls === 0 || (free && workers.length >= size)) {
+        queue.splice(i--, 1);
+        send(free, call);
+      } else if (starting.get(call.start) > 0) {
+        starting.set(call.start, starting.get(call.start) - 1);
+      } else if (workers.length < size || retire(call.start)) {
+        add(call.start);
+      } else {
+        // It waits for a busy worker of its module, which may have let go of the program when
+        // its last call returned: the program runs on until that worker is free and takes it.
+        for (const worker of workers)
+          if (worker.start === call.start && worker.key) worker.thread.ref?.();
+      }
+    }
+  };
+  const retire = (start) => {
+    const idle = workers.find(
+      (worker) =>
+        worker.key && worker.start !== start && !worker.busy && worker.calls === 0 && !worker.owns,
+    );
+
+    if (!idle) return false;
+    idle.thread.terminate();
+    remove(idle, null);
+
+    return true;
+  };
+  // A worker that stopped: its calls fail, and the threads that talked to it forget it.
+  const remove = (worker, error) => {
+    workers.splice(workers.indexOf(worker), 1);
+    for (const [id, call] of calls) {
+      if (call.worker !== worker) continue;
+      calls.delete(id);
+      call.reject(error);
+    }
+
+    if (worker.key) {
+      $$gone(worker.key);
+      for (const other of workers) {
+        if (other.key) other.thread.postMessage({ gone: worker.key });
+      }
+    }
+  };
+  const fail = (worker, error) => {
+    remove(worker, error);
+    dispatch();
+  };
+  const receive = (worker, data) => {
+    if ('op' in data || 'answer' in data) return $$incoming(worker.key, data);
+    if ('hello' in data) {
+      worker.key = data.hello;
+      $$connect(worker.key, worker.thread, false);
+      worker.thread.postMessage({ init: registry.key });
+      // Workers talk to each other through ports of their own, also while this thread is busy.
+      for (const other of workers) {
+        if (other === worker || !other.key) continue;
+
+        const { port1, port2 } = new MessageChannel();
+
+        worker.thread.postMessage({ port: other.key, through: port1 }, [port1]);
+        other.thread.postMessage({ port: worker.key, through: port2 }, [port2]);
+      }
+
+      worker.busy = false;
+      unref(worker);
+    } else if ('free' in data) {
+      worker.busy = false;
+      worker.owns = data.owns;
+    } else if ('returned' in data) {
+      $$adopt(data);
+
+      const call = calls.get(data.returned);
+
+      calls.delete(data.returned);
+      worker.owns = data.owns;
+      worker.calls--;
+      unref(worker);
+      if ('error' in data) call?.reject(data.error);
+      else call?.resolve(data.value);
+    }
+
+    dispatch();
+    // A worker that a waiting call held on to, now free and without a call, lets go again.
+    if ('free' in data) unref(worker);
+  };
+  const add = (start) => {
+    const worker = { start, key: null, thread: null, busy: true, calls: 0, owns: false };
+
+    workers.push(worker);
+    start().then(
+      (thread) => {
+        worker.thread = thread;
+        if (thread.on) {
+          thread.on('message', (data) => receive(worker, data));
+          thread.on('error', (error) => fail(worker, error));
+        } else {
+          thread.onmessage = (event) => receive(worker, event.data);
+          thread.onerror = (event) => fail(worker, new Error(event.message ?? 'a worker failed'));
+        }
+      },
+      (error) => {
+        remove(worker, error);
+        for (const call of queue.filter((each) => each.start === start)) {
+          queue.splice(queue.indexOf(call), 1);
+          call.reject(error);
+        }
+      },
+    );
+  };
+  const tasks = new WeakMap();
+
+  return {
+    call: (start, name, args) => {
+      const call = { id: ids++, start, name, args, resolve: null, reject: null, worker: null };
+      const task = new Promise((resolve, reject) => Object.assign(call, { resolve, reject }));
+
+      tasks.set(task, call);
+      queue.push(call);
+      dispatch();
+
+      return task;
+    },
+    // A call that waits leaves the queue; a worker cannot stop a call, so the worker stops, and
+    // the other calls in it fail too.
+    cancel: (task) => {
+      const call = tasks.get(task);
+
+      tasks.delete(task);
+      if (!call || (!queue.includes(call) && !calls.has(call.id))) return;
+      task.catch(() => {});
+      if (queue.includes(call)) {
+        queue.splice(queue.indexOf(call), 1);
+
+        return call.reject(new Error('goroutine cancelled'));
+      }
+
+      calls.delete(call.id);
+      call.reject(new Error('goroutine cancelled'));
+      call.worker.thread.terminate();
+      remove(
+        call.worker,
+        new Error('the worker of the goroutine stopped: another goroutine in it was cancelled'),
+      );
+      dispatch();
+    },
+  };
+}
+
+// `go` inside a goroutine: the function runs in the same worker, with copies of the arguments. A
+// name is one of the `functions` of the worker module, a function literal is itself.
+export function $$goHere(functions, fn, args) {
+  const copies = $$copy(args);
+  const run = typeof fn === 'function' ? fn : functions[fn];
+  let cancel;
+  const cancelled = new Promise((_, reject) => (cancel = reject));
+  const task = Promise.race([cancelled, Promise.resolve().then(() => cancel && run(...copies))]);
+
+  $$registry().cancels.set(task, () => {
+    task.catch(() => {});
+    cancel(new Error('goroutine cancelled'));
+    cancel = null;
+  });
+
+  return task;
+}
+
+// `go for`: the array is split into parts, one per processor, each part runs as a goroutine of `go`
+// of the module, and
+// the loop waits for all of them. A typed array on a SharedArrayBuffer is shared with the workers,
+// which write into it; other parts are copies, moved to the workers and back into the array.
+export async function $$goFor(go, name, items, ...captured) {
+  const count = Math.min(globalThis.navigator?.hardwareConcurrency ?? 4, items.length);
+  const shared =
+    typeof SharedArrayBuffer === 'function' && items.buffer instanceof SharedArrayBuffer;
+  const parts = [];
+
+  for (let k = 0; k < count; k++) {
+    const from = Math.floor((items.length * k) / count);
+    const to = Math.floor((items.length * (k + 1)) / count);
+    const part = shared ? items.subarray(from, to) : items.slice(from, to);
+
+    if (ArrayBuffer.isView(part) && !shared) $$transfer(part);
+    parts.push(
+      go(name, [part, from, ...captured]).then((written) => {
+        if (written === undefined || shared) return;
+        if (ArrayBuffer.isView(items)) items.set(written, from);
+        else for (let i = 0; i < written.length; i++) items[from + i] = written[i];
+      }),
+    );
+  }
+
+  await Promise.all(parts);
+}
+
+// `make([]T, n, shared)`: the memory of a typed array that the workers of `go for` write into,
+// without copies. A page has SharedArrayBuffer only with cross-origin isolation.
+export function $$shared(bytes) {
+  if (typeof SharedArrayBuffer !== 'function') {
+    throw new Error(
+      'make(..., shared) needs SharedArrayBuffer, which a page has only with cross-origin isolation: the headers Cross-Origin-Opener-Policy: same-origin and Cross-Origin-Embedder-Policy: require-corp',
+    );
+  }
+
+  return new SharedArrayBuffer(bytes);
+}
+
+// The worker: runs the calls that the pool sends and sends back their results or errors. Once the
+// thread is free again, it says so, and the pool may send another call.
+export async function $$serve(functions) {
+  const threads = 'node:worker_threads';
+  const port =
+    typeof postMessage === 'function'
+      ? globalThis
+      : (await import(/* @vite-ignore */ threads)).parentPort;
+  const registry = $$registry();
+
+  // The port to the main thread keeps the worker running.
+  registry.idle = (key) => key !== registry.main && registry.ports.get(key)?.unref?.();
+  port.postMessage({ hello: registry.key });
+
+  const free = () => port.postMessage({ free: true, owns: registry.states.size > 0 });
+  const receive = async (data) => {
+    if ('op' in data || 'answer' in data) return $$incoming(registry.main, data);
+    if ('init' in data) {
+      registry.main = data.init;
+
+      return $$connect(data.init, port, false);
+    }
+
+    if ('port' in data) return $$connect(data.port, data.through, true);
+    if ('gone' in data) return $$gone(data.gone);
+    setTimeout(free);
+    $$adopt(data.args);
+
+    const { main } = registry;
+
+    try {
+      const value = await functions[data.name](...data.args);
+
+      $$post(main, { returned: data.call, value, owns: registry.states.size > 0 });
+    } catch (error) {
+      try {
+        $$post(main, { returned: data.call, error, owns: registry.states.size > 0 });
+      } catch {
+        $$post(main, { returned: data.call, error: new Error(String(error)), owns: true });
+      }
+    }
+  };
+
+  if (port.on) port.on('message', receive);
+  else port.onmessage = (event) => receive(event.data);
+}
+
+// `cancel(task)`: stops a goroutine of `go`, whose Promise fails with "goroutine cancelled". A call
+// in a worker stops with its worker; inside a goroutine, `go` runs in the same worker, so its
+// Promise fails but code that has started goes on.
+export function $$cancel(task) {
+  globalThis[Symbol.for('rook.goroutines')]?.cancel(task);
+  $$registry().cancels.get(task)?.();
+}