toiljs 0.0.59 → 0.0.60

package/src/cli/create.ts

@@ -40,6 +40,38 @@ import { isPackageManager, isValidName, resolveProjectDir } from './validate.js'
 
 export type Template = 'app' | 'minimal';
 
+/**
+ * The `server/migrations/` README, scaffolded by `create` and ensured by `doctor`/`update`. ToilDB
+ * migrations are enforced by convention: a `@migrate` function MUST live in a `*.migration.ts` file
+ * under a `migrations/` folder (the toilscript compiler raises a compile error otherwise), and the
+ * build auto-discovers every such file. Keeping the convention documented next to the folder means a
+ * fresh project ships with a place to evolve `@data` value types from day one.
+ */
+export const MIGRATIONS_README =
+    '# migrations/\n\n' +
+    'ToilDB schema migrations. One file per evolving `@data` value type, named `<Type>.migration.ts`\n' +
+    '(e.g. `User.migration.ts`).\n\n' +
+    'Each file keeps the OLD `@data` shapes (e.g. `UserV1`) alongside the `@migrate` transform(s) that\n' +
+    'carry old records forward, and imports the CURRENT value type from the app. The build\n' +
+    'auto-discovers every `*.migration.ts` under the project.\n\n' +
+    'Do NOT put `@migrate` anywhere else: a `@migrate` outside a `*.migration.ts` file in a\n' +
+    '`migrations/` folder is a compile error.\n\n' +
+    '## Example\n\n' +
+    '`User.migration.ts`:\n\n' +
+    '```ts\n' +
+    "import { User } from '../models/User';\n\n" +
+    '// The previous on-disk shape of a User record.\n' +
+    '@data\n' +
+    'export class UserV1 {\n' +
+    "    name: string = '';\n" +
+    '}\n\n' +
+    '// Carry a UserV1 forward into the current User. Runs once per record on read.\n' +
+    '@migrate\n' +
+    'export function up(old: UserV1, into: User): void {\n' +
+    '    into.name = old.name;\n' +
+    '}\n' +
+    '```\n';
+
 /** Human label for each preprocessor in the styling picker. */
 const PREPROCESSOR_LABEL: Record<Preprocessor, string> = {
     css: 'Plain CSS',
@@ -114,7 +146,7 @@ function scaffold(
         '@types/react-dom': '^19.2.3',
         eslint: '^10.2.0',
         prettier: '^3.8.1',
-        toilscript: '^0.1.36',
+        toilscript: '^0.1.37',
         typescript: '^6.0.3',
     };
     for (const dep of requiredPackages(features).sort()) {
@@ -259,6 +291,10 @@ function scaffold(
             '    npm run build',
             '',
         ].join('\n'),
+        // ToilDB migrations live under server/migrations/ (folder + `*.migration.ts` is enforced by
+        // the compiler; the build auto-discovers them). Ship the folder, documented, with no live
+        // *.migration.ts: a @migrate referencing types a fresh project lacks would break the build.
+        'server/migrations/README.md': MIGRATIONS_README,
     };
 
     // The `app` template's client UI + server are copied from examples/basic at runtime; `minimal`
@@ -377,6 +413,7 @@ function minimalServer(): Record<string, string> {
             '| `main.ts` | The entry point: wires the handler and imports the surface modules. |\n' +
             '| `core/` | The request handler and shared app logic (state, helpers). |\n' +
             '| `models/` | `@data` classes, the typed wire model shared by HTTP and RPC. One type per file. |\n' +
+            '| `migrations/` | ToilDB schema migrations: a `<Type>.migration.ts` per evolving `@data` value type, holding the old shapes + the `@migrate` transform. |\n' +
             '| `routes/` | `@rest` controllers (HTTP). One controller per file, named after its class. |\n' +
             '| `services/` | `@service` classes and free `@remote` functions (typed RPC). |\n' +
             '| `scheduled/` | Reserved for scheduled tasks (not shipped yet). |\n\n' +

package/src/cli/db.ts

@@ -0,0 +1,158 @@
+/**
+ * `toiljs db <action>` — manage the dev server's on-disk ToilDB data.
+ *
+ * Under `toiljs dev`, the in-process ToilDB emulator persists every family
+ * (records / views / unique / events / membership / counter / capacity) plus each
+ * row's schema_version to `<root>/.toil/devdata.json` (so data + migrations survive
+ * restarts). These subcommands let you inspect, reset, snapshot, and restore that
+ * store from outside a running server — handy for fixtures, sharing repro data, or
+ * wiping a corrupt dev state. The snapshot is the exact JSON the dev DB writes, so
+ * an exported file imports cleanly (and a hand-crafted one seeds the dev DB).
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+
+import { accent, bold, danger, dim, success, warn } from './ui.js';
+
+export interface DbOptions {
+    /** Project root (where `.toil/` lives); defaults to the current directory. */
+    root?: string;
+}
+
+/** The top-level families a valid snapshot carries (mirrors DbSnapshot). */
+const FAMILIES = ['store', 'views', 'members', 'counters', 'events', 'eventDedup', 'capacity'] as const;
+
+function devdataPath(opts: DbOptions): string {
+    return path.join(path.resolve(opts.root ?? process.cwd()), '.toil', 'devdata.json');
+}
+
+function out(line: string): void {
+    process.stdout.write(line + '\n');
+}
+
+/** Per-family key counts for a snapshot file, or null if unreadable. */
+function familyCounts(file: string): Record<string, number> | null {
+    let snap: Record<string, unknown>;
+    try {
+        snap = JSON.parse(fs.readFileSync(file, 'utf8')) as Record<string, unknown>;
+    } catch {
+        return null;
+    }
+    const counts: Record<string, number> = {};
+    for (const f of FAMILIES) {
+        const fam = snap[f];
+        counts[f] = fam && typeof fam === 'object' ? Object.keys(fam).length : 0;
+    }
+    return counts;
+}
+
+function isSnapshot(v: unknown): boolean {
+    return typeof v === 'object' && v !== null && FAMILIES.every((f) => f in (v as object));
+}
+
+/** Run `toiljs db <action> [fileArg]`. */
+export function runDb(action: string | undefined, fileArg: string | undefined, opts: DbOptions): void {
+    const file = devdataPath(opts);
+    const rel = path.relative(process.cwd(), file) || file;
+
+    switch (action) {
+        case 'reset':
+        case 'purge': {
+            if (!fs.existsSync(file)) {
+                out(dim(`  dev database already empty (${rel} not found)`));
+                return;
+            }
+            fs.rmSync(file);
+            out(success('  ✓ ') + 'dev database reset ' + dim(`(${rel} deleted)`));
+            return;
+        }
+
+        case 'export': {
+            if (!fs.existsSync(file)) {
+                out(warn('  ! ') + `nothing to export (${rel} not found)`);
+                process.exitCode = 1;
+                return;
+            }
+            let pretty: string;
+            try {
+                pretty = JSON.stringify(JSON.parse(fs.readFileSync(file, 'utf8')), null, 2) + '\n';
+            } catch {
+                out(danger('  ✗ ') + `the dev database at ${rel} is unreadable`);
+                process.exitCode = 1;
+                return;
+            }
+            if (fileArg === undefined) {
+                process.stdout.write(pretty); // no banner/prefix -> pipe-friendly
+                return;
+            }
+            fs.writeFileSync(path.resolve(fileArg), pretty);
+            out(success('  ✓ ') + 'exported dev database to ' + dim(fileArg));
+            return;
+        }
+
+        case 'import': {
+            if (fileArg === undefined) {
+                out(danger('  ✗ ') + 'usage: ' + dim('toiljs db import <file>'));
+                process.exitCode = 1;
+                return;
+            }
+            let parsed: unknown;
+            try {
+                parsed = JSON.parse(fs.readFileSync(path.resolve(fileArg), 'utf8'));
+            } catch (e) {
+                out(danger('  ✗ ') + `cannot read/parse ${fileArg}: ${(e as Error).message}`);
+                process.exitCode = 1;
+                return;
+            }
+            if (!isSnapshot(parsed)) {
+                out(danger('  ✗ ') + `${fileArg} is not a toiljs dev database snapshot`);
+                process.exitCode = 1;
+                return;
+            }
+            fs.mkdirSync(path.dirname(file), { recursive: true });
+            fs.writeFileSync(file, JSON.stringify(parsed)); // compact: the dev DB's on-disk form
+            out(success('  ✓ ') + `imported dev database from ${dim(fileArg)} ` + dim(`(${rel})`));
+            return;
+        }
+
+        case 'path':
+            out(file); // bare path, scriptable
+            return;
+
+        case 'status':
+        case 'info': {
+            out(bold('  dev database') + dim(`  ${rel}`));
+            if (!fs.existsSync(file)) {
+                out(dim('  (empty — no data written yet; run the dev server to populate it)'));
+                return;
+            }
+            const counts = familyCounts(file);
+            if (counts === null) {
+                out(danger('  ✗ unreadable snapshot'));
+                process.exitCode = 1;
+                return;
+            }
+            out(dim(`  ${(fs.statSync(file).size / 1024).toFixed(1)} KiB`));
+            const nonEmpty = FAMILIES.filter((f) => counts[f] > 0);
+            if (nonEmpty.length === 0) out(dim('  (no rows)'));
+            else for (const f of nonEmpty) out('  ' + accent(f.padEnd(12)) + String(counts[f]));
+            return;
+        }
+
+        default:
+            out(
+                [
+                    bold('Usage') + '  ' + dim('toiljs db') + ' <action>',
+                    '',
+                    bold('Actions'),
+                    '  ' + accent('status'.padEnd(16)) + dim('show the dev DB path + per-family row counts'),
+                    '  ' + accent('reset'.padEnd(16)) + dim('delete all dev data (alias: purge)'),
+                    '  ' + accent('export [file]'.padEnd(16)) + dim('write a snapshot to <file> (or stdout)'),
+                    '  ' + accent('import <file>'.padEnd(16)) + dim('replace the dev DB with a snapshot'),
+                    '  ' + accent('path'.padEnd(16)) + dim('print the devdata.json path'),
+                ].join('\n'),
+            );
+            if (action !== undefined) process.exitCode = 1; // unknown action
+            return;
+    }
+}

package/src/cli/diagnostics.ts

@@ -521,6 +521,25 @@ export function checkServerTsPlugin(present: boolean): Check {
           };
 }
 
+/**
+ * Whether the server has a `migrations/` folder. ToilDB `@migrate` functions MUST live in a
+ * `*.migration.ts` file under `migrations/` (the toilscript compiler enforces folder + extension as
+ * a compile error), and the build auto-discovers them, so a server project should keep the folder
+ * ready. Older projects predate the convention, so this WARNS (does not fail) and points at the
+ * one-command fix (`toiljs update` creates it).
+ */
+export function checkMigrationsDir(exists: boolean): Check {
+    return exists
+        ? { id: 'migrations-dir', label: 'server/migrations/ directory', status: 'pass' }
+        : {
+              id: 'migrations-dir',
+              label: 'server/migrations/ directory',
+              status: 'warn',
+              detail: 'no server/migrations/ folder; ToilDB @migrate functions must live in a *.migration.ts file under it',
+              fix: 'Create server/migrations/ (one <Type>.migration.ts per evolving @data value type), or run `toiljs update` to add it.',
+          };
+}
+
 // --- Security -------------------------------------------------------------------------------------
 
 /** Whether the project uses the auth primitive, and whether its session secret is configured. */

package/src/cli/doctor.ts

@@ -27,6 +27,7 @@ import {
     checkDir,
     checkDuplicatePatterns,
     type CheckGroup,
+    checkMigrationsDir,
     checkMountSlots,
     checkNode,
     checkPackageManager,
@@ -279,6 +280,24 @@ function serverTsconfigPath(root: string, toilconfig: Record<string, unknown> |
     return null;
 }
 
+/**
+ * Whether a `migrations/` folder exists under any server dir (the dir of a toilconfig entry,
+ * conventionally `server/`). ToilDB `@migrate` functions live in `*.migration.ts` files there, and
+ * the build auto-discovers them; a missing folder is a warn, not a fail (older projects predate it).
+ */
+function serverMigrationsExist(root: string, toilconfig: Record<string, unknown> | null): boolean {
+    const dirs = new Set<string>();
+    const entries = Array.isArray(toilconfig?.entries)
+        ? (toilconfig.entries as unknown[]).filter((e): e is string => typeof e === 'string')
+        : [];
+    for (const e of entries) dirs.add(path.dirname(path.resolve(root, e)));
+    if (dirs.size === 0) dirs.add(path.join(root, 'server'));
+    for (const dir of dirs) {
+        if (fs.existsSync(path.join(dir, 'migrations'))) return true;
+    }
+    return false;
+}
+
 /** Whether a parsed tsconfig's `compilerOptions.plugins` references the toilscript LS plugin. */
 function tsconfigHasToilPlugin(tsconfig: Record<string, unknown> | null): boolean {
     const plugins = asRecord(tsconfig?.compilerOptions)?.plugins;
@@ -840,6 +859,7 @@ export async function runDoctor(opts: DoctorOptions): Promise<void> {
                       checkRestDispatch(restFacts),
                       checkPrettierPlugin(prettierPresent),
                       checkServerTsPlugin(serverTsPluginPresent),
+                      checkMigrationsDir(serverMigrationsExist(root, toilconfig)),
                   ]
                 : [checkToilconfig(false)],
         },

package/src/cli/index.ts

@@ -8,6 +8,7 @@ import { build, dev, start } from 'toiljs/compiler';
 
 import { runConfigure } from './configure.js';
 import { runCreate, type Template } from './create.js';
+import { runDb } from './db.js';
 import { runDoctor } from './doctor.js';
 import { notifyIfOutdated } from './notify.js';
 import { runUpdate } from './update.js';
@@ -132,6 +133,7 @@ function printHelp(): void {
             cmd('start', 'self-host the built app (hyper-express / uWS)'),
             cmd('doctor', 'diagnose project setup and dependencies'),
             cmd('update', 'check for and apply dependency updates'),
+            cmd('db <action>', 'manage dev DB data (status | reset | export | import)'),
             '',
             bold('Options'),
             cmd('--root <dir>', 'project root (default: current directory)'),
@@ -255,6 +257,14 @@ async function main(): Promise<void> {
             });
             break;
 
+        case 'db': {
+            // `toiljs db <action> [file]` — no banner so `path`/`export` pipe cleanly.
+            const action = rest[0];
+            const fileArg = rest[1] !== undefined && !rest[1].startsWith('-') ? rest[1] : undefined;
+            runDb(action, fileArg, { root: flags.root });
+            break;
+        }
+
         case 'help':
         case '--help':
         case '-h':

package/src/cli/update.ts

@@ -9,6 +9,7 @@ import path from 'node:path';
 
 import { cancel, intro, isCancel, multiselect, note, outro, spinner } from '@clack/prompts';
 
+import { MIGRATIONS_README } from './create.js';
 import { capture, run } from './proc.js';
 import { buildRows, type Bump, parseNcuJson, type UpdateRow } from './updates.js';
 import { accent, danger, dim, success, warn } from './ui.js';
@@ -49,6 +50,53 @@ function readDependencies(pkgPath: string): Record<string, string> {
     return { ...merge(pkg.dependencies), ...merge(pkg.devDependencies) };
 }
 
+/**
+ * The server dir(s) of a project: the directories of the toilconfig.json `entries`, conventionally
+ * `server/`. Falls back to `<root>/server` when there is no toilconfig (or it has no string entries),
+ * matching how `doctor` locates the server tree.
+ */
+function serverDirs(root: string): string[] {
+    const dirs = new Set<string>();
+    try {
+        const parsed: unknown = JSON.parse(
+            fs.readFileSync(path.join(root, 'toilconfig.json'), 'utf8'),
+        );
+        const entries =
+            typeof parsed === 'object' &&
+            parsed !== null &&
+            Array.isArray((parsed as { entries?: unknown }).entries)
+                ? (parsed as { entries: unknown[] }).entries.filter(
+                      (e): e is string => typeof e === 'string',
+                  )
+                : [];
+        for (const e of entries) dirs.add(path.dirname(path.resolve(root, e)));
+    } catch {
+        // no/unreadable toilconfig.json: fall back to the conventional server/ below
+    }
+    if (dirs.size === 0) dirs.add(path.join(root, 'server'));
+    return [...dirs];
+}
+
+/**
+ * Ensures the ToilDB `migrations/` folder exists under each server dir that exists. `@migrate`
+ * functions MUST live in a `*.migration.ts` file under `migrations/` (folder + extension is a
+ * compile error otherwise) and the build auto-discovers them, so an up-to-date project keeps the
+ * folder ready. Idempotent: only writes the folder + README where the server dir exists and the
+ * folder is absent. Returns the project-relative paths created (for the note), empty if nothing.
+ */
+function ensureMigrationsDirs(root: string): string[] {
+    const created: string[] = [];
+    for (const dir of serverDirs(root)) {
+        if (!fs.existsSync(dir)) continue;
+        const migrations = path.join(dir, 'migrations');
+        if (fs.existsSync(migrations)) continue;
+        fs.mkdirSync(migrations, { recursive: true });
+        fs.writeFileSync(path.join(migrations, 'README.md'), MIGRATIONS_README);
+        created.push(path.relative(root, migrations) || 'migrations');
+    }
+    return created;
+}
+
 function bumpColor(bump: Bump, text: string): string {
     if (bump === 'major') return danger(text);
     if (bump === 'minor') return warn(text);
@@ -85,6 +133,16 @@ export async function runUpdate(opts: UpdateOptions): Promise<void> {
 
     intro(accent('toiljs update'));
 
+    // Bring the project's structure up to date: ensure the ToilDB migrations folder exists (it is
+    // newer than some projects, and the compiler requires @migrate functions to live under it).
+    const createdMigrations = ensureMigrationsDirs(root);
+    if (createdMigrations.length > 0) {
+        note(
+            createdMigrations.map((p) => `${dim('+')} ${p}/`).join('\n'),
+            'Created ToilDB migrations folder',
+        );
+    }
+
     const s = spinner();
     s.start('Checking the registry for updates');
     const res = await capture('npx', ncuArgs(['--jsonUpgraded']), root);

package/src/devserver/db/catalog.ts

@@ -0,0 +1,100 @@
+/**
+ * Parse a compiled server wasm's `toildb.catalog` custom section into a map of
+ * `"<Db>/<collection>"` (the resolve key the guest passes to
+ * `data.resolve_collection`) -> the collection's CURRENT `schema_version`.
+ *
+ * The dev DB uses this to STAMP each write with the value type's current schema
+ * version. When the developer evolves a `@data` type and rebuilds, the catalog
+ * version changes; data already on disk keeps its OLD stamp, so a read surfaces
+ * that old version and the guest's woven `decodeInto` runs the `@migrate` - the
+ * dev-side equivalent of the edge binding the cap's schema_version into the row.
+ *
+ * Wire format mirrors `toildb::catalog` / the backend `db_catalog` decoder and the
+ * compiler's `buildToilDbCatalog` emitter (all little-endian).
+ */
+
+/** Read a LEB128 from `buf` at `pos`; throws on overrun (the section is a
+ *  tenant-built, possibly mid-rebuild wasm, so it must never over-read). */
+import { DataReader } from 'toiljs/io';
+
+function leb(buf: Buffer, pos: number): [number, number] {
+    let result = 0;
+    let shift = 0;
+    let p = pos;
+    for (;;) {
+        if (p >= buf.length) throw new RangeError('leb128 past end of buffer');
+        const b = buf[p++];
+        result |= (b & 0x7f) << shift;
+        if ((b & 0x80) === 0) break;
+        shift += 7;
+        if (shift > 35) throw new RangeError('leb128 too long');
+    }
+    return [result >>> 0, p];
+}
+
+/** The bytes of the named wasm custom section, or null if absent. Bounds-checked
+ *  so a truncated/garbage module can never read past the buffer. */
+function customSection(wasm: Buffer, want: string): Buffer | null {
+    let pos = 8; // skip the 8-byte magic + version header
+    while (pos < wasm.length) {
+        const id = wasm[pos++];
+        let size: number;
+        [size, pos] = leb(wasm, pos);
+        const end = pos + size;
+        if (end > wasm.length || end < pos) return null; // truncated section table
+        if (id === 0) {
+            let nameLen: number;
+            let namePos: number;
+            [nameLen, namePos] = leb(wasm, pos);
+            if (namePos + nameLen <= end && wasm.toString('latin1', namePos, namePos + nameLen) === want)
+                return wasm.subarray(namePos + nameLen, end);
+        }
+        pos = end;
+    }
+    return null;
+}
+
+/** `"<Db>/<collection>"` -> current `schema_version` for every collection. Decoded
+ *  with the shared bounds-checked {@link DataReader} (it returns 0/empty + flips
+ *  `.ok` past the end), so a truncated/garbage section - e.g. a mid-rebuild wasm -
+ *  yields only the collections that decoded cleanly and never over-reads. Mirrors
+ *  the compiler's `buildToilDbCatalog` emitter + the backend `db_catalog` decoder. */
+export function parseCatalog(wasm: Buffer): Map<string, number> {
+    const out = new Map<string, number>();
+    let sec: Buffer | null;
+    try {
+        sec = customSection(wasm, 'toildb.catalog');
+    } catch {
+        return out; // garbage section table (mid-rebuild) -> no catalog
+    }
+    if (sec === null) return out;
+
+    const r = new DataReader(sec);
+    r.readU16(); // catalog format version
+    const ndb = r.readU16();
+    for (let d = 0; d < ndb && r.ok; d++) {
+        const db = r.readString();
+        const nc = r.readU16();
+        for (let c = 0; c < nc && r.ok; c++) {
+            const name = r.readString();
+            r.readU8(); // family
+            r.readString(); // keyType
+            r.readString(); // valueType
+            r.readU32(); // valueDataId
+            const schemaVersion = r.readU32();
+            r.readU32(); // generation
+            r.readU8(); // replication (emitter order: replication then placement)
+            r.readU8(); // placement
+            const nFields = r.readU16();
+            for (let f = 0; f < nFields; f++) {
+                r.readString(); // field name
+                r.readString(); // field type
+                r.readU8(); // isArray
+            }
+            const nMig = r.readU16();
+            for (let m = 0; m < nMig; m++) r.readU32(); // migratableFrom versions
+            if (r.ok) out.set(db + '/' + name, schemaVersion >>> 0);
+        }
+    }
+    return out;
+}