toiljs 0.0.116 → 0.0.117

package/src/devserver/db/derives.ts

@@ -1,121 +0,0 @@
-import { customSection } from '../wasm/sections.js';
-
-/**
- * Parses the `toildb.derives` wiring section emitted by toilscript for a
- * `@database` class with `@derive` materializer methods (see the compiler's
- * `buildToilDbDerives`). It maps each derive to its owning `@database` class, so
- * the dev runtime (`runtime/module.ts`) can, after a dispatch writes a source
- * collection, re-run that database's derives under FunctionKind=Derive. Fails
- * closed: any malformed byte yields `[]` (no derive runs) rather than throwing.
- *
- * Section layout (LE), mirroring `buildToilDbDerives`:
- *   u16 format_version = 1
- *   u16 n_derives
- *   per derive: u16 derive_id, str db_name, str method_name   (str = u32 len + bytes)
- */
-export interface DeriveEntry {
-    readonly deriveId: number;
-    readonly dbName: string;
-    readonly methodName: string;
-}
-
-const SECTION = 'toildb.derives';
-const VERSION = 1;
-const MAX_SECTION_BYTES = 128 * 1024;
-const MAX_DERIVES = 1024;
-const MAX_NAME_BYTES = 1024;
-const UTF8_DECODER = new TextDecoder('utf-8', { fatal: true });
-
-export function parseDerives(wasm: Buffer): readonly DeriveEntry[] {
-    let section: Buffer | null;
-    try {
-        section = customSection(wasm, SECTION);
-    } catch {
-        return [];
-    }
-    if (section === null) return [];
-    if (section.length > MAX_SECTION_BYTES) return [];
-
-    const r = new Reader(section);
-    const version = r.u16();
-    if (!r.ok || version !== VERSION) return [];
-    const count = r.u16();
-    if (!r.ok || count > MAX_DERIVES) return [];
-
-    const derives: DeriveEntry[] = [];
-    for (let i = 0; i < count && r.ok; i++) {
-        const deriveId = r.u16();
-        const dbName = r.string();
-        const methodName = r.string();
-        if (!r.ok || dbName.length === 0) return [];
-        derives.push({ deriveId, dbName, methodName });
-    }
-    if (!r.ok || r.remaining() !== 0) return [];
-    return derives;
-}
-
-/**
- * The derives whose owning `@database` had at least one source collection
- * written during this dispatch. `written` holds "Db/coll" store keys; the
- * database is the prefix before the first `/`. Each affected derive appears at
- * most once (coalescing: many writes to one database run its derives once).
- */
-export function derivesForWrites(
-    derives: readonly DeriveEntry[],
-    written: ReadonlySet<string>,
-): readonly DeriveEntry[] {
-    if (derives.length === 0 || written.size === 0) return [];
-    const dbs = new Set<string>();
-    for (const key of written) {
-        const slash = key.indexOf('/');
-        dbs.add(slash >= 0 ? key.slice(0, slash) : key);
-    }
-    return derives.filter((d) => dbs.has(d.dbName));
-}
-
-class Reader {
-    private pos = 0;
-    ok = true;
-
-    constructor(private readonly bytes: Buffer) {}
-
-    remaining(): number {
-        return this.bytes.length - this.pos;
-    }
-
-    u16(): number {
-        if (!this.ok || this.pos + 2 > this.bytes.length) {
-            this.ok = false;
-            return 0;
-        }
-        const out = this.bytes.readUInt16LE(this.pos);
-        this.pos += 2;
-        return out;
-    }
-
-    u32(): number {
-        if (!this.ok || this.pos + 4 > this.bytes.length) {
-            this.ok = false;
-            return 0;
-        }
-        const out = this.bytes.readUInt32LE(this.pos);
-        this.pos += 4;
-        return out;
-    }
-
-    string(): string {
-        const len = this.u32();
-        if (!this.ok || len > MAX_NAME_BYTES || this.pos + len > this.bytes.length) {
-            this.ok = false;
-            return '';
-        }
-        try {
-            const out = UTF8_DECODER.decode(this.bytes.subarray(this.pos, this.pos + len));
-            this.pos += len;
-            return out;
-        } catch {
-            this.ok = false;
-            return '';
-        }
-    }
-}

package/src/devserver/db/index.ts

@@ -1,20 +0,0 @@
-/**
- * The dev ToilDB subsystem: the {@link DevDatabase} class + its process
- * singleton, the per-request state, the catalog parser, and the `data.*` host
- * imports. The production edge backs the SAME imports with ScyllaDB.
- */
-
-export {
-    __resetDbForTests,
-    __setDbCatalogForTests,
-    buildDatabaseImports,
-    commitDeriveCheckpoints,
-    configureDbPersistence,
-    DevDatabase,
-    devDb,
-    persistDb,
-    setDbCatalog,
-} from './database.js';
-export { parseCatalog } from './catalog.js';
-export { type DeriveEntry, derivesForWrites, parseDerives } from './derives.js';
-export { CollectionFamily, DbFunctionKind, type DbDevState, freshDbState } from './types.js';

package/src/devserver/db/routeKinds.ts

@@ -1,191 +0,0 @@
-import { customSection } from '../wasm/sections.js';
-import { DbFunctionKind } from './types.js';
-
-export interface RouteKindEntry {
-    readonly method: number;
-    readonly kind: DbFunctionKind;
-    readonly pattern: string;
-}
-
-export interface RpcKindEntry {
-    readonly methodId: number;
-    readonly kind: DbFunctionKind;
-}
-
-const SECTION = 'toildb.route_kinds';
-const RPC_SECTION = 'toildb.rpc_kinds';
-const VERSION = 1;
-const MAX_SECTION_BYTES = 128 * 1024;
-const MAX_ROUTES = 2048;
-const MAX_PATTERN_BYTES = 2048;
-const UTF8_DECODER = new TextDecoder('utf-8', { fatal: true });
-
-const METHOD_CODES: Readonly<Record<string, number>> = {
-    GET: 0,
-    POST: 1,
-    PUT: 2,
-    DELETE: 3,
-    PATCH: 4,
-    HEAD: 5,
-    OPTIONS: 6,
-};
-
-export function parseRouteKinds(wasm: Buffer): readonly RouteKindEntry[] {
-    let section: Buffer | null;
-    try {
-        section = customSection(wasm, SECTION);
-    } catch {
-        return [];
-    }
-    if (section === null) return [];
-    if (section.length > MAX_SECTION_BYTES) return [];
-
-    const r = new Reader(section);
-    const version = r.u16();
-    if (!r.ok || version !== VERSION) return [];
-    const count = r.u16();
-    if (!r.ok || count > MAX_ROUTES) return [];
-
-    const routes: RouteKindEntry[] = [];
-    for (let i = 0; i < count && r.ok; i++) {
-        const method = r.u8();
-        const kindByte = r.u8();
-        const pattern = r.string();
-        const kind =
-            kindByte === 0 ? DbFunctionKind.Query : kindByte === 1 ? DbFunctionKind.Action : null;
-        if (!r.ok || method < 0 || method > 6 || kind === null || !validPattern(pattern)) return [];
-        routes.push({ method, kind, pattern });
-    }
-    if (!r.ok || r.remaining() !== 0) return [];
-    return routes;
-}
-
-export function routeKindForRequest(
-    routes: readonly RouteKindEntry[],
-    method: string,
-    path: string,
-): DbFunctionKind | null {
-    const methodCode = METHOD_CODES[method.toUpperCase()];
-    if (methodCode === undefined) return null;
-    for (const route of routes) {
-        if (route.method === methodCode && routeMatches(route.pattern, path)) return route.kind;
-    }
-    return null;
-}
-
-export function parseRpcKinds(wasm: Buffer): readonly RpcKindEntry[] {
-    let section: Buffer | null;
-    try {
-        section = customSection(wasm, RPC_SECTION);
-    } catch {
-        return [];
-    }
-    if (section === null) return [];
-    if (section.length > MAX_SECTION_BYTES) return [];
-
-    const r = new Reader(section);
-    const version = r.u16();
-    if (!r.ok || version !== VERSION) return [];
-    const count = r.u16();
-    if (!r.ok || count > MAX_ROUTES) return [];
-
-    const methods: RpcKindEntry[] = [];
-    for (let i = 0; i < count && r.ok; i++) {
-        const methodId = r.u32();
-        const kindByte = r.u8();
-        const kind =
-            kindByte === 0 ? DbFunctionKind.Query : kindByte === 1 ? DbFunctionKind.Action : null;
-        if (!r.ok || kind === null) return [];
-        methods.push({ methodId, kind });
-    }
-    if (!r.ok || r.remaining() !== 0) return [];
-    return methods;
-}
-
-/** DB kind for an RPC method id (the `dacely-rpc` header). An id absent from rpc_kinds is read-only
- *  (Query) - the safe default for RPC, matching the host gate. */
-export function rpcKindForId(methods: readonly RpcKindEntry[], methodId: number): DbFunctionKind {
-    for (const m of methods) {
-        if (m.methodId === methodId) return m.kind;
-    }
-    return DbFunctionKind.Query;
-}
-
-function validPattern(pattern: string): boolean {
-    if (pattern.length === 0 || !pattern.startsWith('/')) return false;
-    for (let i = 0; i < pattern.length; i++) {
-        const c = pattern.charCodeAt(i);
-        if (c < 0x20 || c > 0x7e) return false;
-    }
-    return true;
-}
-
-function routeMatches(pattern: string, pathWithQuery: string): boolean {
-    const q = pathWithQuery.indexOf('?');
-    const path = q >= 0 ? pathWithQuery.slice(0, q) : pathWithQuery;
-    const patternSegs = pattern.split('/').filter(Boolean);
-    const pathSegs = path.split('/').filter(Boolean);
-    if (patternSegs.length !== pathSegs.length) return false;
-    for (let i = 0; i < patternSegs.length; i++) {
-        const p = patternSegs[i] ?? '';
-        const a = pathSegs[i] ?? '';
-        if (p.startsWith(':') && p.length > 1 && a.length > 0) continue;
-        if (p !== a) return false;
-    }
-    return true;
-}
-
-class Reader {
-    private pos = 0;
-    ok = true;
-
-    constructor(private readonly bytes: Buffer) {}
-
-    remaining(): number {
-        return this.bytes.length - this.pos;
-    }
-
-    u8(): number {
-        if (!this.ok || this.pos + 1 > this.bytes.length) {
-            this.ok = false;
-            return 0;
-        }
-        return this.bytes[this.pos++] ?? 0;
-    }
-
-    u16(): number {
-        if (!this.ok || this.pos + 2 > this.bytes.length) {
-            this.ok = false;
-            return 0;
-        }
-        const out = this.bytes.readUInt16LE(this.pos);
-        this.pos += 2;
-        return out;
-    }
-
-    u32(): number {
-        if (!this.ok || this.pos + 4 > this.bytes.length) {
-            this.ok = false;
-            return 0;
-        }
-        const out = this.bytes.readUInt32LE(this.pos);
-        this.pos += 4;
-        return out;
-    }
-
-    string(): string {
-        const len = this.u32();
-        if (!this.ok || len > MAX_PATTERN_BYTES || this.pos + len > this.bytes.length) {
-            this.ok = false;
-            return '';
-        }
-        try {
-            const out = UTF8_DECODER.decode(this.bytes.subarray(this.pos, this.pos + len));
-            this.pos += len;
-            return out;
-        } catch {
-            this.ok = false;
-            return '';
-        }
-    }
-}

package/src/devserver/db/types.ts

@@ -1,168 +0,0 @@
-/**
- * Shared types, limits, and ABI return codes for the dev ToilDB emulation
- * (see `./database.ts`). The wire ABI mirrors the production edge byte-for-byte
- * (toildb/ABI.md): every byte region is a (ptr,len) into guest memory; returns
- * are `>= 0` success (a length/handle/flag), `-1` too-small, `-2` absent,
- * `<= -1000` a typed error (`-(1000+TDLnnn)`).
- */
-
-export enum CollectionFamily {
-    Record = 0,
-    View = 1,
-    Events = 2,
-    Counter = 3,
-    Membership = 4,
-    Unique = 5,
-    Capacity = 6,
-}
-
-export enum DbFunctionKind {
-    Query = 'query',
-    Action = 'action',
-    Derive = 'derive',
-    Job = 'job',
-    Admin = 'admin',
-}
-
-export function isCollectionFamily(value: number): value is CollectionFamily {
-    return value >= CollectionFamily.Record && value <= CollectionFamily.Capacity;
-}
-
-export interface DevField {
-    name: string;
-    typeName: string;
-    isArray: boolean;
-    unique: boolean;
-}
-
-export interface DevCollectionHandle {
-    name: string;
-    family: CollectionFamily;
-    schemaVersion: number;
-    replication: number;
-    placement: number;
-    fillMaxWaitMs: number;
-    fillAllowStale: boolean;
-    /** The value type's field layout (declaration order); present when the catalog
-     *  carried one. Drives @unique enforcement. */
-    fields?: DevField[];
-}
-
-export type DbCatalogState =
-    | { kind: 'no-section' }
-    | { kind: 'malformed' }
-    | { kind: 'present'; collections: Map<string, DevCollectionHandle> };
-
-/** Per-request data state: resolved handles + the last variable-length result +
- *  the schema_version of that result's row (surfaced via result_schema_version). */
-export interface DbDevState {
-    handles: DevCollectionHandle[];
-    lastResult: Buffer | null;
-    lastResultVersion: number;
-    functionKind: DbFunctionKind;
-    /** Names ("Db/coll") of source collections written during this dispatch, so
-     *  the runtime can re-run the affected `@derive` materializers afterward.
-     *  Only populated for non-Derive dispatches (a derive's own writes must not
-     *  re-trigger it - see `database.ts` `recordWrite`). */
-    writtenCollections: Set<string>;
-    /** The derive index currently running (set by `runDerive`), so `events.since` keys its resumable
-     *  checkpoint per (derive, source key). 0 outside a derive. */
-    deriveId: number;
-    /** Checkpoint advances `events.since` staged during THIS derive run, keyed `"<deriveId>\0<storeKey>"` ->
-     *  events-folded count. Committed to the durable `deriveCheckpoints` only after the fold's `derive_run`
-     *  returns (its `view.publish`es have landed): a trapped fold discards this overlay, so the next run
-     *  re-reads the batch rather than skipping it (publish-then-persist, mirroring the edge). */
-    pendingCheckpoints: Map<string, number>;
-}
-
-export function freshDbState(): DbDevState {
-    return {
-        handles: [],
-        lastResult: null,
-        lastResultVersion: -1,
-        functionKind: DbFunctionKind.Job,
-        writtenCollections: new Set<string>(),
-        deriveId: 0,
-        pendingCheckpoints: new Map<string, number>(),
-    };
-}
-
-/** A finite-resource escrow: a ceiling + a set of reservations, each held (in
- *  flight, TTL'd) or confirmed (a permanent consume). Both count against available;
- *  a confirmed reservation never expires. Mirrors `toildb::capacity::Escrow`. */
-export interface Reservation {
-    amount: bigint;
-    expiresMs: number;
-    confirmed: boolean;
-}
-export interface CapLedger {
-    total: bigint;
-    reservations: Map<bigint, Reservation>;
-    nextId: bigint;
-}
-
-/** The on-disk snapshot shape: dev data + its versions, JSON with base64 buffers. */
-export interface DbSnapshot {
-    store: Record<string, { v: string; sv: number }>; // records + unique owners (+ schema_version)
-    recordIdem?: Record<
-        string,
-        { requestHash: string; state: 'pending' | 'done'; outcome?: RecordOutcomeSnapshot }
-    >;
-    uniqueIdem?: Record<string, string>;
-    views: Record<string, { v: string; sv: number }>;
-    members: Record<string, Record<string, { v: string; sv: number }>>;
-    counters: Record<string, string>;
-    counterIdem?: Record<string, string>;
-    events: Record<string, { v: string; sv: number }[]>;
-    eventDedup: Record<string, string[]>;
-    /** Durable `events.since` cursors, `"<deriveId>\0<storeKey>"` -> events-folded count, so a restart
-     *  resumes an incremental `@derive` instead of re-folding the whole log onto the persisted view. */
-    deriveCheckpoints?: Record<string, number>;
-    capacity: Record<
-        string,
-        {
-            total: string;
-            nextId: string;
-            reservations: [string, { amount: string; expiresMs: number; confirmed: boolean }][];
-        }
-    >;
-}
-
-export type RecordOutcomeSnapshot =
-    | { kind: 'unit' }
-    | { kind: 'value'; v: string; sv: number }
-    | { kind: 'absent' }
-    | { kind: 'already_exists' }
-    | { kind: 'not_found' }
-    | { kind: 'conflict' };
-
-/** Edge caps (toildb::capacity::escrow): bound the reservation count + the hold TTL. */
-export const MAX_RESERVATIONS = 4096;
-export const MAX_RESERVATION_TTL_MS = 86_400_000; // 24h
-
-export const MAX_NAME = 512;
-export const MAX_KEY = 4096;
-export const MAX_VALUE = 256 * 1024;
-export const DEFAULT_FILL_WAIT_MS = 50;
-export const MAX_FILL_WAIT_MS = 60_000;
-
-// i64 saturation bounds (the edge `MemEngine`/`ScyllaEngine` counters are i64).
-const I64_MIN = -(2n ** 63n);
-const I64_MAX = 2n ** 63n - 1n;
-export function satI64(v: bigint): bigint {
-    return v < I64_MIN ? I64_MIN : v > I64_MAX ? I64_MAX : v;
-}
-
-// Return codes, mirroring the edge ABI (`toildb::observe::diagnostics`): a typed
-// error is `-(1000 + TDLnnn)`; a plain absence is ABSENT (-2), not a typed error.
-export const ABSENT = -2; // NotFound / absent
-export const TOO_SMALL = -1;
-export const INVALID_HANDLE = -1001; // TDL001
-export const ALREADY_EXISTS = -1003; // TDL003 (create on an existing key)
-export const CONFLICT = -1004; // TDL004 (e.g. unique release by a non-owner)
-export const UNAVAILABLE = -1031; // TDL031 (retryable in-flight/uncertain op)
-export const CODEC_ERR = -1006; // TDL006 (e.g. a non-positive reserve amount)
-export const OP_NOT_ALLOWED_FOR_FAMILY = -1010; // TDL010
-export const OP_NOT_ALLOWED_IN_KIND = -1011; // TDL011
-export const TOO_MANY_KEYS = -1020; // TDL020 (get_many over the per-call cap)
-export const SCHEMA_UNAVAILABLE = -1070; // TDL070

package/src/devserver/email/config.ts

@@ -1,141 +0,0 @@
-/**
- * Resolve the dev / self-host email config: merge the typed `toil.config.ts`
- * `email` section (non-secret) with the reserved `TOIL_EMAIL_*` env keys (which
- * WIN, for edge parity + 12-factor), and pull the API key from
- * `TOIL_EMAIL_API_KEY` (a secret, only ever in `.env.secrets`/the environment).
- *
- * Mirrors the edge's `host/email.rs` (`EmailSection::from_reserved` + `resolve`):
- * provider parse, `from` validation, Gmail/SMTP defaults. Returns a `null`
- * config when email is simply not set up (silent), or a `warning` when it is
- * partially configured but invalid (logged at startup, treated as Disabled).
- */
-import type { EmailBackendConfig } from 'toiljs/shared';
-
-import { validFrom } from './validate.js';
-
-export type ResolvedProvider = 'resend' | 'smtp';
-
-export interface ResolvedSmtp {
-    readonly host: string;
-    readonly port: number;
-    readonly user: string;
-}
-
-export interface ResolvedEmailConfig {
-    readonly provider: ResolvedProvider;
-    readonly from: string;
-    readonly apiKey: string;
-    readonly maxPerMin: number;
-    readonly maxPerDay: number;
-    readonly maxPerRecipientPerHour: number;
-    /** Present iff `provider === 'smtp'`. */
-    readonly smtp?: ResolvedSmtp;
-}
-
-export interface ResolveResult {
-    /** The resolved config, or `null` when email is unconfigured / invalid. */
-    readonly config: ResolvedEmailConfig | null;
-    /** A reason email is off despite partial config (logged at startup). */
-    readonly warning: string | null;
-}
-
-/** `TOIL_EMAIL_<NAME>` from the reserved map, trimmed; `undefined` if absent/empty. */
-function envOf(reserved: Map<string, string>, name: string): string | undefined {
-    const v = reserved.get(`TOIL_EMAIL_${name}`);
-    const t = v?.trim();
-    return t ? t : undefined;
-}
-
-function parseBool(v: string | undefined): boolean | undefined {
-    if (v === undefined) return undefined;
-    return v === '1' || v === 'true' || v === 'yes' || v === 'on';
-}
-
-function parseInt0(v: string | undefined, fallback: number): number {
-    if (v === undefined) return fallback;
-    const n = Number.parseInt(v, 10);
-    return Number.isFinite(n) && n >= 0 ? n : fallback;
-}
-
-export function resolveEmailConfig(
-    cfg: EmailBackendConfig | null | undefined,
-    reserved: Map<string, string>,
-): ResolveResult {
-    const c = cfg ?? {};
-    // Env (TOIL_EMAIL_*) overrides the config file; api key is env-only.
-    const providerId = (envOf(reserved, 'PROVIDER') ?? c.provider ?? 'resend').toLowerCase();
-    const from = envOf(reserved, 'FROM') ?? c.from?.trim();
-    const apiKey = envOf(reserved, 'API_KEY');
-    const enabled = parseBool(envOf(reserved, 'ENABLED'));
-
-    // Unconfigured: explicitly disabled, or no `from` and no key at all -> silent off.
-    if (enabled === false) return { config: null, warning: null };
-    if (from === undefined && apiKey === undefined && cfg == null) {
-        return { config: null, warning: null };
-    }
-
-    if (from === undefined) {
-        return { config: null, warning: 'email config present but `from` is missing' };
-    }
-    if (!validFrom(from)) {
-        return { config: null, warning: 'email `from` is not a valid address (CRLF or no `@`)' };
-    }
-    // The API-key requirement is PER PROVIDER: Resend authenticates with a key, and
-    // Gmail SMTP needs an app password (also carried in the key), but a plain SMTP
-    // relay (a local dev catch-all like Mailpit/MailHog on 127.0.0.1, or an internal
-    // open relay) needs NO credential. Requiring a key for plain SMTP silently
-    // disabled a perfectly good local-SMTP dev setup.
-    let provider: ResolvedProvider;
-    let smtp: ResolvedSmtp | undefined;
-    if (providerId === 'resend') {
-        if (apiKey === undefined) {
-            return {
-                config: null,
-                warning: 'provider `resend` requires TOIL_EMAIL_API_KEY (in .env.secrets)',
-            };
-        }
-        provider = 'resend';
-    } else if (providerId === 'gmail' || providerId === 'smtp') {
-        provider = 'smtp';
-        const isGmail = providerId === 'gmail';
-        const host =
-            envOf(reserved, 'SMTP_HOST') ??
-            c.smtp?.host?.trim() ??
-            (isGmail ? 'smtp.gmail.com' : '');
-        if (!host) {
-            return { config: null, warning: 'provider `smtp` requires TOIL_EMAIL_SMTP_HOST' };
-        }
-        // Gmail always needs an app password; plain SMTP may be auth-less (the send
-        // path omits `auth` when the key is empty). See providers.ts sendSmtp.
-        if (isGmail && apiKey === undefined) {
-            return {
-                config: null,
-                warning: 'provider `gmail` requires TOIL_EMAIL_API_KEY (a Gmail app password)',
-            };
-        }
-        const port = parseInt0(envOf(reserved, 'SMTP_PORT'), c.smtp?.port ?? 0) || 587;
-        const user = envOf(reserved, 'SMTP_USER') ?? c.smtp?.user?.trim() ?? from;
-        smtp = { host, port, user };
-    } else {
-        return {
-            config: null,
-            warning: `unknown email provider "${providerId}" (resend|gmail|smtp)`,
-        };
-    }
-
-    return {
-        config: {
-            provider,
-            from,
-            apiKey: apiKey ?? '',
-            maxPerMin: parseInt0(envOf(reserved, 'MAX_PER_MIN'), c.maxPerMin ?? 60),
-            maxPerDay: parseInt0(envOf(reserved, 'MAX_PER_DAY'), c.maxPerDay ?? 0),
-            maxPerRecipientPerHour: parseInt0(
-                envOf(reserved, 'MAX_PER_RECIPIENT_PER_HOUR'),
-                c.maxPerRecipientPerHour ?? 5,
-            ),
-            smtp,
-        },
-        warning: null,
-    };
-}