toiljs 0.0.43 → 0.0.44

package/docs/environment.md

@@ -0,0 +1,97 @@
+# Environment variables & secrets
+
+`Environment` gives a tenant **per-app environment variables and secrets**, set
+out of band (a dashboard, like GitHub Actions) so the deployed `.wasm` carries
+**no credentials**. It is read-only from app code — there is no `set`; values are
+configured on the deployment side, never from the module.
+
+```ts
+import { Response, RouteContext } from 'toiljs/server/runtime';
+
+@rest('cfg')
+class Cfg {
+    @get('/')
+    show(ctx: RouteContext): Response {
+        const base = Environment.get('PUBLIC_API_BASE'); // plain var, or null
+        const key = Environment.getSecure('STRIPE_KEY'); // secret, or null
+        // Use `key` to call a third party; never log it or return it to a client.
+        return Response.text(base != null ? base : 'unset');
+    }
+}
+```
+
+`Environment` is a global — no import needed (like `EmailService` / `AuthService`).
+
+## Two disjoint buckets
+
+Just like GitHub Actions' `vars` vs `secrets`:
+
+- **`Environment.get(key)`** reads **plain vars** — non-sensitive config (a public
+  API base URL, a feature flag, a region). Returns the string, or `null`.
+- **`Environment.getSecure(key)`** reads **secrets** — sensitive values (a
+  third-party API key). Returns the string, or `null`.
+
+The buckets are **disjoint**: a secret is **never** returned by `get()`, and a
+plain var is never returned by `getSecure()`. That keeps a secret from leaking
+through a code path that logs the result of a `get()`. Keys are case-sensitive,
+exact-match.
+
+> Secrets you read with `getSecure` are plaintext in your module at runtime
+> (that's the point — you need them to call out). Don't log them, don't put them
+> in a response, and don't copy them into a client bundle.
+
+## What is NOT here
+
+Framework-reserved namespaces (today: **email** provider config) are **host-only**
+— resolved and used in Rust where the framework needs them, and **never exposed to
+the `.wasm`**. There is no `Environment.email`; you configure email in the
+`[email]` block of the same env file and the platform uses it for you (see
+[Email](./email.md)). The env imports only ever see your own `vars` / `secrets`.
+
+## Where values live
+
+Vars and secrets live in **two separate dotenv (`.env`) files**, so the disjoint
+split is structural and the secrets file can be locked down on its own. On the
+edge they are per host, **out of `hosts/`** (so the config watcher never sees a
+credential) — the dashboard / edge database replaces them later:
+
+```bash
+# $TOIL_ENV_DIR/<host>.env          (default dir /run/toil/env)
+PUBLIC_API_BASE=https://api.example.com   # -> Environment.get("PUBLIC_API_BASE")
+REGION=eu
+
+# $TOIL_ENV_DIR/<host>.env.secrets  (mode 0600)
+STRIPE_KEY=sk_live_xxx                     # -> Environment.getSecure("STRIPE_KEY")
+
+# host-only email config — reserved TOIL_EMAIL_* keys, NEVER exposed to the .wasm
+TOIL_EMAIL_ENABLED=true
+TOIL_EMAIL_PROVIDER=resend
+TOIL_EMAIL_FROM=noreply@example.com
+TOIL_EMAIL_API_KEY=re_xxx
+```
+
+Each file is plain dotenv: `KEY=value` per line, `#` comments, optional `export`,
+optional quotes. Keys with the reserved **`TOIL_`** prefix are framework/host-only
+and are stripped from BOTH guest buckets — a tenant can never read them via
+`get`/`getSecure` (see [Email](./email.md) for `TOIL_EMAIL_*`).
+
+On the edge, env is loaded **lazily** (the first time your code reads it) into a
+**bounded, shared, read-only cache** with idle eviction: the data lives in one
+place and is never copied per request, a host that never reads env costs nothing,
+secrets are wiped when a host goes cold, and a flood of requests to many distinct
+hosts can never grow memory without bound.
+
+## In dev
+
+`toiljs dev` reads `.env` (vars) and `.env.secrets` (secrets) at your project
+root, and overlays `process.env` as plain vars for convenience. Both are
+gitignored by the scaffold. The ABI is identical to the edge, so code that runs
+in dev runs on the edge.
+
+```bash
+# .env  (vars)
+PUBLIC_API_BASE=http://localhost:4000
+
+# .env.secrets  (secrets; 0600; gitignored)
+STRIPE_KEY=sk_test_xxx
+```

package/examples/basic/server/main.ts

@@ -11,6 +11,7 @@ import './routes/Players';
 import './routes/Leaderboard';
 import './routes/Session';
 import './routes/PqDemo';
+import './routes/EnvDemo';
 import './services/Stats';
 import './services/remotes';
 

package/examples/basic/server/routes/EnvDemo.ts

@@ -0,0 +1,42 @@
+import { Response, RouteContext } from 'toiljs/server/runtime';
+
+/**
+ * Environment demo: per-tenant config + secrets, read from server code with no
+ * import (`Environment` is a server global).
+ *
+ * Values come from out-of-band dotenv files the edge loads lazily and never
+ * bundles into the `.wasm`: plain vars from `<host>.env`, secrets from
+ * `<host>.env.secrets`. In `toiljs dev` those are `.env` / `.env.secrets` at the
+ * project root — copy `.env.example` / `.env.secrets.example` to see this populate.
+ *
+ *   Environment.get("KEY")        -> a plain var, or null
+ *   Environment.getSecure("KEY")  -> a secret, or null   (a secret is NEVER
+ *                                    returned by get(), the buckets are disjoint)
+ *
+ * Reserved `TOIL_*` keys (e.g. the `TOIL_EMAIL_*` mailer config) are host-only:
+ * the edge reads them, but they are unreachable through get()/getSecure().
+ */
+@rest('env')
+class EnvDemo {
+    /** GET /env/show -> the public vars, plus WHETHER the demo secret is set.
+     *  We return the secret's presence, not its value: getSecure() hands the guest
+     *  the real secret, but a demo must never echo a secret back over the wire. */
+    @get('/show')
+    public show(_ctx: RouteContext): Response {
+        let greeting = '(unset)';
+        const g = Environment.get('PUBLIC_GREETING');
+        if (g != null) greeting = g;
+
+        let region = '(unset)';
+        const r = Environment.get('REGION');
+        if (r != null) region = r;
+
+        const apiKeySet = Environment.getSecure('DEMO_API_KEY') != null;
+
+        const body =
+            'PUBLIC_GREETING=' + greeting + '\n' +
+            'REGION=' + region + '\n' +
+            'DEMO_API_KEY set=' + (apiKeySet ? 'yes' : 'no') + '\n';
+        return Response.text(body, 200);
+    }
+}

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "toiljs",
     "type": "module",
-    "version": "0.0.43",
+    "version": "0.0.44",
     "author": "Dacely",
     "description": "The modern React framework: a file-based React frontend and a ToilScript-compiled WebAssembly backend.",
     "repository": {
@@ -116,8 +116,8 @@
         "setup": "npm i && npm run build"
     },
     "dependencies": {
-        "@btc-vision/post-quantum": "^0.5.3",
         "@dacely/hyper-express": "6.17.4",
+        "@dacely/noble-post-quantum": "^0.6.1",
         "@dacely/toilscript-loader": "^0.1.0",
         "@eslint-react/eslint-plugin": "^5.8.8",
         "@eslint/js": "^10.0.1",

package/server/globals/environment.ts

@@ -0,0 +1,82 @@
+// Environment: the per-tenant environment variables + secrets a tenant
+// configures OUT OF BAND (a dashboard today, backed by a file; the edge DB
+// later) so the deployed `.wasm` carries no credentials — the GitHub-Actions
+// model. Available as a no-import global (the toilscript `--lib` mechanism, like
+// `AuthService` / `EmailService`).
+//
+//   const base = Environment.get("PUBLIC_API_BASE");      // plain var, or null
+//   const key  = Environment.getSecure("STRIPE_KEY");     // secret, or null
+//
+// Two DISJOINT buckets (like GitHub `vars.*` vs `secrets.*`): `get` reads ONLY
+// plain vars, `getSecure` reads ONLY secrets, so a secret can never come back
+// through `get()` (and leak into a log of a `get`). READ-ONLY — there is no
+// `set`; a tenant sets values on their dashboard, never from the `.wasm`.
+//
+// Framework-reserved namespaces (e.g. the email provider config) are HOST-ONLY:
+// they are resolved and consumed in Rust where the framework uses them and are
+// NEVER reachable here — these imports only see the tenant's own vars/secrets.
+//
+// Backed by the `env_get` / `env_get_secure` host imports (toil-backend
+// `env_get_import.rs`, reading the lazy + bounded `env_cache`, plus the toiljs
+// dev-server mock). A tenant that never reads env imports neither, so
+// AssemblyScript tree-shakes this away.
+
+// Host imports: copy one value into `outPtr[0..outCap]`. Return the value's byte
+// length (`0` = present but empty), `-1` if `outCap` is too small (retry with a
+// bigger buffer), `-2` if the key is absent. Resolved from the trusted request
+// host, never anything the guest passes.
+// @ts-ignore: decorator
+@external('env', 'env_get')
+declare function __toilEnvGet(keyPtr: usize, keyLen: i32, outPtr: usize, outCap: i32): i32;
+// @ts-ignore: decorator
+@external('env', 'env_get_secure')
+declare function __toilEnvGetSecure(keyPtr: usize, keyLen: i32, outPtr: usize, outCap: i32): i32;
+
+export namespace Environment {
+    /** The key is absent from the bucket. */
+    const ABSENT: i32 = -2;
+    /** The output buffer was too small; retry with a bigger one. */
+    const TOO_SMALL: i32 = -1;
+    /** First attempt buffer; doubles on `TOO_SMALL`. */
+    const INITIAL_CAP: i32 = 256;
+    /** Upper bound on a value, matching the host's per-value cap. */
+    const MAX_CAP: i32 = 64 * 1024;
+
+    /** Shared reader for both buckets; `secure` picks the secret import. */
+    function read(key: string, secure: bool): string | null {
+        const keyB = Uint8Array.wrap(String.UTF8.encode(key));
+        let cap = INITIAL_CAP;
+        while (cap <= MAX_CAP) {
+            const buf = new Uint8Array(cap);
+            const n = secure
+                ? __toilEnvGetSecure(keyB.dataStart, keyB.length, buf.dataStart, cap)
+                : __toilEnvGet(keyB.dataStart, keyB.length, buf.dataStart, cap);
+            if (n == ABSENT) return null;
+            if (n == TOO_SMALL) {
+                cap = cap * 2;
+                continue;
+            }
+            if (n < 0) return null; // unknown negative: fail closed
+            return String.UTF8.decodeUnsafe(buf.dataStart, n);
+        }
+        return null; // value larger than MAX_CAP: treat as absent
+    }
+
+    /**
+     * The plain environment variable `key` (a tenant var, set on the dashboard),
+     * or `null` if it is not set. NEVER returns a secret — use {@link getSecure}
+     * for those.
+     */
+    export function get(key: string): string | null {
+        return read(key, false);
+    }
+
+    /**
+     * The secret `key` (a tenant secret, set on the dashboard), or `null` if it
+     * is not set. Disjoint from {@link get}: a plain var is never returned here.
+     * The value is sensitive — do not log it.
+     */
+    export function getSecure(key: string): string | null {
+        return read(key, true);
+    }
+}

package/src/cli/create.ts

@@ -165,7 +165,7 @@ function scaffold(
         '.prettierignore':
             'node_modules\nbuild\n.toil\nshared/server.ts\ntoil-env.d.ts\ntoil-routes.d.ts\n',
         '.gitignore':
-            'node_modules\nbuild\n.toil\nshared/server.ts\ntoil-env.d.ts\ntoil-routes.d.ts\n',
+            'node_modules\nbuild\n.toil\nshared/server.ts\ntoil-env.d.ts\ntoil-routes.d.ts\n# Local dev env vars/secrets (never commit)\n.env\n.env.secrets\n',
         // Use the project's pinned TypeScript (node_modules) instead of VS Code's bundled version.
         '.vscode/settings.json':
             JSON.stringify({ 'typescript.tsdk': 'node_modules/typescript/lib' }, null, 4) + '\n',
@@ -304,6 +304,7 @@ declare class RenderedEmail { subject: string; body: string; html: string; const
 declare class EmailTemplate { constructor(subject: string, body: string, html?: string); render(vars: Map<string, string>): RenderedEmail; send(to: string, vars: Map<string, string>, purpose?: string): EmailStatus; }
 declare enum RateLimit { FixedWindow, SlidingWindow, TokenBucket }
 declare namespace RateLimitService { function guard(routeId: i32, strategy: i32, limit: i32, window: i32): import('toiljs/server/runtime/response').Response | null; function guardKeyed(routeId: i32, strategy: i32, limit: i32, window: i32, key: string): import('toiljs/server/runtime/response').Response | null; }
+declare namespace Environment { function get(key: string): string | null; function getSecure(key: string): string | null; }
 declare class TwoFactorIssue { code: string; token: string; constructor(code: string, token: string); }
 declare class TwoFactorChallenge { token: string; status: EmailStatus; constructor(token: string, status: EmailStatus); }
 declare namespace TwoFactor { const DEFAULT_TTL_SECS: u64; const DEFAULT_DIGITS: i32; function setSecret(secret: Uint8Array): void; function issue(recipient: string, purpose: string, ttlSecs?: u64, digits?: i32): TwoFactorIssue; function send(recipient: string, purpose: string, ttlSecs?: u64, digits?: i32): TwoFactorChallenge; function verify(token: string, recipient: string, code: string): bool; }

package/src/client/auth.ts

@@ -12,7 +12,7 @@
  */
 
 import { argon2id, sha256 } from 'hash-wasm';
-import { ml_dsa44 } from '@btc-vision/post-quantum/ml-dsa.js';
+import { ml_dsa44 } from '@dacely/noble-post-quantum/ml-dsa.js';
 
 import { DataReader, DataWriter } from 'toiljs/io';
 

package/src/compiler/generate.ts

@@ -117,6 +117,7 @@ export const TOIL_SERVER_ENV_DTS =
     `declare class EmailTemplate { constructor(subject: string, body: string, html?: string); render(vars: Map<string, string>): RenderedEmail; send(to: string, vars: Map<string, string>, purpose?: string): EmailStatus; }\n` +
     `declare enum RateLimit { FixedWindow, SlidingWindow, TokenBucket }\n` +
     `declare namespace RateLimitService { function guard(routeId: i32, strategy: i32, limit: i32, window: i32): import('toiljs/server/runtime/response').Response | null; function guardKeyed(routeId: i32, strategy: i32, limit: i32, window: i32, key: string): import('toiljs/server/runtime/response').Response | null; }\n` +
+    `declare namespace Environment { function get(key: string): string | null; function getSecure(key: string): string | null; }\n` +
     `declare class TwoFactorIssue { code: string; token: string; constructor(code: string, token: string); }\n` +
     `declare class TwoFactorChallenge { token: string; status: EmailStatus; constructor(token: string, status: EmailStatus); }\n` +
     `declare namespace TwoFactor { const DEFAULT_TTL_SECS: u64; const DEFAULT_DIGITS: i32; function setSecret(secret: Uint8Array): void; function issue(recipient: string, purpose: string, ttlSecs?: u64, digits?: i32): TwoFactorIssue; function send(recipient: string, purpose: string, ttlSecs?: u64, digits?: i32): TwoFactorChallenge; function verify(token: string, recipient: string, code: string): bool; }\n` +

package/src/devserver/crypto.ts

@@ -17,7 +17,7 @@
 
 import * as nodeCrypto from 'node:crypto';
 
-import { ml_dsa44 } from '@btc-vision/post-quantum/ml-dsa.js';
+import { ml_dsa44 } from '@dacely/noble-post-quantum/ml-dsa.js';
 
 import type { MemoryRef } from './host.js';
 

package/src/devserver/env.ts

@@ -0,0 +1,87 @@
+/**
+ * Dev-only source for `Environment.get` / `getSecure`, mirroring the edge's
+ * per-tenant env store. Reads two optional dotenv files at the project root:
+ *
+ *   - `.env`          -> plain vars  (`Environment.get`); `process.env` overlays
+ *   - `.env.secrets`  -> secrets     (`Environment.getSecure`); keep gitignored
+ *
+ * DISJOINT like the edge: `get` sees only vars, `getSecure` only secrets, so a
+ * secret never comes back through `get`. Framework-reserved `TOIL_*` keys are
+ * host-only and stripped from BOTH buckets (a tenant can't read them).
+ *
+ * Cached after first read; restart `toiljs dev` to pick up edits. The edge
+ * resolves this PER TENANT from `$TOIL_ENV_DIR/<host>.env` + `<host>.env.secrets`
+ * (and the edge DB later) through a lazy, bounded cache; dev has a single
+ * project, so two files are enough and no eviction is needed.
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+
+/** Keys with this prefix are framework-reserved/host-only (never exposed). */
+const RESERVED_PREFIX = 'TOIL_';
+
+interface DevEnv {
+    vars: Map<string, string>;
+    secrets: Map<string, string>;
+}
+
+let cache: DevEnv | null = null;
+
+/** Parse one dotenv value: take inside matching quotes, else cut an inline ` #`. */
+function parseValue(rest: string): string {
+    const q = rest[0];
+    if (q === '"' || q === "'") {
+        const end = rest.indexOf(q, 1);
+        return end < 0 ? rest.slice(1) : rest.slice(1, end);
+    }
+    const hash = rest.indexOf(' #');
+    return (hash < 0 ? rest : rest.slice(0, hash)).trimEnd();
+}
+
+/** Minimal dotenv parser: `KEY=value`, `#` comments, optional `export`, quotes. */
+function parseDotenv(text: string, into: Map<string, string>): void {
+    for (const raw of text.split('\n')) {
+        let line = raw.trim();
+        if (line.length === 0 || line.startsWith('#')) continue;
+        if (line.startsWith('export ')) line = line.slice('export '.length);
+        const eq = line.indexOf('=');
+        if (eq < 0) continue;
+        const key = line.slice(0, eq).trim();
+        if (key.length === 0 || key.startsWith(RESERVED_PREFIX)) continue; // reserved/host-only
+        into.set(key, parseValue(line.slice(eq + 1).trim()));
+    }
+}
+
+function readFileInto(file: string, into: Map<string, string>): void {
+    try {
+        parseDotenv(fs.readFileSync(path.join(process.cwd(), file), 'utf8'), into);
+    } catch {
+        /* file absent: skip */
+    }
+}
+
+function load(): DevEnv {
+    if (cache) return cache;
+    const vars = new Map<string, string>();
+    const secrets = new Map<string, string>();
+    // process.env overlays as plain vars (convenient in dev); never as secrets.
+    for (const [k, v] of Object.entries(process.env)) {
+        if (typeof v === 'string' && !k.startsWith(RESERVED_PREFIX)) vars.set(k, v);
+    }
+    readFileInto('.env', vars);
+    readFileInto('.env.secrets', secrets);
+    cache = { vars, secrets };
+    return cache;
+}
+
+/** A plain var by exact key, or `null`. Reads ONLY the `.env` (vars) bucket. */
+export function devEnvGet(key: string): string | null {
+    const e = load();
+    return e.vars.has(key) ? (e.vars.get(key) as string) : null;
+}
+
+/** A secret by exact key, or `null`. Reads ONLY the `.env.secrets` bucket. */
+export function devEnvGetSecure(key: string): string | null {
+    const e = load();
+    return e.secrets.has(key) ? (e.secrets.get(key) as string) : null;
+}

package/src/devserver/host.ts

@@ -18,6 +18,7 @@
  */
 
 import { buildCryptoImports, freshCryptoState, type CryptoState } from './crypto.js';
+import { devEnvGet, devEnvGetSecure } from './env.js';
 import { ratelimitCheck } from './ratelimit.js';
 
 /** Limits identical to the edge's `set_header` / `respond_file` bounds. */
@@ -105,6 +106,32 @@ function readGuestString(ref: MemoryRef, ptr: number): string {
     return m.toString('utf16le', ptr, ptr + byteLen);
 }
 
+/**
+ * Resolve one `Environment.get`/`getSecure` lookup against the dev env source
+ * and write it into the guest buffer, with the edge's return protocol: the value
+ * byte length (`0` = present-but-empty), `-1` if `outCap` is too small (the guest
+ * retries with a bigger buffer), `-2` if the key is absent.
+ */
+function envLookup(
+    ref: MemoryRef,
+    keyPtr: number,
+    keyLen: number,
+    outPtr: number,
+    outCap: number,
+    secure: boolean,
+): number {
+    const key = readBytes(ref, keyPtr, keyLen).toString('utf8');
+    const val = secure ? devEnvGetSecure(key) : devEnvGet(key);
+    if (val === null) return -2; // ABSENT
+    const bytes = Buffer.from(val, 'utf8');
+    if (bytes.length > outCap) return -1; // TOO_SMALL
+    const m = mem(ref);
+    if (outPtr < 0 || outPtr + bytes.length > m.length)
+        throw new Error('env_get write out of bounds');
+    bytes.copy(m, outPtr);
+    return bytes.length;
+}
+
 /**
  * Build the `env` import object for one instance. `state` collects what the
  * imperative imports produce during a dispatch; bind a fresh state per request.
@@ -200,6 +227,21 @@ export function buildHostImports(ref: MemoryRef, state: DispatchState): WebAssem
                 return 0; // EmailStatus.Sent
             },
 
+            // `Environment.get` / `getSecure`: copy one tenant env value into the
+            // guest buffer. Returns the byte length (0 = present-but-empty), -1 if
+            // the buffer is too small (the guest retries bigger), -2 if absent.
+            // Disjoint buckets: `env_get` reads vars, `env_get_secure` reads
+            // secrets. Mirrors the edge's `env_get_import.rs`; the dev source is
+            // `.env` (+ process.env vars) and `.env.secrets` (see ./env.ts).
+            env_get: (keyPtr: number, keyLen: number, outPtr: number, outCap: number): number =>
+                envLookup(ref, keyPtr, keyLen, outPtr, outCap, false),
+            env_get_secure: (
+                keyPtr: number,
+                keyLen: number,
+                outPtr: number,
+                outCap: number,
+            ): number => envLookup(ref, keyPtr, keyLen, outPtr, outCap, true),
+
             thread_spawn: (_startArg: number): number => -1,
 
             // `Date.now()` -> wall-clock milliseconds, matching the edge host.

package/src/devserver/module.ts

@@ -53,7 +53,7 @@ interface HandleExports {
 /** Host functions the dev server provides under `env` (see `host.ts`). */
 const PROVIDED_IMPORTS = new Set([
     'abort', 'set_status', 'set_header', 'respond_file', 'thread_spawn', 'Date.now',
-    'client_ip', 'ratelimit_check', 'email_send',
+    'client_ip', 'ratelimit_check', 'email_send', 'env_get', 'env_get_secure',
     // Web Crypto host functions (see ./crypto.ts).
     'crypto.fill_random', 'crypto.random_uuid', 'crypto.take_result', 'crypto.digest',
     'crypto.import_key', 'crypto.export_key', 'crypto.encrypt', 'crypto.decrypt',