vectrojs 0.0.0-stage → 0.1.0

package/src/rows.mjs

@@ -0,0 +1,91 @@
+// Copying rows between tables: the rows of several tables (or ranges of
+// them) one after another as a new table, with category labels merged and
+// free text given its own bytes. Used by datasets (chunks, spills) and by
+// set operations (UNION).
+import { Table } from './table.mjs';
+import { decodeOne } from './pack.mjs';
+import { STR_SHIFT, STR_ESCAPED } from './csv-kernel.mjs';
+
+export const textLength = (v) => { const field = v - Math.floor(v / STR_SHIFT) * STR_SHIFT; return field >= STR_ESCAPED ? field - STR_ESCAPED : field; };
+// Rows [lo, hi) of column c as plain numbers in out (packed: decoded).
+export function plainInto(engine, c, lo, hi, out, f32) {
+  const src = engine.live(c.data);
+  if (!c.enc) { out.set(src.subarray(lo, hi)); return; }
+  for (let i = lo; i < hi; i++) out[i - lo] = decodeOne(c.enc, src[i], f32);
+}
+
+// A table's category codes as codes of the dataset's dictionary (new labels added).
+export function remapCodes(data, dict, labels) {
+  if (dict === labels.dict) return;
+  const lut = new Float64Array(dict.length);
+  let same = dict.length <= labels.dict.length;
+  dict.forEach((s, j) => {
+    let g = labels.codes.get(s);
+    if (g === undefined) { g = labels.dict.length; labels.dict.push(s); labels.codes.set(s, g); }
+    lut[j] = g;
+    if (g !== j) same = false;
+  });
+  if (same) return;
+  for (let i = 0; i < data.length; i++) { const v = data[i]; data[i] = v === v ? lut[v] : NaN; }
+}
+
+const sameDict = (a, b) => a === b || (a.length === b.length && a.every((s, i) => s === b[i]));
+
+// The rows [lo, hi) of each part, one after another, as a new table (owned,
+// plain columns; categories share a dictionary; free text gets its own bytes).
+export function copyRows(engine, parts) {
+  const first = parts[0][0];
+  const T = first.type;
+  const n = parts.reduce((a, [, lo, hi]) => a + (hi - lo), 0);
+  const cols = new Map(), owned = [], ownedBuffers = [];
+  const table = new Table(engine, cols, n, { owned, ownedBuffers });
+  try {
+    for (const name of first.columnNames) {
+      const c0 = first.cols.get(name);
+      if (c0.strings) {
+        let total = 0;
+        for (const [t, lo, hi] of parts) { const p = engine.live(t.cols.get(name).data); for (let i = lo; i < hi; i++) if (p[i] === p[i]) total += textLength(p[i]); }
+        const data = engine.alloc(Float64Array, n), strings = engine.alloc(Uint8Array, total);
+        cols.set(name, { data, dict: null, codes: null, strings });
+        owned.push(name);
+        ownedBuffers.push(strings);
+        const o = engine.live(data), sb = engine.live(strings);
+        let at = 0, row = 0;
+        for (const [t, lo, hi] of parts) {
+          const c = t.cols.get(name), p = engine.live(c.data), b = engine.live(c.strings);
+          for (let i = lo; i < hi; i++, row++) {
+            const v = p[i];
+            if (v !== v) { o[row] = NaN; continue; }
+            const start = Math.floor(v / STR_SHIFT), len = textLength(v);
+            sb.set(b.subarray(start, start + len), at);
+            o[row] = at * STR_SHIFT + (v - start * STR_SHIFT);
+            at += len;
+          }
+        }
+        continue;
+      }
+      const data = engine.alloc(T, n);
+      let dict = null, codes = null;
+      if (c0.dict) {
+        dict = c0.dict;
+        if (parts.some(([t]) => !sameDict(t.cols.get(name).dict, dict))) dict = [...dict];
+        codes = dict === c0.dict ? c0.codes : new Map(dict.map((s, i) => [s, i]));
+      }
+      cols.set(name, { data, dict, codes, date: c0.date });
+      owned.push(name);
+      let row = 0;
+      for (const [t, lo, hi] of parts) {
+        const c = t.cols.get(name), o = engine.live(data).subarray(row, row + hi - lo);
+        plainInto(engine, c, lo, hi, o, T === Float32Array);
+        if (dict && c.dict !== dict && !sameDict(c.dict, dict)) remapCodes(o, c.dict, { dict, codes });
+        row += hi - lo;
+      }
+    }
+    for (const c of cols.values()) { c.data = engine.live(c.data); if (c.strings) c.strings = engine.live(c.strings); }
+    return table;
+  } catch (err) {
+    table.dispose();
+    throw err;
+  }
+}
+

package/src/serve/catalog.mjs

@@ -0,0 +1,148 @@
+// Tables from files in a data folder, for the servers (the HTTP SQL server
+// in http.mjs, the MCP server in mcp/): CSV / TSV, JSON lines (with a vector
+// index when the lines carry embeddings), Arrow / Feather files and .vectro
+// datasets (opened on disk, not loaded). Files outside the folder are refused.
+import { readFileSync, readdirSync, realpathSync, statSync } from 'node:fs';
+import { resolve, sep, extname, basename, join } from 'node:path';
+import { VectorIndex } from '../vectors.mjs';
+import { Engine } from '../node.mjs';
+import { parseNDJSON } from '../ndjson.mjs';
+
+export const MAX_FILE_BYTES = 2 ** 31;
+export const FREE_TEXT_ROWS = 5_000_000;
+export const LOADABLE = new Set(['.csv', '.tsv', '.jsonl', '.ndjson', '.arrow', '.feather', '.ipc', '.vectro']);
+
+// (a mistake by whoever asked: shown to them as it is)
+export class LoadError extends Error {}
+
+export const kindOf = (c) => (c.strings ? 'text' : c.dict ? 'category' : c.date ? 'date' : 'number');
+export const columnsOf = (t) => Object.fromEntries([...t.cols].map(([n, c]) => [n, kindOf(c)]));
+export const tableName = (file) => basename(file, extname(file)).replace(/[^A-Za-z0-9_]/g, '_').replace(/^(\d)/, '_$1');
+
+// A path inside root (symbolic links resolved), or a LoadError.
+export function safePath(root, path) {
+  if (typeof path !== 'string' || !path) throw new LoadError('path is required');
+  let real;
+  try { real = realpathSync(resolve(root, path)); } catch { throw new LoadError(`no such file: ${path}`); }
+  if (real !== root && !real.startsWith(root + sep)) throw new LoadError('that file is outside the data folder this server may read');
+  const st = statSync(real);
+  if (!st.isFile()) throw new LoadError(`not a file: ${path}`);
+  if (st.size > MAX_FILE_BYTES && extname(real).toLowerCase() !== '.vectro') throw new LoadError('file is larger than 2 GB (convert it to a .vectro dataset)');
+  return real;
+}
+
+// One file as a table: { table, index?, embedding?, skipped?, onDisk? }.
+export async function loadFile(engine, real, { types } = {}) {
+  const ext = extname(real).toLowerCase();
+  if (ext === '.vectro') return { table: engine.openDataset(real), onDisk: true };
+  const bytes = new Uint8Array(readFileSync(real));
+  if (ext === '.arrow' || ext === '.feather' || ext === '.ipc') return { table: engine.fromArrow(bytes) };
+  if (ext === '.jsonl' || ext === '.ndjson') return loadJSONLines(engine, bytes, types);
+  if (ext === '.csv' || ext === '.tsv' || ext === '.txt') {
+    const delimiter = ext === '.tsv' ? '\t' : ',';
+    const table = engine.parseCSV(bytes, { types: types ?? {}, delimiter });
+    // text that rarely repeats (names, titles) loads as free text, which can be
+    // shown but not grouped, filtered or joined on: tools need that, so up to
+    // FREE_TEXT_ROWS rows it is loaded again as categories
+    const free = [...table.cols].filter(([n, c]) => c.strings && !(types && n in types)).map(([n]) => n);
+    if (!free.length || table.length > FREE_TEXT_ROWS) return { table };
+    table.dispose();
+    return { table: engine.parseCSV(bytes, { types: { ...types, ...Object.fromEntries(free.map((n) => [n, 'category'])) }, delimiter }) };
+  }
+  throw new LoadError(`can't load ${ext || 'files without an extension'} (CSV, TSV, JSON lines, Arrow / Feather or .vectro)`);
+}
+
+// Every loadable file directly in root: [{ name, path }].
+export function filesIn(root) {
+  return readdirSync(root, { withFileTypes: true })
+    .filter((d) => d.isFile() && LOADABLE.has(extname(d.name).toLowerCase()))
+    .map((d) => ({ name: tableName(d.name), path: join(root, d.name) }));
+}
+
+// A table's memory back (datasets closed).
+export function release(table) { if (table?.isDataset) table.close(); else table?.dispose(); }
+
+async function loadWithEmbeddings(engine, bytes, field, dim, kinds) {
+  const names = [...kinds].filter(([k, v]) => v && k !== field).map(([k]) => k);
+  const cols = Object.fromEntries(names.map((n) => [n, []]));
+  const vectors = [];
+  let skipped = 0;
+  for (const line of new TextDecoder().decode(bytes).split('\n')) {
+    if (!line.trim()) continue;
+    let o;
+    try { o = JSON.parse(line); } catch { skipped++; continue; }
+    const v = o?.[field];
+    if (!o || typeof o !== 'object' || !Array.isArray(v) || v.length !== dim) { skipped++; continue; }
+    vectors.push(v);
+    for (const n of names) {
+      const x = o[n];
+      cols[n].push(kinds.get(n) === 'number' ? (typeof x === 'number' ? x : typeof x === 'boolean' ? +x : typeof x === 'string' && x.trim() ? Number(x) : NaN) : x == null || typeof x === 'object' ? '' : String(x));
+    }
+  }
+  const table = engine.table(cols);
+  const index = await VectorIndex.create(engine, { vectors, dim, metric: 'cosine', columns: cols });
+  return { table, index, skipped, embedding: { field, dim } };
+}
+
+function loadJSONLines(engine, bytes, types) {
+  // column names and kinds from the first 1,000 lines
+  const kinds = new Map(), arrays = new Map();
+  const text = new TextDecoder().decode(bytes.subarray(0, Math.min(bytes.length, 1 << 20)));
+  for (const line of text.split('\n').slice(0, 1000)) {
+    let o;
+    try { o = JSON.parse(line); } catch { continue; }
+    if (!o || typeof o !== 'object' || Array.isArray(o)) continue;
+    for (const [k, v] of Object.entries(o)) {
+      if (Array.isArray(v) && v.length && v.every((x) => typeof x === 'number')) { if (!arrays.has(k)) arrays.set(k, v.length); }
+      if (v === null || typeof v === 'object') { if (!kinds.has(k)) kinds.set(k, null); continue; }
+      const kind = typeof v === 'number' || typeof v === 'boolean' ? 'number' : 'category';
+      if (kinds.get(k) !== 'category') kinds.set(k, kind);
+    }
+  }
+  for (const [k, v] of Object.entries(types ?? {})) kinds.set(k, v === 'number' ? 'number' : 'category');
+  if (arrays.size) { const [field, dim] = arrays.entries().next().value; return loadWithEmbeddings(engine, bytes, field, dim, kinds); }
+  const names = [...kinds].filter(([, v]) => v).map(([k]) => k);
+  if (!names.length) throw new LoadError('no JSON objects with plain values found in the first lines');
+  const r = parseNDJSON(bytes, 0, bytes.length, names, names.map((n) => kinds.get(n)));
+  const cols = Object.fromEntries(names.map((n, k) => [n, r.dicts[k] ? Array.from(r.cols[k], (code) => r.dicts[k][code]) : r.cols[k]]));
+  return { table: engine.table(cols), skipped: r.bad };
+}
+
+// The tables a server serves: given in code, or the files in a data folder
+// (reload() reads it again). Shared by the HTTP and PostgreSQL servers.
+//   { engine, entries: Map name -> { table, source, owned }, named(), reload(), version, close() }
+export async function openTables({ tables = {}, root = null, engine = null, log = () => {} } = {}) {
+  const own = !engine && !Object.values(tables).some((t) => t?.engine);
+  engine ??= Object.values(tables).find((t) => t?.engine)?.engine ?? new Engine();
+  const entries = new Map();
+  for (const [name, table] of Object.entries(tables)) entries.set(name, { table, source: 'given', owned: false });
+  const rootPath = root ? realpathSync(resolve(String(root).replace(/^~(?=\/|$)/, process.env.HOME ?? '~'))) : null;
+  const self = {
+    engine, entries, root: rootPath, version: 0,
+    named: () => Object.fromEntries([...entries].map(([n, e]) => [n, e.table])),
+    async reload() {
+      if (!rootPath) throw new LoadError('this server has no data folder to reload');
+      for (const [name, e] of entries) if (e.owned) { release(e.table); entries.delete(name); }
+      const loaded = [], failed = [];
+      for (const { name, path } of filesIn(rootPath)) {
+        if (entries.has(name)) continue; // (tables given in code win)
+        try {
+          const { table } = await loadFile(engine, path);
+          entries.set(name, { table, source: path, owned: true });
+          loaded.push(name);
+        } catch (err) { failed.push({ file: path, error: err.message }); log(`skipped ${path}: ${err.message}`); }
+      }
+      self.version++;
+      return { loaded, failed };
+    },
+    async close() {
+      for (const e of entries.values()) if (e.owned) release(e.table);
+      entries.clear();
+      if (own) await engine.close();
+    },
+  };
+  if (rootPath) await self.reload();
+  return self;
+}
+
+export { realpathSync };

package/src/serve/flight.mjs

@@ -0,0 +1,270 @@
+// Arrow Flight SQL over Vectro tables: pandas, Polars, DuckDB and anything
+// with an ADBC or Flight SQL driver ask in SQL and get Arrow back, column by
+// column, with no conversion to rows on either side.
+//
+//   import { serveFlight } from 'vectrojs/serve';
+//   await serveFlight({ tables: { orders }, port: 31337 });     // grpc://localhost:31337
+//
+//   # Python: pip install adbc-driver-flightsql
+//   import adbc_driver_flightsql.dbapi as flight
+//   with flight.connect('grpc://localhost:31337') as c: c.cursor().execute('SELECT ...').fetch_arrow_table()
+//
+// gRPC over HTTP/2 (node:http2; TLS with a certificate) and protobuf are
+// written here; no dependencies. Served: statements and prepared statements
+// (the SQL travels in the ticket), catalogs, schemas, tables (with their
+// Arrow schemas), table types; keys are empty. Read only. With a password,
+// clients sign in with Handshake (basic) and get a bearer token.
+import { createServer, createSecureServer, constants } from 'node:http2';
+import { randomBytes, timingSafeEqual } from 'node:crypto';
+import { readFileSync } from 'node:fs';
+import { once } from 'node:events';
+import { gunzipSync } from 'node:zlib';
+import { sql, columnType, displayName } from '../sql.mjs';
+import { typedSchema, typedBatch, typedDictionary } from '../arrow.mjs';
+import { openTables } from './catalog.mjs';
+
+const BATCH = 65_536;
+const LOOPBACK = new Set(['127.0.0.1', '::1', 'localhost']);
+const SQL_PREFIX = 'type.googleapis.com/arrow.flight.protocol.sql.';
+const STATUS = { ok: 0, invalid: 3, notFound: 5, unimplemented: 12, internal: 13, unauthenticated: 16 };
+class FlightError extends Error { constructor(code, message) { super(message); this.code = code; } }
+const fail = (code, m) => { throw new FlightError(code, m); };
+
+// ---- protobuf ----------------------------------------------------------------------------
+// decode: Map field number -> [values] (varints as numbers / BigInt-safe Numbers,
+// length-delimited as Buffers); encode: [[field, kind, value]], kind 'v' (varint),
+// 'b' (bytes / string / encoded message), 'q' (fixed64)
+export function pbDecode(buf) {
+  const out = new Map();
+  let i = 0;
+  const varint = () => { let x = 0n, s = 0n; for (;;) { const b = buf[i++]; x |= BigInt(b & 0x7f) << s; if (!(b & 0x80)) return x; s += 7n; } };
+  while (i < buf.length) {
+    const key = Number(varint()), field = key >>> 3, wire = key & 7;
+    let v;
+    if (wire === 0) v = Number(BigInt.asIntN(64, varint()));
+    else if (wire === 2) { const n = Number(varint()); v = buf.subarray(i, i + n); i += n; }
+    else if (wire === 1) { v = buf.readBigInt64LE(i); i += 8; }
+    else if (wire === 5) { v = buf.readInt32LE(i); i += 4; }
+    else throw new FlightError(STATUS.invalid, `bad protobuf wire type ${wire}`);
+    if (!out.has(field)) out.set(field, []);
+    out.get(field).push(v);
+  }
+  return out;
+}
+const pbVarint = (n) => { const out = []; let x = BigInt.asUintN(64, BigInt(n)); do { let b = Number(x & 0x7fn); x >>= 7n; if (x) b |= 0x80; out.push(b); } while (x); return Buffer.from(out); };
+export function pbEncode(fields) {
+  const parts = [];
+  for (const [field, kind, value] of fields) {
+    if (value === undefined || value === null) continue;
+    if (kind === 'v') { parts.push(pbVarint(field << 3), pbVarint(value)); continue; }
+    if (kind === 'q') { const b = Buffer.alloc(8); b.writeBigInt64LE(BigInt(value)); parts.push(pbVarint((field << 3) | 1), b); continue; }
+    const b = Buffer.isBuffer(value) ? value : value instanceof Uint8Array ? Buffer.from(value.buffer, value.byteOffset, value.byteLength) : Buffer.from(String(value), 'utf8');
+    parts.push(pbVarint((field << 3) | 2), pbVarint(b.length), b);
+  }
+  return Buffer.concat(parts);
+}
+const str = (m, f) => (m.get(f)?.[0] ? Buffer.from(m.get(f)[0]).toString('utf8') : null);
+const bytesOf = (m, f) => m.get(f)?.[0] ?? null;
+const any = (type, value) => pbEncode([[1, 'b', SQL_PREFIX + type], [2, 'b', value]]);
+function unpackAny(buf) {
+  const m = pbDecode(buf);
+  const url = str(m, 1) ?? '';
+  return { type: url.startsWith(SQL_PREFIX) ? url.slice(SQL_PREFIX.length) : url, value: pbDecode(bytesOf(m, 2) ?? Buffer.alloc(0)), raw: bytesOf(m, 2) ?? Buffer.alloc(0) };
+}
+
+// ---- results: Arrow fields and columns --------------------------------------------------------
+const ARROW = { int: 'int64', float: 'float64', bool: 'bool', text: 'utf8', date: 'timestamp', day: 'date32' };
+const TABLE_TYPES = ['TABLE'];
+// a Flight data message: header (flatbuffer Message) and body, from an IPC message
+const flightData = (m) => pbEncode([[2, 'b', m.bytes.subarray(8, m.metaLength)], [1000, 'b', m.bytes.subarray(m.metaLength)]]);
+
+export async function serveFlight({ tables = {}, root = null, engine = null, catalog = null, port = 31337, host = '127.0.0.1', user = null, password = null, tls = null, database = 'vectro', log = () => {}, logQueries = false } = {}) {
+  if (!password && !LOOPBACK.has(host)) throw new Error('vectro: a password is needed to accept Flight connections from other machines (--password, or --host 127.0.0.1)');
+  const shared = !!catalog;
+  catalog ??= await openTables({ tables, root, engine, log });
+  const tokens = new Set();
+  const named = () => Object.fromEntries([...catalog.entries].map(([n, e]) => [n, e.table]));
+
+  const checkBasic = (header) => {
+    const m = /^Basic\s+(.+)$/i.exec(header ?? '');
+    if (!m) return false;
+    const [u, ...rest] = Buffer.from(m[1], 'base64').toString().split(':');
+    const given = Buffer.from(rest.join(':')), want = Buffer.from(String(password));
+    return (!user || u === user) && given.length === want.length && timingSafeEqual(given, want);
+  };
+  const authorized = (headers) => {
+    if (!password) return true;
+    const h = headers.authorization ?? '';
+    const bearer = /^Bearer\s+(.+)$/i.exec(h);
+    return (bearer && tokens.has(bearer[1])) || checkBasic(h);
+  };
+
+  // a query's Arrow fields (from its column types, before it runs)
+  const compile = (text) => {
+    let q;
+    try { q = sql(text, named(), { types: true, foldCase: true, database }); } catch (err) { fail(STATUS.invalid, err.message.replace(/^vectro( SQL)?:\s*/, '')); }
+    return { q, fields: q.columns.map((c) => ({ name: displayName(c.name), col: c.name, type: ARROW[c.type] ?? 'float64' })) };
+  };
+  const tableFields = (t) => t.columnNames.map((c) => ({ name: displayName(c), type: ARROW[columnType(t, c)] ?? 'float64' }));
+
+  // catalog answers: { fields, columns (arrays), n }
+  const like = (pattern) => (pattern == null ? () => true : ((re) => (s) => re.test(s))(new RegExp(`^${pattern.replace(/[.*+?^${}()|[\]\\]/g, '\\$&').replace(/%/g, '.*').replace(/_/g, '.')}$`, 's')));
+  function catalogAnswer(type, m) {
+    const fieldsOf = (spec) => spec.map(([name, t, nullable = true]) => ({ name, type: t, nullable }));
+    switch (type) {
+      case 'CommandGetCatalogs': return { fields: fieldsOf([['catalog_name', 'utf8', false]]), columns: [[database]], n: 1 };
+      case 'CommandGetDbSchemas': {
+        const ok = (str(m, 1) ?? database) === database && like(str(m, 2))('public');
+        return { fields: fieldsOf([['catalog_name', 'utf8'], ['db_schema_name', 'utf8', false]]), columns: ok ? [[database], ['public']] : [[], []], n: ok ? 1 : 0 };
+      }
+      case 'CommandGetTableTypes': return { fields: fieldsOf([['table_type', 'utf8', false]]), columns: [TABLE_TYPES], n: TABLE_TYPES.length };
+      case 'CommandGetTables': {
+        const wantTypes = (m.get(4) ?? []).map((b) => Buffer.from(b).toString());
+        const inCatalog = (str(m, 1) ?? database) === database && like(str(m, 2))('public') && (!wantTypes.length || wantTypes.includes('TABLE'));
+        const names = inCatalog ? Object.keys(named()).filter((n) => !n.includes('.')).filter(like(str(m, 3))).sort() : [];
+        const withSchema = !!m.get(5)?.[0];
+        const spec = [['catalog_name', 'utf8'], ['db_schema_name', 'utf8'], ['table_name', 'utf8', false], ['table_type', 'utf8', false]];
+        if (withSchema) spec.push(['table_schema', 'binary', false]);
+        const columns = [names.map(() => database), names.map(() => 'public'), names, names.map(() => 'TABLE')];
+        if (withSchema) columns.push(names.map((n) => typedSchema(tableFields(named()[n])).bytes));
+        return { fields: fieldsOf(spec), columns, n: names.length };
+      }
+      case 'CommandGetPrimaryKeys': return { fields: fieldsOf([['catalog_name', 'utf8'], ['db_schema_name', 'utf8'], ['table_name', 'utf8', false], ['column_name', 'utf8', false], ['key_name', 'utf8'], ['key_sequence', 'int32', false]]), columns: [[], [], [], [], [], []], n: 0 };
+      case 'CommandGetImportedKeys': case 'CommandGetExportedKeys': case 'CommandGetCrossReference':
+        return { fields: fieldsOf([['pk_catalog_name', 'utf8'], ['pk_db_schema_name', 'utf8'], ['pk_table_name', 'utf8', false], ['pk_column_name', 'utf8', false], ['fk_catalog_name', 'utf8'], ['fk_db_schema_name', 'utf8'], ['fk_table_name', 'utf8', false], ['fk_column_name', 'utf8', false], ['key_sequence', 'int32', false], ['fk_key_name', 'utf8'], ['pk_key_name', 'utf8'], ['update_rule', 'int8', false], ['delete_rule', 'int8', false]]), columns: Array.from({ length: 13 }, () => []), n: 0 };
+      default: return null;
+    }
+  }
+  const CATALOG = new Set(['CommandGetCatalogs', 'CommandGetDbSchemas', 'CommandGetTableTypes', 'CommandGetTables', 'CommandGetPrimaryKeys', 'CommandGetImportedKeys', 'CommandGetExportedKeys', 'CommandGetCrossReference']);
+
+  // FlightDescriptor -> { fields, ticket }
+  function describe(desc) {
+    const d = pbDecode(desc);
+    if (d.get(1)?.[0] !== 2) fail(STATUS.invalid, 'only command descriptors (Flight SQL) are served');
+    const cmd = unpackAny(bytesOf(d, 2) ?? Buffer.alloc(0));
+    if (cmd.type === 'CommandStatementQuery' || cmd.type === 'CommandPreparedStatementQuery') {
+      const text = cmd.type === 'CommandStatementQuery' ? str(cmd.value, 1) : Buffer.from(bytesOf(cmd.value, 1) ?? []).toString();
+      if (!text) fail(STATUS.invalid, 'no SQL given');
+      if (logQueries) log(`[flight] ${text.replace(/\s+/g, ' ').slice(0, 2000)}`);
+      const { fields } = compile(text);
+      return { fields, ticket: any('TicketStatementQuery', pbEncode([[1, 'b', text]])), desc };
+    }
+    if (CATALOG.has(cmd.type)) return { fields: catalogAnswer(cmd.type, cmd.value).fields, ticket: bytesOf(d, 2), desc };
+    if (cmd.type === 'CommandStatementUpdate' || cmd.type === 'CommandPreparedStatementUpdate') fail(STATUS.invalid, 'Vectro serves its tables read only');
+    fail(STATUS.unimplemented, `${cmd.type || 'this command'} is not supported`);
+  }
+  const flightInfo = ({ fields, ticket, desc }) => pbEncode([
+    [1, 'b', typedSchema(fields).bytes], [2, 'b', desc],
+    [3, 'b', pbEncode([[1, 'b', pbEncode([[1, 'b', ticket]])]])],
+    [4, 'v', -1], [5, 'v', -1],
+  ]);
+
+  // DoGet: the rows as FlightData messages
+  async function* rowsOf(ticket) {
+    const t = unpackAny(ticket);
+    let fields, columns, n, table = null;
+    if (t.type === 'TicketStatementQuery') {
+      const text = Buffer.from(bytesOf(t.value, 1) ?? []).toString();
+      const c = compile(text);
+      fields = c.fields;
+      table = await c.q.collectAsync();
+      const data = await table.toColumnsAsync();
+      columns = fields.map((f) => data[f.col]);
+      n = table.length;
+      table.dispose();
+    } else if (CATALOG.has(t.type)) ({ fields, columns, n } = catalogAnswer(t.type, t.value));
+    else fail(STATUS.invalid, 'unknown ticket');
+    yield flightData(typedSchema(fields));
+    for (let a = 0; a < n || a === 0; a += BATCH) {
+      yield flightData(typedBatch(fields, columns, a, Math.min(n, a + BATCH)));
+      if (n === 0) break;
+    }
+  }
+
+  // ---- gRPC ----
+  const frame = (msg) => { const h = Buffer.alloc(5); h.writeUInt32BE(msg.length, 1); return Buffer.concat([h, msg]); };
+  function messagesOf(buf) {
+    const out = [];
+    for (let i = 0; i + 5 <= buf.length;) {
+      const compressed = buf[i], len = buf.readUInt32BE(i + 1);
+      const body = buf.subarray(i + 5, i + 5 + len);
+      out.push(compressed ? gunzipSync(body) : body);
+      i += 5 + len;
+    }
+    return out;
+  }
+  const respondError = (stream, code, message) => {
+    if (stream.destroyed) return;
+    const trailers = { 'grpc-status': String(code), 'grpc-message': encodeURIComponent(message).slice(0, 4000) };
+    if (stream.headersSent) { stream.sendTrailers?.(trailers); return; }
+    stream.respond({ ':status': 200, 'content-type': 'application/grpc', ...trailers }, { endStream: true });
+  };
+  async function reply(stream, headers, gen) {
+    stream.respond({ ':status': 200, 'content-type': 'application/grpc', ...headers }, { waitForTrailers: true });
+    let status = 0, message = '';
+    try {
+      for await (const msg of gen) if (!stream.write(frame(msg))) await once(stream, 'drain');
+    } catch (err) { status = err.code ?? STATUS.internal; message = err.message; if (!(err instanceof FlightError)) log(`[flight] error: ${err.stack}`); }
+    stream.once('wantTrailers', () => stream.sendTrailers({ 'grpc-status': String(status), ...(message ? { 'grpc-message': encodeURIComponent(message).slice(0, 4000) } : {}) }));
+    stream.end();
+  }
+
+  const ACTIONS = [['CreatePreparedStatement', 'a prepared statement: its handle and schema'], ['ClosePreparedStatement', 'forget a prepared statement']];
+  async function handle(stream, headers) {
+    const method = String(headers[':path'] ?? '').replace('/arrow.flight.protocol.FlightService/', '');
+    const chunks = [];
+    for await (const c of stream) chunks.push(c);
+    const msgs = messagesOf(Buffer.concat(chunks));
+    if (method === 'Handshake') { // basic sign-in: a bearer token in the answer's headers
+      if (password && !checkBasic(headers.authorization)) return respondError(stream, STATUS.unauthenticated, 'wrong user name or password');
+      const token = randomBytes(24).toString('base64url');
+      tokens.add(token);
+      if (tokens.size > 10_000) tokens.delete(tokens.values().next().value);
+      return reply(stream, password ? { authorization: `Bearer ${token}` } : {}, (async function* () { yield pbEncode([[1, 'v', 0], [2, 'b', Buffer.alloc(0)]]); })());
+    }
+    if (!authorized(headers)) return respondError(stream, STATUS.unauthenticated, 'sign in first (Handshake with a user name and password)');
+    const one = msgs[0] ?? Buffer.alloc(0);
+    switch (method) {
+      case 'GetFlightInfo': return reply(stream, {}, (async function* () { yield flightInfo(describe(one)); })());
+      case 'GetSchema': return reply(stream, {}, (async function* () { yield pbEncode([[1, 'b', typedSchema(describe(one).fields).bytes]]); })());
+      case 'DoGet': return reply(stream, {}, rowsOf(bytesOf(pbDecode(one), 1) ?? Buffer.alloc(0)));
+      case 'ListActions': return reply(stream, {}, (async function* () { for (const [type, description] of ACTIONS) yield pbEncode([[1, 'b', type], [2, 'b', description]]); })());
+      case 'DoAction': {
+        const a = pbDecode(one), type = str(a, 1), body = unpackAny(bytesOf(a, 2) ?? Buffer.alloc(0));
+        if (type === 'CreatePreparedStatement') {
+          const text = str(body.value, 1) ?? '';
+          const { fields } = compile(text);
+          const result = pbEncode([[1, 'b', text], [2, 'b', typedSchema(fields).bytes]]);
+          return reply(stream, {}, (async function* () { yield pbEncode([[1, 'b', any('ActionCreatePreparedStatementResult', result)]]); })());
+        }
+        if (type === 'ClosePreparedStatement') return reply(stream, {}, (async function* () {})());
+        return respondError(stream, STATUS.unimplemented, `action ${type} is not supported (Vectro serves its tables read only)`);
+      }
+      case 'ListFlights': return reply(stream, {}, (async function* () {})());
+      case 'DoPut': return respondError(stream, STATUS.unimplemented, 'Vectro serves its tables read only (no DoPut, no parameters for prepared statements yet)');
+      default: return respondError(stream, STATUS.unimplemented, `${method} is not supported`);
+    }
+  }
+
+  const server = tls ? createSecureServer({ cert: tls.cert ?? readFileSync(tls.certFile), key: tls.key ?? readFileSync(tls.keyFile) }) : createServer();
+  server.on('stream', (stream, headers) => {
+    stream.on('error', () => {});
+    handle(stream, headers).catch((err) => { if (!(err instanceof FlightError)) log(`[flight] error: ${err.stack}`); respondError(stream, err.code ?? STATUS.internal, err.message); });
+  });
+  server.on('sessionError', () => {});
+  server.listen(port, host);
+  await once(server, 'listening');
+  const actual = server.address().port;
+  const sessions = new Set();
+  server.on('session', (s) => { sessions.add(s); s.on('close', () => sessions.delete(s)); });
+  return {
+    port: actual, host, server, catalog,
+    url: `${tls ? 'grpc+tls' : 'grpc'}://${host.includes(':') ? `[${host}]` : host}:${actual}`,
+    async close() {
+      for (const s of sessions) s.destroy();
+      await new Promise((r) => server.close(r));
+      if (!shared) await catalog.close();
+    },
+  };
+}
+export { constants as http2Constants };

package/src/serve/formats.mjs

@@ -0,0 +1,87 @@
+// Query results as bytes for HTTP clients: JSON, CSV, JSON lines and Arrow,
+// written a slice of rows at a time (a large result never becomes one string).
+//   columns: [{ name, type: 'number' | 'text' | 'date' }]
+//   data: table.toColumns() (numbers as typed arrays, text as { codes, labels } or strings)
+// Each cell is text made once per label (a category's labels are few), or a
+// number / date written straight away.
+import { writeArrow } from '../arrow.mjs';
+import { displayName } from '../sql.mjs';
+
+const SLICE = 65_536;
+
+export const FORMATS = {
+  json: 'application/json; charset=utf-8',
+  csv: 'text/csv; charset=utf-8',
+  ndjson: 'application/x-ndjson; charset=utf-8',
+  arrow: 'application/vnd.apache.arrow.stream',
+};
+
+export const columnsOf = (table) => [...table.cols].map(([name, c]) => ({ name: displayName(name), col: name, type: c.dict || c.strings ? 'text' : c.date ? 'date' : 'number' }));
+
+// RFC 4180: quoted when it holds a comma, a quote or a line break
+const csvText = (v) => (/[",\r\n]/.test(v) ? `"${v.replaceAll('"', '""')}"` : v);
+
+// ISO 8601 in UTC, as Date.toISOString writes it, with each day's part made once
+const days = new Map();
+const p2 = (x) => (x < 10 ? `0${x}` : `${x}`);
+function iso(ms) {
+  const day = Math.floor(ms / 86_400_000);
+  let d = days.get(day);
+  if (d === undefined) { if (days.size > 100_000) days.clear(); const x = new Date(day * 86_400_000).toISOString(); d = x.slice(0, x.indexOf('T') + 1); days.set(day, d); }
+  let t = ms - day * 86_400_000;
+  const h = Math.floor(t / 3_600_000); t -= h * 3_600_000;
+  const m = Math.floor(t / 60_000); t -= m * 60_000;
+  const s = Math.floor(t / 1000), f = Math.floor(t - s * 1000);
+  return `${d}${p2(h)}:${p2(m)}:${p2(s)}.${f < 10 ? `00${f}` : f < 100 ? `0${f}` : f}Z`;
+}
+const dateOK = (v) => v === v && v > -8.64e15 && v < 8.64e15;
+
+// cell(i) for one column, as text for the format; null cells as `missing`
+function cells(col, d, format) {
+  const missing = format === 'csv' ? '' : 'null';
+  const quote = format === 'csv' ? csvText : JSON.stringify;
+  if (d && d.labels) {
+    const text = d.labels.map((s) => (s === '' ? missing : quote(s)));
+    const codes = d.codes;
+    return (i) => text[codes[i]] ?? missing;
+  }
+  if (col.type === 'text') return (i) => (d[i] === '' ? missing : quote(d[i]));
+  if (col.type === 'date') return format === 'csv' ? (i) => (dateOK(d[i]) ? iso(d[i]) : missing) : (i) => (dateOK(d[i]) ? `"${iso(d[i])}"` : missing);
+  return (i) => { const v = d[i]; return v === v && v !== Infinity && v !== -Infinity ? `${v}` : missing; };
+}
+
+// The body as chunks (strings or bytes), for res.write one after another.
+export function* encode(format, columns, data, n, extra = {}) {
+  if (format === 'arrow') {
+    yield writeArrow(Object.fromEntries(columns.map((c) => [c.name, data[c.col ?? c.name]])), { dates: columns.filter((c) => c.type === 'date').map((c) => c.name) });
+    return;
+  }
+  const cell = columns.map((c) => cells(c, data[c.col ?? c.name], format));
+  const K = cell.length;
+  const rows = (a, b, start, sep, end, keys) => {
+    let s = '';
+    for (let i = a; i < b; i++) {
+      s += start;
+      for (let k = 0; k < K; k++) s += (k ? sep : '') + (keys ? keys[k] : '') + cell[k](i);
+      s += end;
+    }
+    return s;
+  };
+  if (format === 'csv') {
+    yield `${columns.map((c) => csvText(c.name)).join(',')}\r\n`;
+    for (let a = 0; a < n; a += SLICE) yield rows(a, Math.min(n, a + SLICE), '', ',', '\r\n', null);
+    return;
+  }
+  if (format === 'ndjson') {
+    const keys = columns.map((c) => `${JSON.stringify(c.name)}:`);
+    for (let a = 0; a < n; a += SLICE) yield rows(a, Math.min(n, a + SLICE), '{', ',', '}\n', keys);
+    return;
+  }
+  // json: { columns, rows: [[...]...], ... }
+  yield `{"columns":${JSON.stringify(columns.map(({ name, type }) => ({ name, type })))},"rows":[`;
+  for (let a = 0; a < n; a += SLICE) {
+    const s = rows(a, Math.min(n, a + SLICE), '[', ',', '],', null);
+    yield a + SLICE >= n ? s.slice(0, -1) : s; // (no comma after the last row)
+  }
+  yield Object.keys(extra).length ? `],${JSON.stringify(extra).slice(1)}` : ']}';
+}