sorodb 0.0.0

package/docs/reference/schema-and-query.md

@@ -0,0 +1,87 @@
+# Schema and query reference
+
+Register a schema with `db.schema({ version }, definitions)` before using a collection. The version must be an integer of at least 1, and `definitions` must contain at least one table.
+
+## Table definitions
+
+```js
+db.schema({ version: 1 }, [
+	{
+		table: "users",
+		columns: {
+			id: { type: "number", primary: true },
+			name: { type: "string" },
+			nickname: { type: "string?" },
+			joinedAt: { type: "date", default: () => new Date() },
+		},
+		indexes: [{ name: "joined_at", columns: [{ field: "joinedAt", direction: "desc" }] }],
+	},
+]);
+```
+
+Each table needs a unique name and a `columns` object. Table, column, and index names must start with a letter and contain only letters, digits, or underscores. Exactly one column must be marked `primary: true`, and it must be named `id`. Its type must be a required string or number type; stored IDs must be strings or finite numbers.
+
+Each column has a `type` containing an [ArkType definition](https://arktype.io/docs/intro). Common strings include `"string"`, `"number"`, `"string[]"`, `"bigint"`, and `"unknown"`. SoroDB also accepts `"date"` for a `Date`, `"string?"` for an optional string, and expressions such as `"string | null"` for nullable values. You can pass an ArkType type object where a string definition is insufficient.
+
+`default` can be a value or a zero-argument function. SoroDB applies it when a field is `undefined` on create or replace, then validates the result. Value defaults are cloned for each document; function defaults are called for each document. Fields without optional types or defaults must be supplied. Undeclared fields are rejected.
+
+Documents are stored as CBOR. Supported data includes null, booleans, finite numbers, strings, BigInts, valid Dates, `Uint8Array` values, dense arrays, and plain objects containing these values. Sparse arrays, cycles, invalid Dates, and unsupported class instances cannot be stored. `Date`, BigInt, and byte array values round-trip through storage.
+
+For persistent data, [schema upgrades](../guides/schema-upgrades.md) explains version changes and backfilling.
+
+## Index definitions
+
+An index has a unique `name`, a nonempty `columns` array, optional `unique: true`, and optional `method: "btree"`. `btree` is the only current method and is the default. Each column is a field name or `{ field, direction }`; direction defaults to `"asc"` and can be `"desc"`. Dotted paths can point into nested document data, provided the top-level column is declared.
+
+```js
+indexes: [
+	{ name: "email", columns: ["email"], unique: true },
+	{
+		name: "recent_by_team",
+		columns: ["team", { field: "joinedAt", direction: "desc" }],
+	},
+];
+```
+
+B-tree fields support missing values, null, booleans, finite numbers, BigInts, strings, valid Dates, and byte arrays. Indexing an array or object value fails validation on write. A unique index skips a document when an indexed field is missing; `null` is an indexed value and must be unique when `unique: true`.
+
+## Filters
+
+Pass `where` to `collection.filter`. It receives a builder and must return one expression. Paths may be dotted, such as `"profile.city"`.
+
+```js
+const query = users.filter({
+	where: (w) => w.and(w.gte("age", 18), w.startsWith("name", "A")),
+});
+```
+
+| Builder                                                        | Matches                                                                                  |
+| -------------------------------------------------------------- | ---------------------------------------------------------------------------------------- |
+| `eq(path, value)`, `ne(path, value)`                           | Equal or unequal values.                                                                 |
+| `gt`, `gte`, `lt`, `lte`                                       | Ordered comparisons between compatible numbers, strings, BigInts, Dates, or byte arrays. |
+| `in(path, values)`                                             | A field equal to any value in an array.                                                  |
+| `contains(path, value)`                                        | A substring in a string, or an equal element in an array.                                |
+| `startsWith(path, prefix)`                                     | A string with the given prefix.                                                          |
+| `exists(path, present?)`                                       | Field is defined; pass `false` to match missing fields. `null` counts as present.        |
+| `and(...expressions)`, `or(...expressions)`, `not(expression)` | Combine filter expressions.                                                              |
+
+Filter results are checked against complete documents, even when an index is used to find candidates. Equality on `id` can use the primary key directly. An equality filter may use a B-tree index when its first field matches the filter path and is ascending; other filters can scan documents. See [indexes and sorting](../guides/indexes-and-sorting.md).
+
+## Ordering, projection, and limits
+
+This example assumes `users` has a `team` column and the `recent_by_team` index shown above.
+
+```js
+const page = await users
+	.filter({
+		where: (w) => w.eq("team", "platform"),
+		orderBy: (order) => order.asc("team").desc("joinedAt"),
+		select: ["name", "joinedAt"],
+		limit: 20,
+	})
+	.page();
+```
+
+`orderBy` requires an index whose leading fields and directions match the requested order. Ascending `id` works without a declared index. Fields in one order must be distinct. Without `orderBy`, do not rely on a particular result order.
+
+`select` accepts field paths and always retains `id`. Missing selected fields are omitted. `limit` must be a positive integer. It caps async iteration and sets the size of a page; `page()` defaults to 100. See [pagination](../guides/pagination.md) for cursors and snapshot behavior.

package/examples/basic.js

@@ -0,0 +1,26 @@
+// @ts-check
+import SoroDB from "../src/index.js";
+
+const db = SoroDB({ store: "memory:///", path: "example" });
+db.schema({ version: 1 }, [
+	{
+		table: "users",
+		columns: {
+			id: { type: "number", primary: true },
+			name: { type: "string" },
+			joinedAt: { type: "date", default: () => new Date() },
+		},
+		indexes: [{ name: "joined_at", columns: [{ field: "joinedAt", direction: "desc" }] }],
+	},
+]);
+
+try {
+	const users = await db.collection("users");
+	await users.create({ id: 1, name: "Ada" });
+	await users.create({ id: 2, name: "Grace" });
+	for await (const user of users.filter({ orderBy: (order) => order.desc("joinedAt") })) {
+		console.log(user);
+	}
+} finally {
+	await db.close();
+}

package/package.json

@@ -0,0 +1,45 @@
+{
+	"name": "sorodb",
+	"version": "0.0.0",
+	"description": "Embedded document database built on SlateDB",
+	"license": "MIT",
+	"files": [
+		"src",
+		"types",
+		"LICENSE",
+		"README.md",
+		"CONTRIBUTING.md",
+		"docs",
+		"examples"
+	],
+	"type": "module",
+	"types": "./types/index.d.ts",
+	"exports": {
+		".": {
+			"types": "./types/index.d.ts",
+			"import": "./src/index.js"
+		}
+	},
+	"scripts": {
+		"test": "node --test",
+		"typecheck": "tsc --noEmit",
+		"build:types": "tsc -p tsconfig.build.json && node scripts/finalize-types.js",
+		"prepack": "npm run build:types",
+		"pack:dry": "npm pack --dry-run",
+		"fmt": "oxfmt",
+		"lint": "oxlint"
+	},
+	"dependencies": {
+		"@slatedb/uniffi": "^0.16.0",
+		"@types/node": "^26.6.2",
+		"arktype": "^2.1.20",
+		"cbor2": "^2.3.0"
+	},
+	"devDependencies": {
+		"oxlint": "^1.85.0",
+		"typescript": "^7.0.2"
+	},
+	"engines": {
+		"node": ">=20"
+	}
+}

package/src/collection.js

@@ -0,0 +1,90 @@
+// @ts-check
+import { ConflictError, SchemaError, ValidationError } from "./errors.js";
+import { decodeData, docKey, encodeData } from "./encoding.js";
+import { validateDocument } from "./schema.js";
+import { addIndexes, removeIndexes } from "./indexes.js";
+import { Query } from "./query.js";
+
+export class Collection {
+	/** @param {import("./storage.js").Database} db @param {import("./types.js").Table} table @param {import("@slatedb/uniffi").DbTransaction | null} [transaction] */
+	constructor(db, table, transaction = null) {
+		this.db = db;
+		this.table = table;
+		this.transaction = transaction;
+	}
+
+	/** @template T @param {(tx: import("@slatedb/uniffi").DbTransaction) => Promise<T>} callback @returns {Promise<T>} */
+	async _write(callback) {
+		if (this.transaction) {
+			return callback(this.transaction);
+		}
+		return this.db._atomic(callback);
+	}
+
+	/** @param {import("./types.js").Document} document */
+	async create(document) {
+		const value = validateDocument(this.table, document);
+		const key = docKey(this.table.name, value.id);
+		return this._write(async (tx) => {
+			if (await tx.get(key)) {
+				throw new ConflictError(`${this.table.name} id already exists`);
+			}
+			await tx.put(key, encodeData(value));
+			await addIndexes(tx, this.table, value);
+			return value;
+		});
+	}
+
+	/** @param {string | number} id */
+	async get(id) {
+		await this.db._ready();
+		const reader = this.transaction ?? this.db.native;
+		if (!reader) {
+			throw new SchemaError("Database is not open");
+		}
+		const bytes = await reader.get(docKey(this.table.name, id));
+		if (bytes) {
+			return decodeData(bytes);
+		}
+		return null;
+	}
+
+	/** @param {string | number} id @param {import("./types.js").Document} document */
+	async replace(id, document) {
+		if (!document || document.id !== id) {
+			throw new ValidationError("Replacement document must retain the same id");
+		}
+		const value = validateDocument(this.table, document);
+		const key = docKey(this.table.name, id);
+		return this._write(async (tx) => {
+			const bytes = await tx.get(key);
+			if (!bytes) {
+				throw new ConflictError(`${this.table.name} id does not exist`);
+			}
+			const before = decodeData(bytes);
+			await removeIndexes(tx, this.table, before);
+			await tx.put(key, encodeData(value));
+			await addIndexes(tx, this.table, value);
+			return value;
+		});
+	}
+
+	/** @param {string | number} id */
+	async delete(id) {
+		const key = docKey(this.table.name, id);
+		return this._write(async (tx) => {
+			const bytes = await tx.get(key);
+			if (!bytes) {
+				return false;
+			}
+			await removeIndexes(tx, this.table, decodeData(bytes));
+			await tx.delete(key);
+			return true;
+		});
+	}
+
+	/** @param {import("./types.js").QueryOptions} [options] */
+	filter(options = {}) {
+		return new Query(this, options);
+	}
+}

package/src/encoding.js

@@ -0,0 +1,200 @@
+// @ts-check
+import { decode, encode } from "cbor2";
+import { ValidationError } from "./errors.js";
+
+/** @param {string} value */
+const utf8 = (value) => Buffer.from(value, "utf8");
+/** @param {Uint8Array} bytes */
+export function part(bytes) {
+	const out = Buffer.allocUnsafe(bytes.length * 2 + 2);
+	let length = 0;
+	for (const byte of bytes) {
+		out[length++] = byte;
+		if (byte === 0) {
+			out[length++] = 255;
+		}
+	}
+	out[length++] = 0;
+	out[length++] = 0;
+	return out.subarray(0, length);
+}
+
+/** @param {number} value */
+function sortableNumber(value) {
+	if (!Number.isFinite(value)) {
+		throw new ValidationError("Indexed numbers must be finite");
+	}
+	const bytes = Buffer.alloc(8);
+	let normalized = value;
+	if (Object.is(value, -0)) {
+		normalized = 0;
+	}
+	bytes.writeDoubleBE(normalized);
+	if (bytes[0] & 128) {
+		for (let i = 0; i < 8; i++) {
+			bytes[i] ^= 255;
+		}
+	} else {
+		bytes[0] ^= 128;
+	}
+	return bytes;
+}
+
+export { sortableNumber };
+
+/** @param {unknown} name @param {string} [label] */
+export function validName(name, label = "name") {
+	if (typeof name !== "string" || !/^[A-Za-z][A-Za-z0-9_]*$/.test(name)) {
+		throw new ValidationError(
+			`${label} must start with a letter and contain only letters, digits, or underscores`,
+		);
+	}
+	return name;
+}
+
+/** @param {unknown} id */
+export function idBytes(id) {
+	if (typeof id === "string") {
+		return Buffer.concat([Buffer.from([115]), part(utf8(id))]);
+	}
+	if (typeof id === "number" && Number.isFinite(id)) {
+		return Buffer.concat([Buffer.from([110]), sortableNumber(id)]);
+	}
+	throw new ValidationError("id must be a string or finite number");
+}
+
+/** @param {string} table */
+export function docPrefix(table) {
+	return utf8(`D\0${table}\0`);
+}
+/** @param {string} table @param {string | number} id */
+export function docKey(table, id) {
+	return Buffer.concat([docPrefix(table), idBytes(id)]);
+}
+/** @param {string} table @param {string} name */
+export function indexPrefix(table, name) {
+	return utf8(`I\0${table}\0${name}\0`);
+}
+
+/** @param {unknown} object @param {string} path @returns {unknown} */
+export function getPath(object, path) {
+	let value = object;
+	for (const key of path.split(".")) {
+		if (value == null) {
+			return undefined;
+		}
+		value = /** @type {Record<string, unknown>} */ (Object(value))[key];
+	}
+	return value;
+}
+
+/** @param {Uint8Array} prefix */
+export function prefixEnd(prefix) {
+	const end = Buffer.from(prefix);
+	for (let i = end.length - 1; i >= 0; i--) {
+		if (end[i] !== 255) {
+			end[i]++;
+			return end.subarray(0, i + 1);
+		}
+	}
+	return undefined;
+}
+
+/** @param {Uint8Array} prefix @param {Uint8Array} [after] */
+export function range(prefix, after) {
+	return {
+		start: after || prefix,
+		start_inclusive: !after,
+		end: prefixEnd(prefix),
+		end_inclusive: false,
+	};
+}
+
+const cborOptions = { collapseBigInts: false, dateTag: 0 };
+
+/** @param {unknown} value @param {boolean} [allowUndefined] @param {boolean} [normalizeNumbers] */
+function checked(value, allowUndefined = false, normalizeNumbers = false) {
+	const seen = new Set();
+	/** @param {any} item @returns {any} */
+	function visit(item) {
+		if (item === undefined && allowUndefined) {
+			return item;
+		}
+		if (
+			item === null ||
+			typeof item === "string" ||
+			typeof item === "boolean" ||
+			typeof item === "bigint"
+		) {
+			return item;
+		}
+		if (typeof item === "number" && Number.isFinite(item)) {
+			if (normalizeNumbers && Object.is(item, -0)) {
+				return 0;
+			}
+			return item;
+		}
+		if (item instanceof Date) {
+			if (Number.isNaN(item.getTime())) {
+				throw new ValidationError("Invalid Date cannot be stored");
+			}
+			return item;
+		}
+		if (item instanceof Uint8Array) {
+			if (item.constructor === Uint8Array) {
+				return item;
+			}
+			return new Uint8Array(item.buffer, item.byteOffset, item.byteLength);
+		}
+		if (typeof item !== "object" || item === null) {
+			throw new ValidationError("Documents must contain CBOR-compatible values");
+		}
+		if (seen.has(item)) {
+			throw new ValidationError("Cyclic documents cannot be stored");
+		}
+		seen.add(item);
+		let result;
+		if (Array.isArray(item)) {
+			result = [];
+			for (let i = 0; i < item.length; i++) {
+				if (!Object.hasOwn(item, i)) {
+					throw new ValidationError("Sparse arrays cannot be stored");
+				}
+				result.push(visit(item[i]));
+			}
+		} else {
+			if (
+				Object.getPrototypeOf(item) !== Object.prototype &&
+				Object.getPrototypeOf(item) !== null
+			) {
+				throw new ValidationError("Documents must contain plain objects");
+			}
+			result = /** @type {Record<string, any>} */ (Object.create(null));
+			for (const [key, child] of Object.entries(item)) {
+				result[key] = visit(child);
+			}
+		}
+		seen.delete(item);
+		return result;
+	}
+	return visit(value);
+}
+
+/** @param {unknown} value */
+export function encodeData(value) {
+	return Buffer.from(encode(checked(value), cborOptions));
+}
+
+/** @param {Uint8Array} bytes @returns {any} */
+export function decodeData(bytes) {
+	return decode(new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength), {
+		collapseBigInts: false,
+	});
+}
+
+/** @param {unknown} value */
+export function stable(value) {
+	return Buffer.from(encode(checked(value, true, true), { ...cborOptions, cde: true })).toString(
+		"hex",
+	);
+}

package/src/errors.js

@@ -0,0 +1,43 @@
+// @ts-check
+export class SoroError extends Error {
+	/** @param {string} message @param {string} code @param {unknown} [details] */
+	constructor(message, code, details) {
+		super(message);
+		this.name = this.constructor.name;
+		this.code = code;
+		if (details !== undefined) {
+			this.details = details;
+		}
+	}
+}
+
+export class ValidationError extends SoroError {
+	/** @param {string} message @param {unknown} [details] */
+	constructor(message, details) {
+		super(message, "VALIDATION_ERROR", details);
+	}
+}
+export class SchemaError extends SoroError {
+	/** @param {string} message @param {unknown} [details] */
+	constructor(message, details) {
+		super(message, "SCHEMA_ERROR", details);
+	}
+}
+export class ConflictError extends SoroError {
+	/** @param {string} message @param {unknown} [details] */
+	constructor(message, details) {
+		super(message, "CONFLICT", details);
+	}
+}
+export class QueryError extends SoroError {
+	/** @param {string} message @param {unknown} [details] */
+	constructor(message, details) {
+		super(message, "QUERY_ERROR", details);
+	}
+}
+export class CursorError extends SoroError {
+	/** @param {string} message @param {unknown} [details] */
+	constructor(message, details) {
+		super(message, "CURSOR_ERROR", details);
+	}
+}

package/src/index-methods.js

@@ -0,0 +1,131 @@
+// @ts-check
+import { SchemaError, ValidationError } from "./errors.js";
+import { docKey, getPath, indexPrefix, part, sortableNumber } from "./encoding.js";
+
+/** @param {bigint} value */
+function sortableBigInt(value) {
+	if (value === 0n) {
+		return Buffer.from([1]);
+	}
+	const negative = value < 0n;
+	let magnitude = value;
+	if (negative) {
+		magnitude = -value;
+	}
+	const hex = magnitude.toString(16);
+	let paddedHex = hex;
+	if (hex.length % 2) {
+		paddedHex = `0${hex}`;
+	}
+	const digits = Buffer.from(paddedHex, "hex");
+	const length = Buffer.alloc(8);
+	length.writeBigUInt64BE(BigInt(digits.length));
+	if (negative) {
+		for (let i = 0; i < length.length; i++) {
+			length[i] ^= 255;
+		}
+		for (let i = 0; i < digits.length; i++) {
+			digits[i] ^= 255;
+		}
+	}
+	let sign = 2;
+	if (negative) {
+		sign = 0;
+	}
+	return Buffer.concat([Buffer.from([sign]), length, digits]);
+}
+
+/** @param {unknown} value */
+function supportsBtreeValue(value) {
+	return (
+		value === undefined ||
+		value === null ||
+		typeof value === "boolean" ||
+		(typeof value === "number" && Number.isFinite(value)) ||
+		typeof value === "bigint" ||
+		typeof value === "string" ||
+		(value instanceof Date && !Number.isNaN(value.getTime())) ||
+		value instanceof Uint8Array
+	);
+}
+
+/** @param {unknown} value @param {import("./types.js").Direction} [direction] */
+function btreeScalar(value, direction = "asc") {
+	let bytes;
+	if (value === undefined) {
+		bytes = Buffer.from([0]);
+	} else if (value === null) {
+		bytes = Buffer.from([16]);
+	} else if (value === false) {
+		bytes = Buffer.from([32]);
+	} else if (value === true) {
+		bytes = Buffer.from([33]);
+	} else if (typeof value === "number") {
+		bytes = Buffer.concat([Buffer.from([48]), sortableNumber(value)]);
+	} else if (typeof value === "bigint") {
+		bytes = Buffer.concat([Buffer.from([56]), sortableBigInt(value)]);
+	} else if (value instanceof Date && !Number.isNaN(value.getTime())) {
+		bytes = Buffer.concat([Buffer.from([64]), sortableNumber(value.getTime())]);
+	} else if (typeof value === "string") {
+		bytes = Buffer.concat([Buffer.from([80]), part(Buffer.from(value, "utf8"))]);
+	} else if (value instanceof Uint8Array) {
+		bytes = Buffer.concat([Buffer.from([96]), part(value)]);
+	} else {
+		throw new ValidationError("btree indexes support scalars, Dates, BigInts, and byte arrays");
+	}
+	if (direction === "desc") {
+		return Buffer.from(bytes.map((byte) => byte ^ 255));
+	}
+	return bytes;
+}
+
+const btree = {
+	supportsValue: supportsBtreeValue,
+	encodeValue: btreeScalar,
+	/** @param {import("./types.js").Index} index @param {import("./types.js").IndexField[]} order */
+	supportsOrder: (index, order) =>
+		order.every(
+			(field, i) =>
+				index.fields[i]?.name === field.name && index.fields[i]?.direction === field.direction,
+		),
+	/** @param {string} table @param {import("./types.js").Index} index @param {string} path @param {unknown} value */
+	equalityPrefix(table, index, path, value) {
+		if (
+			index.fields[0]?.name !== path ||
+			index.fields[0].direction !== "asc" ||
+			!supportsBtreeValue(value)
+		) {
+			return null;
+		}
+		return Buffer.concat([indexPrefix(table, index.name), btreeScalar(value)]);
+	},
+};
+
+/** @type {Record<string, typeof btree>} */
+const methods = { btree };
+
+/** @param {string} name */
+export function indexMethod(name) {
+	if (!Object.hasOwn(methods, name)) {
+		throw new SchemaError(`Unsupported index method ${name}`);
+	}
+	return methods[name];
+}
+
+/** @param {string} table @param {import("./types.js").Index} index @param {import("./types.js").Document} doc */
+export function indexKey(table, index, doc) {
+	const method = indexMethod(index.method);
+	const values = [];
+	for (const field of index.fields) {
+		const value = getPath(doc, field.name);
+		if (value === undefined && index.unique) {
+			return null;
+		}
+		values.push(method.encodeValue(value, field.direction));
+	}
+	const prefix = Buffer.concat([indexPrefix(table, index.name), ...values]);
+	if (index.unique) {
+		return prefix;
+	}
+	return Buffer.concat([prefix, part(docKey(table, doc.id))]);
+}

package/src/index.js

@@ -0,0 +1,16 @@
+// @ts-check
+import { Database } from "./storage.js";
+
+/** @param {import("./types.js").DatabaseConfig} config */
+export default function SoroDB(config) {
+	return new Database(config);
+}
+export { SoroDB };
+export {
+	SoroError,
+	ValidationError,
+	SchemaError,
+	ConflictError,
+	QueryError,
+	CursorError,
+} from "./errors.js";

package/src/indexes.js

@@ -0,0 +1,32 @@
+// @ts-check
+import { ConflictError } from "./errors.js";
+import { docKey } from "./encoding.js";
+import { indexKey } from "./index-methods.js";
+
+/** @param {import("@slatedb/uniffi").DbTransaction} tx @param {import("./types.js").Table} table @param {import("./types.js").Document} document @param {import("./types.js").Index[]} [indexes] */
+export async function removeIndexes(tx, table, document, indexes = table.indexes) {
+	for (const index of indexes) {
+		const key = indexKey(table.name, index, document);
+		if (key) {
+			await tx.delete(key);
+		}
+	}
+}
+
+/** @param {import("@slatedb/uniffi").DbTransaction} tx @param {import("./types.js").Table} table @param {import("./types.js").Document} document @param {import("./types.js").Index[]} [indexes] */
+export async function addIndexes(tx, table, document, indexes = table.indexes) {
+	const keyOfDocument = docKey(table.name, document.id);
+	for (const index of indexes) {
+		const key = indexKey(table.name, index, document);
+		if (!key) {
+			continue;
+		}
+		if (index.unique) {
+			const existing = await tx.get(key);
+			if (existing && !Buffer.from(existing).equals(keyOfDocument)) {
+				throw new ConflictError(`Unique index ${table.name}.${index.name} already has this value`);
+			}
+		}
+		await tx.put(key, keyOfDocument);
+	}
+}