tsondb 0.21.8 → 0.21.11

package/dist/src/node/index.d.ts

@@ -50,6 +50,10 @@ export interface TSONDBOptions<T extends DefaultTSONDBTypes = DefaultTSONDBTypes
      * Options for formatting the data in the database.
      */
     formatterOptions?: Partial<FormatterOptions>;
+    /**
+     * If set, the database will be migrated through the specified migration stages before being used.
+     */
+    migrationStages?: MigrationStage<T>[];
 }
 /**
  * Options for validating the data in the database.
@@ -103,6 +107,11 @@ export interface TranslationValidationOptions {
      */
     matchParametersInKeys?: boolean;
 }
+export interface MigrationStage<T extends DefaultTSONDBTypes = DefaultTSONDBTypes> {
+    fromVersion: string;
+    toVersion: string;
+    willMigrate?(context: TSONDB<T>): Promise<void>;
+}
 export type HookTrigger = "transaction:before" | "transaction:after";
 /**
  * The main class for managing a typed JSON database.

package/dist/src/node/index.js

@@ -1,3 +1,4 @@
+import { exists } from "@elyukai/utils/async/node";
 import { on } from "@elyukai/utils/function";
 import { Lazy } from "@elyukai/utils/lazy";
 import { nullableToArray } from "@elyukai/utils/nullable";
@@ -5,10 +6,12 @@ import { compareNumber, reduceCompare } from "@elyukai/utils/ordering";
 import { isError } from "@elyukai/utils/result";
 import { assertExhaustive } from "@elyukai/utils/typeSafety";
 import Debug from "debug";
-import { mkdir, writeFile } from "node:fs/promises";
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { findPackageJSON } from "node:module";
 import { join, sep } from "node:path";
 import { stderr } from "node:process";
 import { styleText } from "node:util";
+import semver from "semver";
 import { simpleGit } from "simple-git";
 import { NodeKind } from "../shared/schema/Node.js";
 import { countError, countErrors, getErrorMessageForDisplay, HTTPError, wrapErrorsIfAny, } from "../shared/utils/error.js";
@@ -28,9 +31,27 @@ import { attachGitStatusToDatabaseInMemory } from "./utils/git.js";
 import { getReferencesToInstances, isReferencedByOtherInstances, } from "./utils/references.js";
 import { checkUniqueConstraintsForAllEntities } from "./utils/unique.js";
 const debug = Debug("tsondb:class");
-const prepareFolders = async (dataRootPath, entities) => {
+const getMetaFilePath = (dataRootPath) => join(dataRootPath, ".meta.json");
+const getTSONDBVersion = async () => {
+    const packageJsonPath = findPackageJSON(".", import.meta.url);
+    if (!packageJsonPath) {
+        throw new Error("Could not find package.json for TSONDB");
+    }
+    const packageJson = JSON.parse(await readFile(packageJsonPath, "utf-8"));
+    return packageJson.version;
+};
+const readMetaFile = async (dataRootPath) => JSON.parse(await readFile(getMetaFilePath(dataRootPath), "utf-8"));
+const writeMetaFile = async (dataRootPath, meta) => writeFile(getMetaFilePath(dataRootPath), JSON.stringify(meta, null, 2), "utf-8");
+const prepareFolders = async (schemaVersion, dataRootPath, entities) => {
     await mkdir(dataRootPath, { recursive: true });
     await Promise.all(entities.map(entity => mkdir(join(dataRootPath, entity.name), { recursive: true })));
+    if (!(await exists(getMetaFilePath(dataRootPath)))) {
+        const newMetaFile = {
+            version: await getTSONDBVersion(),
+            schemaVersion,
+        };
+        await writeMetaFile(dataRootPath, newMetaFile);
+    }
 };
 const generateOutputs = async (schema, outputs) => {
     await Promise.all(outputs.map(output => output.run(schema)));
@@ -119,7 +140,7 @@ export class TSONDB {
     static async create(options, skipReferenceCache = false) {
         const { entities } = options.schema;
         debug("using data root path: %s", options.dataRootPath);
-        await prepareFolders(options.dataRootPath, entities);
+        await prepareFolders(options.schema.version, options.dataRootPath, entities);
         debug("prepared folders");
         const { git: gitClient, root: gitRoot, status: gitStatus } = await getGit(options.dataRootPath);
         if (gitRoot) {
@@ -134,7 +155,7 @@ export class TSONDB {
         }
         const git = gitRoot ? { client: gitClient, root: gitRoot } : undefined;
         const { data, referencesToInstances } = await initData(options.dataRootPath, options.schema, options.locales ?? [], git, gitStatus, skipReferenceCache);
-        return new TSONDB({
+        const db = new TSONDB({
             ...options,
             data,
             referencesToInstances,
@@ -145,6 +166,34 @@ export class TSONDB {
                 ...options.validationOptions,
             },
         });
+        const metaFile = await readMetaFile(options.dataRootPath);
+        if (metaFile.schemaVersion !== options.schema.version && options.migrationStages) {
+            debug("migrating database from schema version %s to %s", metaFile.schemaVersion, options.schema.version);
+            for (const stage of options.migrationStages) {
+                if (semver.valid(stage.toVersion) === null ||
+                    semver.validRange(stage.fromVersion) === null) {
+                    throw new Error(`Invalid semver version in migration stage from ${stage.fromVersion} to ${stage.toVersion}`);
+                }
+            }
+            for (const stage of options.migrationStages) {
+                if (semver.satisfies(metaFile.schemaVersion, stage.fromVersion)) {
+                    debug("applying migration stage from %s to %s", stage.fromVersion, stage.toVersion);
+                    // oxlint-disable-next-line no-await-in-loop
+                    await stage.willMigrate?.(db);
+                    metaFile.schemaVersion = stage.toVersion;
+                }
+                else {
+                    debug("skipping migration stage from %s to %s", stage.fromVersion, stage.toVersion);
+                }
+            }
+        }
+        else {
+            debug("no migration needed, schema version is up to date");
+        }
+        metaFile.version = await getTSONDBVersion();
+        metaFile.schemaVersion = options.schema.version;
+        await writeMetaFile(options.dataRootPath, metaFile);
+        return db;
     }
     /**
      * Generates outputs based on the provided schema and output configurations.
@@ -186,6 +235,11 @@ export class TSONDB {
             useStyling: true,
         };
         const errors = [];
+        const summary = [];
+        const debugAndAddToSummary = (message) => {
+            debug(message);
+            summary.push(message);
+        };
         if (!skipStructuralValidation) {
             debug("Checking structural integrity ...");
             const structureErrors = onlyEntities
@@ -209,7 +263,7 @@ export class TSONDB {
                 .toSorted((a, b) => a.message.localeCompare(b.message));
             if (structureErrors.length > 0) {
                 const errorCount = countErrors(structureErrors);
-                debug(`${errorCount.toString()} structural integrity violation${errorCount === 1 ? "" : "s"} found`);
+                debugAndAddToSummary(`${errorCount.toString()} structural integrity violation${errorCount === 1 ? "" : "s"} found`);
                 errors.push(...structureErrors);
             }
             else {
@@ -243,7 +297,7 @@ export class TSONDB {
                         .toSorted((a, b) => a.message.localeCompare(b.message));
                     if (referenceErrors.length > 0) {
                         const errorCount = countErrors(referenceErrors);
-                        debug(`${errorCount.toString()} referential integrity violation${errorCount === 1 ? "" : "s"} found`);
+                        debugAndAddToSummary(`${errorCount.toString()} referential integrity violation${errorCount === 1 ? "" : "s"} found`);
                         errors.push(...referenceErrors);
                     }
                     else {
@@ -259,7 +313,7 @@ export class TSONDB {
             const uniqueConstraintResult = checkUniqueConstraintsForAllEntities(data, (changedEntities ?? onlyEntities).filter(e => isEntityDecl(e)), instanceOverviewsByEntityName);
             if (isError(uniqueConstraintResult)) {
                 const errorCount = countError(uniqueConstraintResult.error);
-                debug(`${errorCount.toString()} unique constraint violation${errorCount === 1 ? "" : "s"} found`);
+                debugAndAddToSummary(`${errorCount.toString()} unique constraint violation${errorCount === 1 ? "" : "s"} found`);
                 errors.push(uniqueConstraintResult.error);
             }
             else {
@@ -269,7 +323,7 @@ export class TSONDB {
             const customConstraintResult = checkCustomConstraintsForAllEntities((...args) => getDisplayName(getEntity, this.#locales, data, ...args), (...args) => getDisplayNameWithId(getEntity, this.#locales, data, ...args), (...args) => getInstanceOverview(data.getInstanceContainerOfEntityById.bind(data), data.getAllChildInstanceContainersForParent.bind(data, getEntity), getEntity, this.#locales, args), getEntity, data, entities);
             if (isError(customConstraintResult)) {
                 const errorCount = countError(customConstraintResult.error);
-                debug(`${errorCount.toString()} custom constraint violation${errorCount === 1 ? "" : "s"} found`);
+                debugAndAddToSummary(`${errorCount.toString()} custom constraint violation${errorCount === 1 ? "" : "s"} found`);
                 errors.push(customConstraintResult.error);
             }
             else {
@@ -277,7 +331,7 @@ export class TSONDB {
             }
         }
         else {
-            debug("Skipping further checks due to previous structural integrity errors");
+            debugAndAddToSummary("Skipping further checks due to previous structural integrity errors");
         }
         if (logToConsole) {
             console.log(`${data.totalSize.toString()} instance${data.totalSize === 1 ? "" : "s"} checked`);
@@ -286,7 +340,14 @@ export class TSONDB {
             }
             else {
                 const errorCount = countErrors(errors);
-                console.error(styleText("red", `${errorCount.toString()} validation error${errorCount === 1 ? "" : "s"} found\n\n${errors.map(err => getErrorMessageForDisplay(err)).join("\n\n")}`, { stream: stderr }));
+                console.error(styleText("red", `${errors.map(err => getErrorMessageForDisplay(err)).join("\n\n")}\n`, {
+                    stream: stderr,
+                }));
+                console.error(styleText(["red", "bold"], "Summary", { stream: stderr }));
+                console.error(styleText("red", [
+                    `${errorCount.toString()} validation error${errorCount === 1 ? "" : "s"} found`,
+                    ...summary,
+                ].join("\n"), { stream: stderr }));
             }
         }
         return errors;

package/dist/src/node/schema/helpers.js

@@ -35,6 +35,8 @@ export const reduceNodes = (reducer, nodes, options = {}) => {
                 else {
                     return { results: reducer(parentNodes, node, collectedResults), reducedDecls };
                 }
+            case NodeKind.NestedEntityMapType:
+                return reduceNodeTree([...parentNodes, node], node.type.value, reducer(parentNodes, node, collectedResults), reducedDecls);
             case NodeKind.BooleanType:
             case NodeKind.DateType:
             case NodeKind.FloatType:
@@ -42,7 +44,6 @@ export const reduceNodes = (reducer, nodes, options = {}) => {
             case NodeKind.StringType:
             case NodeKind.TypeArgumentType:
             case NodeKind.ReferenceIdentifierType:
-            case NodeKind.NestedEntityMapType:
             case NodeKind.TypeParameter:
             case NodeKind.TranslationObjectType:
                 return { results: reducer(parentNodes, node, collectedResults), reducedDecls };

package/dist/src/node/schema/index.d.ts

@@ -4,7 +4,8 @@ import { type EntityDecl } from "./dsl/declarations/EntityDecl.ts";
 import type { SingletonEntityDecl } from "./dsl/index.ts";
 export declare class Schema<T extends DefaultTSONDBTypes = DefaultTSONDBTypes> {
     #private;
-    constructor(declarations: Decl[], localeEntity?: EntityDecl);
+    constructor(version: string, declarations: Decl[], localeEntity?: EntityDecl);
+    get version(): string;
     get entities(): EntityDecl<EntityName<T>>[];
     get resolvedEntities(): EntityDecl<EntityName<T>>[];
     getEntity<E extends EntityName<T>>(name: E): EntityDecl<E> | undefined;

package/dist/src/node/schema/index.js

@@ -3,6 +3,7 @@ import { Dictionary } from "@elyukai/utils/dictionary";
 import { deepEqual } from "@elyukai/utils/equality";
 import { assertExhaustive, trySafe } from "@elyukai/utils/typeSafety";
 import Debug from "debug";
+import semver from "semver";
 import { NodeKind } from "../../shared/schema/Node.js";
 import { renderKeyPath } from "../../shared/schema/utils/keyPath.js";
 import { getParameterNames, resolveTypeArgumentsInDecls, walkNodeTree, } from "./dsl/declarations/Decl.js";
@@ -22,6 +23,7 @@ import { getNestedDeclarations } from "./treeOperations/implicitSchema.js";
 const debug = Debug("tsondb:schema");
 // const RESERVED_DECLARATION_IDENTIFIER = ["EntityMap", "StringableTranslationParameter"]
 export class Schema {
+    #version;
     #declarations;
     #entities;
     #singletonEntities;
@@ -31,7 +33,10 @@ export class Schema {
     #localeEntity;
     #declarationMap;
     #resolvedDeclarationMap;
-    constructor(declarations, localeEntity) {
+    constructor(version, declarations, localeEntity) {
+        if (semver.valid(version) === null) {
+            throw new TypeError(`Invalid schema version "${version}". Must be a valid semver string.`);
+        }
         debug("creating schema from %d declarations", declarations.length);
         debug("collecting nested declarations ...");
         const allDecls = addDeclarations([], localeEntity ? declarations.concat(localeEntity) : declarations);
@@ -56,6 +61,7 @@ export class Schema {
         checkUniqueConstraints(allDeclsWithoutNestedEntities);
         debug("checking sort orders ...");
         checkSortOrders(allDeclsWithoutNestedEntities);
+        this.#version = version;
         this.#localeEntity = localeEntity;
         this.#declarations = allDeclsWithoutNestedEntities;
         this.#resolvedDeclarations = resolveTypeArgumentsInDecls(allDeclsWithoutNestedEntities);
@@ -67,6 +73,9 @@ export class Schema {
         this.#resolvedDeclarationMap = Dictionary.fromEntries(this.#resolvedDeclarations.map(decl => [decl.name, decl]));
         debug("created schema, no integrity violations found");
     }
+    get version() {
+        return this.#version;
+    }
     get entities() {
         return this.#entities;
     }

package/dist/src/web/web.js

@@ -2468,10 +2468,10 @@ const useMappedAPIResource = (apiFn, mapFn, ...args)=>{
         setNewData
     ];
 };
-const letterOrDigit = /[a-zA-Z0-9]/;
-const uppercase = /[A-Z]/;
-const lowerCaseOrDigit = /[a-z0-9]/;
-const string_separator = /[^a-z0-9]/;
+const letterOrDigit = /[a-zA-Z0-9]/u;
+const uppercase = /[A-Z]/u;
+const lowerCaseOrDigit = /[a-z0-9]/u;
+const string_separator = /[^a-z0-9]/u;
 const lastChar = (str)=>str[str.length - 1];
 const lastElement = (arr)=>arr[arr.length - 1];
 const isAllUppercase = (str)=>str === str.toUpperCase();
@@ -3035,8 +3035,8 @@ class Parser {
             ];
         });
     }
-    static space = Parser.regex(/^\s*/);
-    static hspace = Parser.regex(/^[^\S\r\n]*/);
+    static space = Parser.regex(/^\s*/u);
+    static hspace = Parser.regex(/^[^\S\r\n]*/u);
     token() {
         return this.then((result)=>Parser.space.then(()=>Parser.of(result)));
     }
@@ -3054,7 +3054,7 @@ class Parser {
     }
     static lookahead(parser) {
         return new Parser((syntax)=>{
-            const result = parser.parse(syntax)[0];
+            const [result] = parser.parse(syntax);
             return void 0 !== result ? [
                 [
                     result[0],
@@ -3065,7 +3065,7 @@ class Parser {
     }
     static negativeLookahead(parser) {
         return new Parser((syntax)=>{
-            const result = parser.parse(syntax)[0];
+            const [result] = parser.parse(syntax);
             return void 0 === result ? [
                 [
                     void 0,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tsondb",
-  "version": "0.21.8",
+  "version": "0.21.11",
   "description": "",
   "license": "ISC",
   "author": "Lukas Obermann",
@@ -40,6 +40,7 @@
     "@types/debug": "^4.1.13",
     "@types/express": "^5.0.6",
     "@types/node": "^26.6.4",
+    "@types/semver": "^7.8.0",
     "@typescript/native": "npm:typescript@^7.0.2",
     "commit-and-tag-version": "^13.2.1",
     "tsx": "^4.23.15",
@@ -48,7 +49,7 @@
   "dependencies": {
     "@elyukai/markdown": "^0.4.0",
     "@elyukai/oxc-config": "^0.1.2",
-    "@elyukai/utils": "^0.3.8",
+    "@elyukai/utils": "^0.3.11",
     "@preact/signals": "^2.11.3",
     "debug": "^4.4.3",
     "express": "^5.2.1",
@@ -58,6 +59,7 @@
     "oxlint-tsgolint": "^7.0.2003",
     "preact": "^11.0.0",
     "preact-iso": "^2.12.2",
+    "semver": "^7.8.5",
     "simple-cli-args": "^0.1.3",
     "simple-git": "^4.0.2"
   },