tsondb 0.22.0 → 0.23.1

package/dist/src/node/shared.js

@@ -0,0 +1,267 @@
+import { exists } from "@elyukai/utils/async/node";
+import { nullableToArray } from "@elyukai/utils/nullable";
+import { isError } from "@elyukai/utils/result";
+import { assertExhaustive } from "@elyukai/utils/typeSafety";
+import Debug from "debug";
+import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
+import { findPackageJSON } from "node:module";
+import { join, sep } from "node:path";
+import { stderr } from "node:process";
+import { styleText } from "node:util";
+import { NodeKind } from "../shared/schema/Node.js";
+import { countError, countErrors, getErrorMessageForDisplay, wrapErrorsIfAny, } from "../shared/utils/error.js";
+import { parallelizeErrors } from "../shared/utils/validation.js";
+import { getDisplayName, getDisplayNameWithId } from "./schema/detached.js";
+import { isEntityDecl } from "./schema/guards.js";
+import { serializeNode } from "./schema/treeOperations/serialization.js";
+import { createEntityNameValidator, createReferenceValidator, validateDeclReferentialIntegrity, validateDeclStructuralIntegrity, } from "./schema/treeOperations/validation.js";
+import { checkCustomConstraintsForAllEntities } from "./utils/customConstraints.js";
+import { DatabaseInMemory } from "./utils/databaseInMemory.js";
+import { getAllInstanceOverviewsByEntityName, getInstanceOverview } from "./utils/displayName.js";
+import { getFileNameForId, getSingletonFileName } from "./utils/files.js";
+import { attachGitStatusToDatabaseInMemory } from "./utils/git.js";
+import { getReferencesToInstances } from "./utils/references.js";
+import { checkUniqueConstraintsForAllEntities } from "./utils/unique.js";
+const debug = Debug("tsondb:class:shared");
+const getMetaFilePath = (dataRootPath) => join(dataRootPath, ".meta.json");
+export const getTSONDBVersion = async () => {
+    const packageJsonPath = findPackageJSON(".", import.meta.url);
+    if (!packageJsonPath) {
+        throw new Error("Could not find package.json for TSONDB");
+    }
+    const packageJson = JSON.parse(await readFile(packageJsonPath, "utf-8"));
+    return packageJson.version;
+};
+export const readMetaFile = async (dataRootPath) => JSON.parse(await readFile(getMetaFilePath(dataRootPath), "utf-8"));
+// try {
+// } catch (err) {
+//   if (err instanceof Error && (err as NodeJS.ErrnoException).code === "ENOENT") {
+//     return undefined
+//   }
+//   throw err
+// }
+export const writeMetaFile = async (dataRootPath, meta) => writeFile(getMetaFilePath(dataRootPath), JSON.stringify(meta, null, 2), "utf-8");
+export const prepareFolders = async (schemaVersion, dataRootPath, entities) => {
+    await mkdir(dataRootPath, { recursive: true });
+    await Promise.all(entities.map(entity => mkdir(join(dataRootPath, entity.name), { recursive: true })));
+    if (!(await exists(getMetaFilePath(dataRootPath)))) {
+        const newMetaFile = {
+            version: await getTSONDBVersion(),
+            schemaVersion,
+        };
+        await writeMetaFile(dataRootPath, newMetaFile);
+    }
+};
+export const generateOutputs = async (schema, outputs) => {
+    await Promise.all(outputs.map(output => output.run(schema)));
+};
+export const checkLocales = (schema, data, locales) => {
+    const { localeEntity } = schema;
+    if (localeEntity &&
+        !locales.every(locale => data.hasInstanceOfEntityById(localeEntity.name, locale))) {
+        throw new Error("All provided locales must exist in the database.");
+    }
+};
+const loadDataAndMeta = async (dataRootPath, schema, allowCompressedFile) => {
+    const rootStat = await stat(dataRootPath).catch(() => undefined);
+    if (rootStat === undefined) {
+        throw new Error(`Data root path "${dataRootPath}" does not exist.`);
+    }
+    if (rootStat.isFile()) {
+        if (!allowCompressedFile) {
+            throw new Error(`A writable database does not allow writing to a compressed database, but "${dataRootPath}" is a file.`);
+        }
+        debug("loading compressed database into memory ...");
+        const compressedData = JSON.parse(await readFile(dataRootPath, "utf-8"));
+        const data = await DatabaseInMemory.loadFromCompressedFile(compressedData, schema.isEntityName.bind(schema));
+        debug("done");
+        return { data, meta: compressedData.meta };
+    }
+    else if (rootStat.isDirectory()) {
+        debug("loading database into memory ...");
+        const data = await DatabaseInMemory.load(dataRootPath, schema.entities, schema.singletonEntities);
+        const meta = await readMetaFile(dataRootPath);
+        debug("done");
+        return { data, meta };
+    }
+    else {
+        throw new Error(`Data root path "${dataRootPath}" is not a directory.`);
+    }
+};
+export const initData = async (dataRootPath, schema, locales, git, gitStatus, skipReferenceCache, allowCompressedFile) => {
+    const { data, meta } = await loadDataAndMeta(dataRootPath, schema, allowCompressedFile);
+    checkLocales(schema, data, locales);
+    const serializedDeclarationsByName = Object.fromEntries(schema.resolvedDeclarations.map(decl => [decl.name, serializeNode(decl)]));
+    let referencesToInstances = {};
+    if (!skipReferenceCache) {
+        debug("creating references cache ...");
+        referencesToInstances = await getReferencesToInstances(data, serializedDeclarationsByName);
+        debug("done");
+    }
+    else {
+        debug("skipping references cache creation");
+    }
+    if (git) {
+        const status = gitStatus ?? (await git.client.status());
+        const dataWithGit = attachGitStatusToDatabaseInMemory(data, dataRootPath, git.root, status);
+        debug("attached git status to instances");
+        return { data: dataWithGit, referencesToInstances, meta };
+    }
+    else {
+        return { data, referencesToInstances, meta };
+    }
+};
+export const validate = (properties, options = {}) => {
+    const { data, schema, validationOptions, dataRootPath, locales } = properties;
+    const { checkReferentialIntegrity, checkOnlyEntities } = validationOptions;
+    const { skipStructuralValidation = false, logToConsole = true, changedEntities: changedEntityNames, } = options;
+    const entities = [...schema.entities, ...schema.singletonEntities];
+    const getEntity = schema.getEntity.bind(schema);
+    for (const onlyEntity of checkOnlyEntities) {
+        if (!entities.find(entity => entity.name === onlyEntity)) {
+            throw new Error(`Entity "${onlyEntity}" not found in schema`);
+        }
+    }
+    const onlyEntities = checkOnlyEntities.length > 0
+        ? entities.filter(entity => checkOnlyEntities.includes(entity.name))
+        : entities;
+    const changedEntities = changedEntityNames
+        ? entities.filter(entity => changedEntityNames.includes(entity.name))
+        : undefined;
+    const validationContext = { validationOptions, useStyling: true };
+    const errors = [];
+    const summary = [];
+    const debugAndAddToSummary = (message) => {
+        debug(message);
+        summary.push(message);
+    };
+    if (!skipStructuralValidation) {
+        debug("Checking structural integrity ...");
+        const structureErrors = onlyEntities
+            .flatMap(entity => {
+            switch (entity.kind) {
+                case NodeKind.EntityDecl:
+                    return parallelizeErrors(data
+                        .getAllInstanceContainersOfEntity(entity.name)
+                        .map(instance => wrapErrorsIfAny(`in file ${styleText("white", `"${dataRootPath}${sep}${styleText("bold", join(entity.name, getFileNameForId(instance.id)))}"`)}`, validateDeclStructuralIntegrity(validationContext, [], entity, [], instance.content))));
+                case NodeKind.SingletonEntityDecl: {
+                    const instance = data.getSingletonInstanceContainerOfEntity(entity.name);
+                    if (instance) {
+                        return nullableToArray(wrapErrorsIfAny(`in file ${styleText("white", `"${dataRootPath}${sep}${styleText("bold", getSingletonFileName(entity.name))}"`)}`, validateDeclStructuralIntegrity(validationContext, [], entity, [], instance.content)));
+                    }
+                    return [];
+                }
+                default:
+                    return assertExhaustive(entity);
+            }
+        })
+            .toSorted((a, b) => a.message.localeCompare(b.message));
+        if (structureErrors.length > 0) {
+            const errorCount = countErrors(structureErrors);
+            debugAndAddToSummary(`${errorCount.toString()} structural integrity violation${errorCount === 1 ? "" : "s"} found`);
+            errors.push(...structureErrors);
+        }
+        else {
+            debug("No structural integrity violations found");
+        }
+    }
+    if (errors.length === 0) {
+        if (!skipStructuralValidation) {
+            if (checkReferentialIntegrity) {
+                debug("Checking referential integrity ...");
+                const entityNameValidator = createEntityNameValidator(schema.isEntityName.bind(schema), true);
+                const referenceValidator = createReferenceValidator(schema.isEntityName.bind(schema), data, true);
+                const referenceErrors = onlyEntities
+                    .flatMap(entity => {
+                    switch (entity.kind) {
+                        case NodeKind.EntityDecl:
+                            return parallelizeErrors(data
+                                .getAllInstanceContainersOfEntity(entity.name)
+                                .map(instance => wrapErrorsIfAny(`in file ${styleText("white", `"${dataRootPath}${sep}${styleText("bold", join(entity.name, getFileNameForId(instance.id)))}"`)}`, validateDeclReferentialIntegrity(validationContext, entityNameValidator, referenceValidator, [], entity, [], instance.content))));
+                        case NodeKind.SingletonEntityDecl: {
+                            const instance = data.getSingletonInstanceContainerOfEntity(entity.name);
+                            if (instance) {
+                                return nullableToArray(wrapErrorsIfAny(`in file ${styleText("white", `"${dataRootPath}${sep}${styleText("bold", getSingletonFileName(entity.name))}"`)}`, validateDeclReferentialIntegrity(validationContext, entityNameValidator, referenceValidator, [], entity, [], instance.content)));
+                            }
+                            return [];
+                        }
+                        default:
+                            return assertExhaustive(entity);
+                    }
+                })
+                    .toSorted((a, b) => a.message.localeCompare(b.message));
+                if (referenceErrors.length > 0) {
+                    const errorCount = countErrors(referenceErrors);
+                    debugAndAddToSummary(`${errorCount.toString()} referential integrity violation${errorCount === 1 ? "" : "s"} found`);
+                    errors.push(...referenceErrors);
+                }
+                else {
+                    debug("No referential integrity violations found");
+                }
+            }
+            else {
+                debug("Disabled referential integrity checks, skipping them");
+            }
+        }
+        debug("Checking unique constraints ...");
+        const instanceOverviewsByEntityName = getAllInstanceOverviewsByEntityName(getEntity, data, locales);
+        const uniqueConstraintResult = checkUniqueConstraintsForAllEntities(data, (changedEntities ?? onlyEntities).filter(e => isEntityDecl(e)), instanceOverviewsByEntityName);
+        if (isError(uniqueConstraintResult)) {
+            const errorCount = countError(uniqueConstraintResult.error);
+            debugAndAddToSummary(`${errorCount.toString()} unique constraint violation${errorCount === 1 ? "" : "s"} found`);
+            errors.push(uniqueConstraintResult.error);
+        }
+        else {
+            debug("No unique constraint violations found");
+        }
+        debug("Checking custom constraints ...");
+        const customConstraintResult = checkCustomConstraintsForAllEntities((...args) => getDisplayName(getEntity, locales, data, ...args), (...args) => getDisplayNameWithId(getEntity, locales, data, ...args), (...args) => getInstanceOverview(data.getInstanceContainerOfEntityById.bind(data), data.getAllChildInstanceContainersForParent.bind(data, getEntity), getEntity, locales, args), getEntity, data, entities);
+        if (isError(customConstraintResult)) {
+            const errorCount = countError(customConstraintResult.error);
+            debugAndAddToSummary(`${errorCount.toString()} custom constraint violation${errorCount === 1 ? "" : "s"} found`);
+            errors.push(customConstraintResult.error);
+        }
+        else {
+            debug("No custom constraint violations found");
+        }
+    }
+    else {
+        debugAndAddToSummary("Skipping further checks due to previous structural integrity errors");
+    }
+    if (logToConsole) {
+        console.log(`${data.totalSize.toString()} instance${data.totalSize === 1 ? "" : "s"} checked`);
+        if (errors.length === 0) {
+            console.log(styleText("green", "All instances are valid"));
+        }
+        else {
+            const errorCount = countErrors(errors);
+            console.error(styleText("red", `${errors.map(err => getErrorMessageForDisplay(err)).join("\n\n")}\n`, {
+                stream: stderr,
+            }));
+            console.error(styleText(["red", "bold"], "Summary", { stream: stderr }));
+            console.error(styleText("red", [
+                `${errorCount.toString()} validation error${errorCount === 1 ? "" : "s"} found`,
+                ...summary,
+            ].join("\n"), { stream: stderr }));
+        }
+    }
+    return errors;
+};
+export const createCompressedDatabaseFile = (data, schemaVersion) => {
+    const compressedData = {
+        meta: {
+            version: "1.0.0",
+            schemaVersion,
+        },
+        data: Object.assign({}, Object.fromEntries(data
+            .getAllInstances()
+            .map(([entityName, instances]) => [
+            entityName,
+            Object.fromEntries(instances.map(instance => [instance.id, instance.content])),
+        ])), Object.fromEntries(data
+            .getAllSingletonInstances()
+            .map(([entityName, container]) => [entityName, container.content]))),
+    };
+    return compressedData;
+};
+export const isEntityPair = (pair, isEntityName) => isEntityName(pair[0]);

package/dist/src/node/utils/databaseInMemory.d.ts

@@ -3,10 +3,12 @@ import { type KeyPath } from "../../shared/schema/utils/keyPath.ts";
 import type { InstanceContainer, InstanceContent, SingletonInstanceContainer } from "../../shared/utils/instances.ts";
 import type { EntityDecl, SingletonEntityDecl } from "../schema/dsl/index.ts";
 import { type AnyChildEntityMap, type AnyEntityMap, type AnySingletonEntityMap, type GetEntityByName, type IdArgsVariant, type RegisteredEntityMap, type RegisteredSingletonEntityMap } from "../schema/generatedTypeHelpers.ts";
+import { type CompressedDatabaseFile } from "../shared.ts";
 export declare class DatabaseInMemory<EM extends AnyEntityMap = RegisteredEntityMap, SEM extends AnySingletonEntityMap = RegisteredSingletonEntityMap> {
     #private;
     private constructor();
     static load<EM extends AnyEntityMap = RegisteredEntityMap, SEM extends AnySingletonEntityMap = RegisteredSingletonEntityMap>(dataRoot: string, entities: readonly EntityDecl<Extract<keyof EM, string>>[], singletonEntities: readonly SingletonEntityDecl<Extract<keyof SEM, string>>[]): Promise<DatabaseInMemory<EM, SEM>>;
+    static loadFromCompressedFile<EM extends AnyEntityMap = RegisteredEntityMap, SEM extends AnySingletonEntityMap = RegisteredSingletonEntityMap>(file: CompressedDatabaseFile<EM, SEM>, isEntityName: (entityName: Extract<keyof EM | keyof SEM, string>) => entityName is Extract<keyof EM, string>): Promise<DatabaseInMemory<EM, SEM>>;
     getAllInstanceContainersOfEntity<E extends Extract<keyof EM, string>>(entityName: E): InstanceContainer<EM[E]>[];
     getAllInstancesOfEntity<E extends Extract<keyof EM, string>>(entityName: E): EM[E][];
     getAllInstances(): [entityName: Extract<keyof EM, string>, instances: InstanceContainer[]][];

package/dist/src/node/utils/databaseInMemory.js

@@ -1,3 +1,4 @@
+import { partition } from "@elyukai/utils/array/groups";
 import { reorder } from "@elyukai/utils/array/modify";
 import { mapAsync } from "@elyukai/utils/async";
 import { Dictionary } from "@elyukai/utils/dictionary";
@@ -13,6 +14,7 @@ import { promisify } from "node:util";
 import { getValueAtKeyPathIfDefined, setValueAtKeyPath, } from "../../shared/schema/utils/keyPath.js";
 import { HTTPError } from "../../shared/utils/error.js";
 import { normalizedIdArgs, } from "../schema/generatedTypeHelpers.js";
+import { isEntityPair } from "../shared.js";
 import { getPathToSingletonInstance, readSingletonInstance } from "./files.js";
 const exec = promisify(child_process.exec);
 const ulimit = platform === "win32" ? 2048 : Number.parseInt((await exec("ulimit -n")).stdout, 10);
@@ -62,6 +64,14 @@ export class DatabaseInMemory {
             }
         }, 1)).filter((pair) => pair[1] !== undefined)));
     }
+    static async loadFromCompressedFile(file, isEntityName) {
+        const dataEntries = Object.entries(file.data);
+        const [entityEntries, singletonEntityEntries] = partition(dataEntries, pair => isEntityPair(pair, isEntityName));
+        return new DatabaseInMemory(Dictionary.fromEntries(entityEntries.map(([entityName, instances]) => [
+            entityName,
+            Dictionary.fromEntries(Object.entries(instances).map(([id, content]) => [id, { id, content }])),
+        ])), Dictionary.fromEntries(singletonEntityEntries.map(([entityName, content]) => [entityName, { content }])));
+    }
     getAllInstanceContainersOfEntity(entityName) {
         return (this.#data.getMap(entityName, instances => instances.values()) ?? []);
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tsondb",
-  "version": "0.22.0",
+  "version": "0.23.1",
   "description": "",
   "license": "ISC",
   "author": "Lukas Obermann",