prostgles-server 4.2.625 → 4.2.626

package/lib/StorageClient/getFileTableConfig.ts

@@ -1,37 +1,27 @@
-import { omitKeys } from "prostgles-types";
+import { asName, omitKeys } from "prostgles-types";
+import { md5 } from "prostgles-types/dist/md5";
 import { onDeleteFromFileTable } from "../DboBuilder/TableHandler/onDeleteFromFileTable";
 import { updateFile } from "../DboBuilder/TableHandler/updateFile";
 import { assertFileObjectValid, uploadFile } from "../DboBuilder/TableHandler/uploadFile";
 import type { Prostgles } from "../Prostgles";
-import type { BeforeEachTsTrigger } from "../PublishParser/publishTypesAndUtils";
+import type {
+  AfterEachTsTrigger,
+  BeforeEachTsTrigger,
+} from "../PublishParser/publishTypesAndUtils";
 import type { TableConfig } from "../TableConfig/TableConfigTypes";
-import type { TableRowFromColumnDefinitions } from "../TableConfig/TableRowFromColumnDefinitions";
 import type { TableHooks } from "../TableHooks/TableHooks";
 import { setupFileServeHandler } from "./setupFileServeHandler";
 import type { TableHandlers } from "../DboBuilder/DboBuilder";
+import { getFileTableHandler, getFileVersionTableName } from "./fileVersionUtils";
+import { backfillFileVersions, saveFileVersion } from "./saveFileVersion";
+import {
+  FILE_TABLE_COLUMN_DEFINITIONS,
+  FILE_TABLE_VERSION_COLUMN_DEFINITIONS,
+  FILE_VERSION_TABLE_COLUMN_DEFINITIONS,
+  type FileTableRow,
+} from "./fileTableDefinitions";
 
-const FILE_TABLE_COLUMN_DEFINITIONS = {
-  id: `UUID PRIMARY KEY DEFAULT gen_random_uuid()`,
-  storage_key: `UUID UNIQUE`, // NULL for legacy objects stored under their file ID.
-  extension: `TEXT NOT NULL DEFAULT ''`,
-  content_type: `TEXT NOT NULL DEFAULT ''`,
-  content_length: `BIGINT NOT NULL DEFAULT 0`,
-  etag: `TEXT NOT NULL DEFAULT ''`,
-  original_name: `TEXT NOT NULL`,
-  original_last_modified: `TIMESTAMPTZ`,
-  description: `TEXT`,
-  url: `TEXT NOT NULL DEFAULT ''`,
-  cloud_url: `TEXT`,
-  signed_url: `TEXT`,
-  signed_url_expires: `BIGINT`,
-  added: `TIMESTAMP NOT NULL DEFAULT NOW()`,
-  updated: `TIMESTAMP NOT NULL DEFAULT NOW()`,
-  deleted: `TIMESTAMPTZ`,
-  deleted_from_storage: `TIMESTAMPTZ`,
-  data: `BYTEA NOT NULL CHECK (data = decode('01', 'hex'))`, // Used as a placeholder to ensure insert types are correct. Actual data is uploaded to storageClient and not stored in the DB
-} as const;
-
-export type FileTableRow = TableRowFromColumnDefinitions<typeof FILE_TABLE_COLUMN_DEFINITIONS>;
+export type { FileTableInsertRow, FileTableRow } from "./fileTableDefinitions";
 
 export const getFileTableConfig = (
   prg: Prostgles,
@@ -44,7 +34,30 @@ export const getFileTableConfig = (
   const { expressApp } = fileTable;
 
   const { tableName: fileTableName, storageClient } = fileTable;
+  const versionTableName = getFileVersionTableName(fileTable);
+  const versionConstraintName = `prostgles_file_version_${md5(JSON.stringify([versionTableName, fileTableName]))}`;
+  const versionIndexName = `prostgles_file_version_${md5(versionTableName)}_key`;
+  if (fileTable.versioning) {
+    if (versionTableName === fileTableName) {
+      throw new Error("File version table name must differ from the file table name");
+    }
+    if (
+      fileTable.versioning.maxVersions !== undefined &&
+      (!Number.isInteger(fileTable.versioning.maxVersions) || fileTable.versioning.maxVersions < 1)
+    ) {
+      throw new Error("fileTable.versioning.maxVersions must be a positive integer");
+    }
+    if (tableConfig?.[versionTableName]) {
+      throw new Error(
+        `File version table name (${versionTableName}) is managed and cannot be configured in tableConfig`,
+      );
+    }
+  }
 
+  const fileColumnDefinitions = {
+    ...FILE_TABLE_COLUMN_DEFINITIONS,
+    ...(fileTable.versioning && FILE_TABLE_VERSION_COLUMN_DEFINITIONS),
+  };
   const userFileTableConfig = tableConfig?.[fileTableName];
   if (userFileTableConfig) {
     if ("isLookupTable" in userFileTableConfig) {
@@ -59,12 +72,21 @@ export const getFileTableConfig = (
     }
     if ("columns" in userFileTableConfig && userFileTableConfig.columns) {
       const userCols = new Set(Object.keys(userFileTableConfig.columns));
-      const clashingColumns = Object.keys(FILE_TABLE_COLUMN_DEFINITIONS).filter((col) =>
+      const clashingFileColumns = Object.keys(fileColumnDefinitions).filter((col) =>
         userCols.has(col),
       );
-      if (clashingColumns.length) {
+      if (clashingFileColumns.length) {
+        throw new Error(
+          `FileManager table name (${fileTableName}) has clashing column names in tableConfig: ${clashingFileColumns}`,
+        );
+      }
+
+      const clashingFileRevisionColumns = Object.keys(FILE_VERSION_TABLE_COLUMN_DEFINITIONS).filter(
+        (col) => userCols.has(col),
+      );
+      if (clashingFileRevisionColumns.length) {
         throw new Error(
-          `FileManager table name (${fileTableName}) has clashing column names in tableConfig: ${clashingColumns}`,
+          `FileManager table name (${fileTableName}) has clashing column names with file revision table in tableConfig: ${clashingFileRevisionColumns}`,
         );
       }
     }
@@ -74,7 +96,7 @@ export const getFileTableConfig = (
     [fileTableName]: {
       ...userFileTableConfig,
       columns: {
-        ...FILE_TABLE_COLUMN_DEFINITIONS,
+        ...fileColumnDefinitions,
         ...userFileTableConfig?.columns,
       },
       onMount: ({ _db }) => {
@@ -89,6 +111,24 @@ export const getFileTableConfig = (
         };
       },
     },
+    ...(fileTable.versioning && {
+      [versionTableName]: {
+        /** Mirror user columns in the file revision table */
+        columns: { ...FILE_VERSION_TABLE_COLUMN_DEFINITIONS, ...userFileTableConfig?.columns },
+        constraints: {
+          [versionConstraintName]: `FOREIGN KEY (file_id) REFERENCES ${asName(fileTableName)}(id) ON DELETE CASCADE`,
+        },
+        indexes: {
+          [versionIndexName]: {
+            unique: true,
+            columns: "file_id, version",
+          },
+        },
+        onMount: async ({ dbo }) => {
+          await backfillFileVersions(dbo, fileTable);
+        },
+      },
+    }),
     ...omitKeys(tableConfig ?? {}, [fileTableName]),
   };
 
@@ -156,9 +196,24 @@ export const getFileTableConfig = (
         } satisfies BeforeEachTsTrigger<FileTableRow, TableHandlers>,
         ...(userFileTableHooks?.beforeEach || []),
       ],
-      onInsteadOfDelete: async ({ onCommit, tx, returningQuery, isOneOrNone, filterOpts }) => {
+      afterEach: [
+        ...(userFileTableHooks?.afterEach || []),
+        {
+          commands: { insert: 1, update: 1 },
+          validate: async ({ row, dbx, onCommit }) => {
+            const fileRow = await getFileTableHandler(dbx, fileTable).findOne({ id: row.id });
+            if (!fileRow) throw new Error(`File not found after write: ${row.id}`);
+            await saveFileVersion(fileTable, fileRow, dbx, onCommit);
+          },
+        } satisfies AfterEachTsTrigger<FileTableRow, TableHandlers> as AfterEachTsTrigger<
+          any,
+          TableHandlers
+        >,
+      ],
+      onInsteadOfDelete: async ({ onCommit, dbx, tx, returningQuery, isOneOrNone, filterOpts }) => {
         return onDeleteFromFileTable(fileTable, {
           onCommit,
+          dbx,
           t: tx,
           returningQuery,
           isOneOrNone,

package/lib/StorageClient/saveFileVersion.ts

@@ -0,0 +1,82 @@
+import { pickKeys } from "prostgles-types";
+import type { TableHandlers } from "../DboBuilder/DboBuilder";
+import type { TableHandler } from "../DboBuilder/TableHandler/TableHandler";
+import type { FileTableConfig } from "../ProstglesTypes";
+import type { OnCommit } from "../PublishParser/publishTypesAndUtils";
+import { deleteUnreferencedFile } from "./deleteUnreferencedFile";
+import type { FileTableRow, FileVersionTableInsertRow } from "./fileTableDefinitions";
+import { getFileRevisionsTableHandler, getFileTableHandler } from "./fileVersionUtils";
+import { getFileStorageKey } from "./getFileStorageKey";
+
+export const saveFileVersion = async (
+  config: FileTableConfig,
+  row: FileTableRow,
+  dbx: TableHandlers,
+  onCommit: OnCommit,
+) => {
+  if (!config.versioning) return;
+  const version = Number(row.version);
+  if (!Number.isInteger(version) || version < 1) {
+    throw new Error(`Invalid file version: ${row.version}`);
+  }
+
+  const versions = getFileRevisionsTableHandler(dbx, config);
+  const versionColumns = (versions as TableHandler).column_names;
+  await versions.insert(getFileVersionRow(row, versionColumns), {
+    onConflict: {
+      action: "DoNothing",
+      conflictColumns: ["file_id", "version"],
+    },
+  });
+
+  const { maxVersions } = config.versioning;
+  if (maxVersions === undefined) return;
+  const removed = (await versions.find(
+    { file_id: row.id },
+    {
+      select: { id: 1, storage_key: 1 },
+      orderBy: { version: -1 },
+      offset: maxVersions,
+      limit: null,
+    },
+  )) as { id: string; storage_key: string }[];
+  if (!removed.length) return;
+
+  await versions.delete({ id: { $in: removed.map(({ id }) => id) } });
+  for (const storageKey of new Set(removed.map(({ storage_key }) => storage_key))) {
+    onCommit(({ db }) => deleteUnreferencedFile(db, config, storageKey));
+  }
+};
+
+export const backfillFileVersions = async (dbo: TableHandlers, config: FileTableConfig) => {
+  if (!config.versioning) return;
+  const files = getFileTableHandler(dbo, config);
+  const fileRows = await files.find({}, { limit: null });
+  if (!fileRows.length) return;
+  console.log(`Backfilling file versions for ${fileRows.length} files`);
+  const fileRevisionsTableHandler = getFileRevisionsTableHandler(dbo, config);
+  const fileRevisionColumns = (fileRevisionsTableHandler as TableHandler).column_names;
+  await fileRevisionsTableHandler.insertMany(
+    fileRows.map((fileRow) => getFileVersionRow(fileRow, fileRevisionColumns)),
+    {
+      onConflict: {
+        action: "DoNothing",
+        conflictColumns: ["file_id", "version"],
+      },
+    },
+  );
+};
+
+/** Must ensure we copy over any columns that might have been created by the user */
+const getFileVersionRow = (row: FileTableRow, cols: string[]): FileVersionTableInsertRow => {
+  const { id, ...insertRow } = pickKeys(row, cols as (keyof FileTableRow)[]);
+  const version = Number(row.version ?? 1);
+  return {
+    ...insertRow,
+    file_id: id,
+    version,
+    storage_key: getFileStorageKey(row),
+    url: `${row.url}?version=${version}`,
+    created: row.updated,
+  };
+};

package/lib/StorageClient/setupFileServeHandler.ts

@@ -6,9 +6,11 @@ import type { DB } from "../initProstgles";
 import type { Prostgles } from "../Prostgles";
 import type { FileTableConfig } from "../ProstglesTypes";
 import { runClientRequest } from "../runClientRequest";
-import type { StorageClient } from "./StorageClientTypes";
-import type { FileTableRow } from "./getFileTableConfig";
+import type { FileVersionTableRow } from "./fileTableDefinitions";
+import { getFileVersionTableName } from "./fileVersionUtils";
 import { getFileStorageKey } from "./getFileStorageKey";
+import type { FileTableRow } from "./getFileTableConfig";
+import type { StorageClient } from "./StorageClientTypes";
 
 export const getFileServeRoute = (config: FileTableConfig) => {
   const fileTableName = config.tableName;
@@ -36,22 +38,39 @@ export const setupFileServeHandler = (
       if (typeof id !== "string" || !id) {
         throw "Invalid media name";
       }
+      const versionParam = req.query.version;
+      const version = typeof versionParam === "string" ? Number(versionParam) : undefined;
+      if (
+        versionParam !== undefined &&
+        (typeof versionParam !== "string" ||
+          !Number.isInteger(version) ||
+          version! < 1 ||
+          String(version) !== versionParam)
+      ) {
+        throw "Invalid file version";
+      }
+
+      if (version && !config.versioning) {
+        throw "File version requested but versioning is not enabled";
+      }
+
+      const select = {
+        id: 1,
+        storage_key: 1,
+        signed_url: 1,
+        signed_url_expires: 1,
+        content_type: 1,
+      } satisfies
+        Partial<Record<keyof FileTableRow, 1>> | Partial<Record<keyof FileVersionTableRow, 1>>;
 
-      const selectParams = {
-        select: {
-          id: 1,
-          storage_key: 1,
-          signed_url: 1,
-          signed_url_expires: 1,
-          content_type: 1,
-        },
-      };
       const file = (await runClientRequest.bind(prg)(
         {
           command: "findOne",
-          tableName: fileTableName,
-          param1: { id },
-          param2: selectParams,
+          tableName: version === undefined ? fileTableName : getFileVersionTableName(config),
+          param1: version === undefined ? { id } : { file_id: id, version },
+          param2: {
+            select,
+          },
           param3: undefined,
         },
         {
@@ -59,16 +78,18 @@ export const setupFileServeHandler = (
           httpReq: req,
         },
         undefined,
-      )) as FileTableRow | undefined;
+      )) as Pick<FileTableRow, keyof typeof select> | undefined;
 
       if (!file) {
-        res.status(HTTP_FAIL_CODES.NOT_FOUND).send("File not found or not allowed");
+        res
+          .status(HTTP_FAIL_CODES.NOT_FOUND)
+          .send(version === undefined ? "File not found or not allowed" : "File version not found");
         return;
       }
 
       if (storageClient.type === "cloud") {
-        let url = file.signed_url;
-        const expires = +(file.signed_url_expires || 0);
+        let url = version === undefined ? file.signed_url : undefined;
+        const expires = version === undefined ? +(file.signed_url_expires || 0) : 0;
 
         const HOUR = 3600 * 1000;
         const EXPIRES = Date.now() + HOUR;
@@ -76,20 +97,22 @@ export const setupFileServeHandler = (
           url = await storageClient.getSignedUrlForDownload(getFileStorageKey(file), 60 * 60);
 
           // Match the storage key to avoid caching a stale URL after replacement; NULL matches legacy files.
-          await db.any(
-            "UPDATE ${fileTableName:name} \
+          if (version === undefined) {
+            await db.any(
+              "UPDATE ${fileTableName:name} \
               SET signed_url = ${signed_url}, \
               signed_url_expires = ${signed_url_expires} \
             WHERE id = ${id} \
             AND storage_key IS NOT DISTINCT FROM ${storage_key}::uuid",
-            {
-              fileTableName,
-              id: file.id,
-              storage_key: file.storage_key,
-              signed_url: url,
-              signed_url_expires: EXPIRES,
-            },
-          );
+              {
+                fileTableName,
+                id: file.id,
+                storage_key: file.storage_key,
+                signed_url: url,
+                signed_url_expires: EXPIRES,
+              },
+            );
+          }
         }
 
         res.redirect(url);

package/lib/TableConfig/TableRowFromColumnDefinitions.ts

@@ -1,25 +1,49 @@
 import type { JSONB } from "prostgles-types";
 
+type ColumnDefinition =
+  | string
+  | { enum: readonly (string | number)[]; nullable?: boolean }
+  | { jsonbSchema: JSONB.JSONBSchema; nullable?: boolean };
+
 /** Row types for the column definitions used by managed tables. */
 export type TableRowFromColumnDefinitions<
-  T extends Record<
-    string,
-    | string
-    | { enum: readonly (string | number)[]; nullable?: boolean }
-    | { jsonbSchema: JSONB.JSONBSchema; nullable?: boolean }
-  >,
-> = {
-  [K in keyof T]: T[K] extends { jsonbSchema: infer S extends JSONB.JSONBSchema }
+  T extends Record<string, ColumnDefinition>,
+> = Omit<TableRow<T>, GeneratedColumnNames<T>> &
+  Partial<Pick<TableRow<T>, GeneratedColumnNames<T>>>;
+
+type TableRow<T extends Record<string, ColumnDefinition>> = {
+  [K in keyof T]: T[K] extends {
+    jsonbSchema: infer S extends JSONB.JSONBSchema;
+  }
     ? JSONB.GetType<S> | (T[K] extends { nullable: true } ? null : never)
     : T[K] extends { enum: readonly (infer V)[] }
       ? V | (T[K] extends { nullable: true } ? null : never)
-      : | (T[K] extends `BYTEA${string}`
-            ? Buffer
-            : T[K] extends `JSONB${string}`
-              ? Record<string, unknown>
-              : string)
+      : | ColumnValue<T[K]>
         | (T[K] extends
             `${string} NOT NULL${string}` | `${string}PRIMARY KEY${string}`
             ? never
             : null);
 };
+
+type GeneratedColumnNames<T extends Record<string, ColumnDefinition>> = {
+  [K in keyof T]: T[K] extends string
+    ? T[K] extends
+        | `${string}DEFAULT${string}`
+        | `${string}SERIAL${string}`
+        | `${string}GENERATED${string}AS IDENTITY${string}`
+      ? K
+      : never
+    : never;
+}[keyof T];
+
+type ColumnValue<T> = T extends `BYTEA${string}`
+  ? Buffer
+  : T extends `JSONB${string}`
+    ? Record<string, unknown>
+    : T extends `${"SMALLINT" | "INTEGER" | "SERIAL"}${string}`
+      ? number
+      : T extends `${"BIGINT" | "BIGSERIAL" | "NUMERIC" | "DECIMAL"}${string}`
+        ? string | number
+        : T extends `${"REAL" | "DOUBLE PRECISION"}${string}`
+          ? number
+          : string;

package/lib/TableConfig/initTableConfig.ts

@@ -183,6 +183,8 @@ export const initTableConfig = async function (this: TableConfigurator) {
     });
     if (tableColumnDefinition?.isCreate) {
       queries.push(...(constraintDefs?.map((c) => c.alterQuery) ?? []));
+      await runQueries(queries);
+      await this.prostgles.rebuildDBO();
     } else if (tableColumnDefinition) {
       const fullSchema = await getFutureTableSchema({
         db: this.db,
@@ -215,17 +217,19 @@ export const initTableConfig = async function (this: TableConfigurator) {
 
       /** Add missing named constraints */
       constraintDefs?.forEach((c) => {
-        if (c.name && !currCons.some((cc) => cc.name === c.name)) {
-          const fc = futureCons.find((nc) => nc.name === c.name);
-          if (fc) {
-            queries.push(`${ALTER_TABLE_Q} ADD CONSTRAINT ${asName(c.name)} ${c.content};`);
-          }
+        if (!c.name) return;
+        const fc = futureCons.find((nc) => nc.name === c.name);
+        if (
+          fc &&
+          !currCons.some((cc) => cc.name === c.name && cc.definition === fc.definition)
+        ) {
+          queries.push(`${ALTER_TABLE_Q} ADD CONSTRAINT ${asName(c.name)} ${c.content};`);
         }
       });
 
       /** Add remaining missing constraints */
       futureCons
-        .filter((nc) => !currCons.some((c) => c.definition === nc.definition))
+        .filter((nc) => !nc.isNamed && !currCons.some((c) => c.definition === nc.definition))
         .forEach((c) => {
           queries.push(`${ALTER_TABLE_Q} ADD ${c.definition};`);
         });

package/lib/index.ts

@@ -52,11 +52,12 @@ export type * from "./TableHooks/TableHooks";
 export * from "./Auth/utils/upsertNamedExpressMiddleware";
 export type { RequestWithUser } from "./Auth/middleware/userContextMiddleware";
 export default prostgles;
-export type { FileTableRow } from "./StorageClient/getFileTableConfig";
+export type { FileTableInsertRow, FileTableRow } from "./StorageClient/getFileTableConfig";
 export { defineJoin };
 export {
   AUDIT_TABLE_COLUMN_DEFINITIONS,
   AUDIT_TABLE_COLUMN_NAMES,
   AUDIT_TABLE_COLUMNS,
+  type AuditTableInsertRow,
   type AuditTableRow,
 } from "./Audit/AuditTable";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "prostgles-server",
-  "version": "4.2.625",
+  "version": "4.2.626",
   "description": "",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",