prostgles-server 4.2.567 → 4.2.569

package/lib/DboBuilder/getColumns.ts

@@ -44,7 +44,7 @@ export async function getColumns(
       }
     }
 
-    const columns: ValidatedColumnInfo[] = this.columns
+    const rawColumns: ValidatedColumnInfo[] = this.columns
       .filter((c) => {
         const { insert, select, update } = rules;
 
@@ -64,13 +64,13 @@ export async function getColumns(
         let update = !!c.privileges.UPDATE;
 
         const prostgles = this.dboBuilder.prostgles;
-        const fileConfig = prostgles.fileManager?.getColInfo({
+        const fileConfig = getFileColumnInfo({
           colName: c.name,
-          tableName: this.name,
+          tableHandler: this,
         });
 
         /** Do not allow updates to file table unless it's to delete fields */
-        if (prostgles.fileManager?.config && prostgles.fileManager.tableName === this.name) {
+        if (prostgles.opts.fileTable?.tableName === this.name) {
           update = false;
         }
 
@@ -110,6 +110,17 @@ export async function getColumns(
       })
       .filter((c) => c.select || c.update || c.delete || c.insert);
 
+    const modifiedTableSchema = await this.dboBuilder.prostgles.opts.modifyClientSchema?.(
+      {
+        name: this.name,
+        ...(await this.getInfo()),
+        columns: rawColumns,
+      },
+      localParams?.isRemoteRequest?.clientInfo,
+    );
+
+    const columns = modifiedTableSchema?.columns ?? rawColumns;
+
     await this._log({
       command: "getColumns",
       localParams,
@@ -141,3 +152,28 @@ function capitalizeFirstLetter(string: string, nonalpha_replacement?: string): s
   const str = replaceNonAlphaNumeric(string, nonalpha_replacement);
   return str.charAt(0).toUpperCase() + str.slice(1);
 }
+
+const getFileColumnInfo = (args: {
+  tableHandler: ViewHandler;
+  colName: string;
+}): ValidatedColumnInfo["file"] | undefined => {
+  const fileTableConfig = args.tableHandler.dboBuilder.prostgles.opts.fileTable;
+  if (!fileTableConfig) return undefined;
+  const { colName, tableHandler } = args;
+  const tableName = tableHandler.name;
+  const tableConfig = fileTableConfig.referencedTables?.[tableName];
+  const isReferencingFileTable = tableHandler.columns.some(
+    (c) =>
+      c.name === colName &&
+      c.references &&
+      c.references.some(({ ftable }) => ftable === fileTableConfig.tableName),
+  );
+  const allowAllFiles = { acceptedContent: "*" } as const;
+  if (isReferencingFileTable) {
+    if (tableConfig && typeof tableConfig !== "string") {
+      return tableConfig.referenceColumns[colName] ?? allowAllFiles;
+    }
+    return allowAllFiles;
+  }
+  return undefined;
+};

package/lib/DboBuilder/uploadFile.ts

@@ -1,8 +1,10 @@
+import { randomUUID } from "crypto";
 import type { AnyObject } from "prostgles-types";
 import { getKeys, isObject } from "prostgles-types";
-import type { LocalParams, Media } from "./DboBuilder";
-import type { ValidateRowBasic } from "../PublishParser/PublishParser";
-import type { TableHandler } from "./TableHandler/TableHandler";
+import type { FileTableRow } from "../FileManager/getFileTableConfig";
+import { getValidatedFileType } from "../FileManager/getValidatedFileType";
+import type { FileTableConfig } from "../ProstglesTypes";
+import type { LocalParams } from "./DboBuilder";
 
 export const isFile = (row: any): row is { data: Buffer; name: string } => {
   return Boolean(
@@ -17,41 +19,39 @@ export const isFile = (row: any): row is { data: Buffer; name: string } => {
 
 type UploadFileArgs = {
   row: AnyObject;
-  validate: ValidateRowBasic | undefined;
   localParams: LocalParams | undefined;
   /**
    * Used to update an existing file
    */
-  mediaId?: string;
+  mediaId: string | undefined;
 };
 
-export async function uploadFile(
-  this: TableHandler,
-  { row, localParams, validate, mediaId }: UploadFileArgs,
-): Promise<Media> {
-  if (!this.dboBuilder.prostgles.fileManager) throw "fileManager not set up";
-
-  if (!isFile(row))
+export const uploadFile = async (
+  config: FileTableConfig,
+  { row, localParams, mediaId }: UploadFileArgs,
+): Promise<FileTableRow> => {
+  if (!isFile(row)) {
     throw (
       "Expecting only two properties for file upload: { name: string; data: File | string | Buffer }; but got: " +
       Object.entries(row)
         .map(([k, v]) => `${k}: ${typeof v}`)
         .join(", ")
     );
+  }
+  const storageClient = config.storageClient;
   const { data, name } = row;
 
-  const media_id =
-    mediaId ?? (await this.db.one<{ name: string }>("SELECT gen_random_uuid() as name")).name;
+  const media_id = mediaId ?? randomUUID();
   const nestedInsert = localParams?.nestedInsert;
-  const type = await this.dboBuilder.prostgles.fileManager.getValidatedFileType({
+  const type = await getValidatedFileType(config, {
     file: data,
     fileName: name,
     tableName: nestedInsert?.previousTable,
     colName: nestedInsert?.referencingColumn,
   });
   const media_name = `${media_id}.${type.ext}`;
-  const parsedMediaKeys = ["id", "name", "original_name", "extension", "content_type"] as const;
-  const media: Required<Pick<Media, (typeof parsedMediaKeys)[number]>> = {
+  const _parsedMediaKeys = ["id", "name", "original_name", "extension", "content_type"] as const;
+  const coreInfo: Required<Pick<FileTableRow, (typeof _parsedMediaKeys)[number]>> = {
     id: media_id,
     name: media_name,
     original_name: name,
@@ -59,41 +59,25 @@ export async function uploadFile(
     content_type: type.mime,
   };
 
-  if (validate) {
-    if (!localParams) throw "localParams missing";
-    const parsedMedia = await validate({
-      tx: localParams.tx?.t || this.tx?.t || this.db,
-      row: media,
-      dbx: this.getFinalDbo(localParams),
-      localParams,
-      command: "insert",
-      data: media,
-    });
-    const missingKeys = parsedMediaKeys.filter((k) => !parsedMedia[k]);
-    if (missingKeys.length) {
-      throw `Some keys are missing from file insert validation: ${missingKeys.join(", ")}`;
-    }
-  }
-
-  const _media: Media = await this.dboBuilder.prostgles.fileManager.uploadAsMedia({
-    item: {
-      data,
-      name: media.name,
-      content_type: media.content_type,
-      extension: media.extension,
-    },
-    // imageCompression: {
-    //     inside: {
-    //         width: 1100,
-    //         height: 630
-    //     }
-    // }
+  const uploadedInfo = await storageClient.upload({
+    file: data,
+    fileName: coreInfo.name,
+    contentType: coreInfo.content_type,
   });
 
-  const mediaRow = {
-    ...media,
-    ..._media,
+  const mediaRow: FileTableRow = {
+    signed_url: null,
+    signed_url_expires: null,
+    added: new Date().toISOString(),
+    deleted: null,
+    deleted_from_storage: null,
+    description: "",
+    ...coreInfo,
+    ...uploadedInfo,
+    extension: type.ext,
+    url: uploadedInfo.cloud_url,
+    content_length: String(uploadedInfo.content_length),
   };
 
   return mediaRow;
-}
+};

package/lib/FileManager/FileManager.ts

@@ -1,40 +1,27 @@
-import * as fs from "fs";
 import type * as stream from "stream";
 
 // import * as sharp from "sharp";
-import checkDiskSpace from "check-disk-space";
 
-import type { ALLOWED_CONTENT_TYPE, ALLOWED_EXTENSION, ValidatedColumnInfo } from "prostgles-types";
+import type { ALLOWED_CONTENT_TYPE, ALLOWED_EXTENSION } from "prostgles-types";
 import { CONTENT_TYPE_TO_EXT, getKeys } from "prostgles-types";
-import type { DB, DBHandlerServer, Prostgles } from "../Prostgles";
 
-import * as path from "path";
-import { removeExpressRoute } from "../Auth/utils/removeExpressRoute";
-import type { ExpressApp } from "../RestApi";
-import { getValidatedFileType } from "./getValidatedFileType";
-import { initFileManager } from "./initFileManager";
-import { upload } from "./upload";
-import { uploadStream } from "./uploadStream";
-
-export const HOUR = 3600 * 1000;
+export type LocalConfig = {
+  /**
+   * example: path.join(__dirname+'/media')
+   * note that this location will be relative to the compiled file location
+   */
+  localFolderPath: string;
 
-export const asSQLIdentifier = async (name: string, db: DB): Promise<string> => {
-  return (await db.one<{ name: string }>("select format('%I', $1) as name", [name])).name;
+  /**
+   * Minimum amount of free bytes available to allow saving files
+   * Defaults to 100MB
+   */
+  minFreeBytes?: number;
 };
 
-export type OnProgress = (progress: { total: number; loaded: number }) => void;
-
-type ImageCompressionOptions =
-  | { inside: { width: number; height: number } }
-  | { contain: { width: number } | { height: number } };
+export const HOUR = 3600 * 1000;
 
-/**
- * Deprecated
- */
-export type ImageOptions = {
-  keepMetadata?: boolean;
-  compression?: ImageCompressionOptions;
-};
+export type OnProgress = (progress: { total: number; loaded: number }) => void;
 
 export type UploadedCloudFile = {
   cloud_url: string;
@@ -45,31 +32,22 @@ export type FileUploadArgs = {
   fileName: string;
   contentType: string;
   file: string | Buffer | stream.PassThrough;
-  onFinish: (
-    ...args: [error: Error, result: undefined] | [error: undefined, result: UploadedCloudFile]
-  ) => void;
   onProgress?: (bytesUploaded: number) => void;
 };
-export type CloudClient = {
-  upload: (file: FileUploadArgs) => Promise<void>;
+export type StorageClientBase = {
+  upload: (file: FileUploadArgs) => Promise<UploadedCloudFile>;
   downloadAsStream: (name: string) => Promise<stream.Readable>;
   delete: (fileName: string) => Promise<void>;
-  getSignedUrlForDownload: (fileName: string, expiresInSeconds: number) => Promise<string>;
 };
-
-export type LocalConfig = {
-  /**
-   * example: path.join(__dirname+'/media')
-   * note that this location will be relative to the compiled file location
-   */
+export type LocalStorageClient = StorageClientBase & {
+  type: "local";
   localFolderPath: string;
-
-  /**
-   * Minimum amount of free bytes available to allow saving files
-   * Defaults to 100MB
-   */
-  minFreeBytes?: number;
 };
+export type CloudStorageClient = StorageClientBase & {
+  type: "cloud";
+  getSignedUrlForDownload: (fileName: string, expiresInSeconds: number) => Promise<string>;
+};
+export type StorageClient = LocalStorageClient | CloudStorageClient;
 
 export type UploadItem = {
   name: string;
@@ -96,210 +74,6 @@ export type UploadedItem = {
   content_length: number;
 };
 
-export class FileManager {
-  cloudClient?: CloudClient;
-
-  config: CloudClient | LocalConfig;
-  imageOptions?: ImageOptions;
-
-  prostgles?: Prostgles;
-  get dbo(): DBHandlerServer {
-    if (!this.prostgles?.dbo) {
-      throw "this.prostgles.dbo missing";
-    }
-    return this.prostgles.dbo;
-  }
-
-  get db(): DB {
-    if (!this.prostgles?.db) throw "this.prostgles.db missing";
-    return this.prostgles.db;
-  }
-
-  tableName?: string;
-
-  path?: string;
-  get fileRouteExpress() {
-    return this.path + "/:name";
-  }
-
-  constructor(config: FileManager["config"], imageOptions?: ImageOptions) {
-    this.config = config;
-    this.imageOptions = imageOptions;
-
-    if ("upload" in config) {
-      this.cloudClient = config;
-    }
-
-    const fullConfig = this.prostgles?.opts.fileTable;
-    if (fullConfig?.delayedDelete) {
-      setInterval(
-        () => {
-          void (async () => {
-            const fileTable = fullConfig.tableName;
-            const daysDelay = fullConfig.delayedDelete?.deleteAfterNDays ?? 0;
-            if (fileTable && this.dbo[fileTable]?.delete && daysDelay) {
-              const filesToDelete = await this.dbo[fileTable].find({
-                deleted_from_storage: null,
-                deleted: { ">": Date.now() - daysDelay * HOUR * 24 },
-              });
-              for (const file of filesToDelete) {
-                await this.deleteFile(file.name);
-              }
-            } else {
-              console.error(
-                "FileManager checkInterval delayedDelete FAIL: Could not access file table tableHandler.delete()",
-              );
-            }
-          })();
-        },
-        Math.max(10000, (fullConfig.delayedDelete.checkIntervalHours || 0) * HOUR),
-      );
-    }
-  }
-
-  async getFileStream(name: string): Promise<stream.Readable> {
-    if (this.cloudClient) {
-      return this.cloudClient.downloadAsStream(name);
-    } else if ("localFolderPath" in this.config) {
-      const filePath = path.resolve(`${this.config.localFolderPath}/${name}`);
-      if (!fs.existsSync(filePath)) {
-        throw `File ${filePath} could not be found`;
-      }
-      return fs.createReadStream(filePath, { encoding: undefined });
-    } else throw new Error("Not expected");
-  }
-
-  async deleteFile(name: string) {
-    if (this.cloudClient) {
-      const res = await this.cloudClient.delete(name);
-      return res;
-    } else if ("localFolderPath" in this.config) {
-      const path = `${this.config.localFolderPath}/${name}`;
-      if (!fs.existsSync(path)) {
-        throw `File ${path} could not be found`;
-      }
-      fs.unlinkSync(path);
-      if (fs.existsSync(path)) throw new Error("Could not delete file");
-    }
-    return true;
-  }
-
-  getValidatedFileType = getValidatedFileType.bind(this);
-
-  getLocalFileUrl = (name: string) => (this.path ? `${this.path}/${name}` : "");
-
-  checkFreeSpace = async (folderPath: string, fileSize = 0) => {
-    if (!this.cloudClient && "localFolderPath" in this.config) {
-      const { minFreeBytes = 1.048e6 } = this.config;
-      const required = Math.max(fileSize, minFreeBytes);
-      if (required) {
-        const diskSpace = await checkDiskSpace(folderPath);
-        if (diskSpace.free < required) {
-          const err = `There is not enough space on the server to save files.\nTotal: ${bytesToSize(diskSpace.size)} \nRemaning: ${bytesToSize(diskSpace.free)} \nRequired: ${bytesToSize(required)}`;
-          throw new Error(err);
-        }
-      }
-    }
-  };
-
-  uploadStream = uploadStream.bind(this);
-
-  upload = upload.bind(this);
-
-  uploadAsMedia = async (params: {
-    item: UploadItem;
-    allowedExtensions?: Array<ALLOWED_EXTENSION>;
-    dissallowedExtensions?: Array<ALLOWED_EXTENSION>;
-    imageOptions?: ImageOptions;
-  }): Promise<UploadedItem> => {
-    const { item } = params;
-    const { name, data, content_type } = item;
-    // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
-    if (!data) throw "No file provided";
-    if (!name || typeof name !== "string") throw "Expecting a string name";
-
-    // const type = await this.getMIME(data, name, allowedExtensions, dissallowedExtensions);
-
-    const _data = data;
-
-    /** Resize/compress/remove exif from photos */
-    // if(content_type.startsWith("image") && extension.toLowerCase() !== "gif"){
-
-    //   const compression = imageOptions?.compression
-    //   if(compression){
-    //     console.log("Resizing image")
-    //     let opts;
-    //     if("contain" in compression){
-    //       opts = {
-    //         fit: sharp.fit.contain,
-    //         ...compression.contain
-    //       }
-    //     } else if("inside" in compression){
-    //       opts = {
-    //         fit: sharp.fit.inside,
-    //         ...compression.inside
-    //       }
-    //     }
-    //     _data = await sharp(data)
-    //       .resize(opts as any)
-    //       .withMetadata(Boolean(imageOptions?.keepMetadata) as any)
-    //       // .jpeg({ quality: 80 })
-    //       .toBuffer()
-    //   } else if(!imageOptions?.keepMetadata) {
-    //     /**
-    //      * Remove exif
-    //      */
-    //     // const metadata = await simg.metadata();
-    //     // const simg = await sharp(data);
-
-    //     _data = await sharp(data).clone().withMetadata({
-    //         exif: {}
-    //       })
-    //       .toBuffer()
-    //   }
-    // }
-
-    const res = await this.upload(_data, name, content_type);
-
-    return res;
-  };
-
-  async getFileCloudDownloadURL(fileName: string, expiresInSecondsRaw = 30 * 60) {
-    const expiresInSeconds = Math.max(1, Math.round(expiresInSecondsRaw));
-    return await this.cloudClient!.getSignedUrlForDownload(fileName, expiresInSeconds);
-  }
-
-  parseSQLIdentifier = async (name: string) => asSQLIdentifier(name, this.prostgles!.db!); //  this.prostgles.dbo.sql<"value">("select format('%I', $1)", [name], { returnType: "value" } )
-
-  getColInfo = (args: {
-    tableName: string;
-    colName: string;
-  }): ValidatedColumnInfo["file"] | undefined => {
-    const { colName, tableName } = args;
-    const tableConfig = this.prostgles?.opts.fileTable?.referencedTables?.[tableName];
-    const isReferencingFileTable = this.dbo[tableName]?.columns.some(
-      (c) =>
-        c.name === colName &&
-        c.references &&
-        c.references.some(({ ftable }) => ftable === this.tableName),
-    );
-    const allowAllFiles = { acceptedContent: "*" } as const;
-    if (isReferencingFileTable) {
-      if (tableConfig && typeof tableConfig !== "string") {
-        return tableConfig.referenceColumns[colName] ?? allowAllFiles;
-      }
-      return allowAllFiles;
-    }
-    return undefined;
-  };
-
-  init = initFileManager.bind(this);
-
-  destroy = () => {
-    removeExpressRoute(this.prostgles?.opts.fileTable?.expressApp, [this.fileRouteExpress]);
-  };
-}
-
 export const getFileTypeFromFilename = (
   fileName: string,
 ): { mime: ALLOWED_CONTENT_TYPE; ext: ALLOWED_EXTENSION } | undefined => {

package/lib/FileManager/getFileTableConfig.ts

@@ -0,0 +1,142 @@
+import { omitKeys } from "prostgles-types";
+import { updateFile } from "../DboBuilder/TableHandler/updateFile";
+import { uploadFile } from "../DboBuilder/uploadFile";
+import type { Prostgles } from "../Prostgles";
+import type { TableConfig } from "../TableConfig/TableConfigTypes";
+import { setupFileServeHandler } from "./setupFileServeHandler";
+import { onDeleteFromFileTable } from "../DboBuilder/TableHandler/onDeleteFromFileTable";
+import type { BeforeEachTsTrigger } from "../PublishParser/publishTypesAndUtils";
+
+const FILE_TABLE_COLUMN_DEFINITIONS = {
+  name: `TEXT NOT NULL UNIQUE`,
+  extension: `TEXT NOT NULL`,
+  content_type: `TEXT NOT NULL`,
+  content_length: `BIGINT NOT NULL DEFAULT 0`,
+  added: `TIMESTAMP NOT NULL DEFAULT NOW()`,
+  url: `TEXT NOT NULL`,
+  id: `UUID PRIMARY KEY DEFAULT gen_random_uuid()`,
+  original_name: `TEXT NOT NULL`,
+  description: `TEXT`,
+  cloud_url: `TEXT`,
+  signed_url: `TEXT`,
+  signed_url_expires: `BIGINT`,
+  etag: `TEXT`,
+  deleted: `BIGINT`,
+  deleted_from_storage: `BIGINT`,
+} as const;
+
+type FileTableColumnDefinitions = typeof FILE_TABLE_COLUMN_DEFINITIONS;
+
+export type FileTableRow = {
+  [K in keyof FileTableColumnDefinitions as K]: FileTableColumnDefinitions[K] extends (
+    `${string} NOT NULL${string}` | `${string}PRIMARY KEY${string}`
+  ) ?
+    string
+  : string | null;
+};
+
+export const getFileTableConfig = (prg: Prostgles): TableConfig | undefined => {
+  const { fileTable, tableConfig, expressApp } = prg.opts;
+  if (!fileTable) {
+    return tableConfig;
+  }
+
+  if (!expressApp) {
+    throw new Error(`FileManager requires an expressApp to be provided in prostgles.opts.`);
+  }
+
+  const { tableName: fileTableName, storageClient } = fileTable;
+
+  const userFileTableConfig = tableConfig?.[fileTableName];
+  if (userFileTableConfig) {
+    if ("isLookupTable" in userFileTableConfig) {
+      throw new Error(
+        `FileManager table name (${fileTableName}) cannot have isLookupTable set in tableConfig`,
+      );
+    }
+    if (userFileTableConfig.dropIfExists || userFileTableConfig.dropIfExistsCascade) {
+      throw new Error(
+        `FileManager table name (${fileTableName}) cannot have dropIfExists or dropIfExistsCascade set in tableConfig`,
+      );
+    }
+    if ("columns" in userFileTableConfig && userFileTableConfig.columns) {
+      const userCols = new Set(Object.keys(userFileTableConfig.columns));
+      const clashingColumns = Object.keys(FILE_TABLE_COLUMN_DEFINITIONS).filter((col) =>
+        userCols.has(col),
+      );
+      if (clashingColumns.length) {
+        throw new Error(
+          `FileManager table name (${fileTableName}) has clashing column names in tableConfig: ${clashingColumns}`,
+        );
+      }
+    }
+  }
+
+  return {
+    [fileTableName]: {
+      ...userFileTableConfig,
+      columns: {
+        ...FILE_TABLE_COLUMN_DEFINITIONS,
+        ...userFileTableConfig?.columns,
+      },
+      onMount: ({ _db }) => {
+        const { destroy } = setupFileServeHandler(_db, fileTable, storageClient, expressApp, prg);
+
+        const maxBfSizeMB = (prg.opts.io?.engine.opts.maxHttpBufferSize || 1e6) / 1e6;
+        console.log(
+          `Prostgles: Initiated file manager. Max allowed file size: ${maxBfSizeMB}MB (maxHttpBufferSize = 1e6). To increase this set maxHttpBufferSize in socket.io server init options`,
+        );
+        return {
+          onUnmount: destroy,
+        };
+      },
+      hooks: {
+        ...userFileTableConfig?.hooks,
+        beforeEach: [
+          {
+            commands: {
+              insert: 1,
+              update: 1,
+            },
+            validate: async ({ data, localParams, command, filter }) => {
+              const tableHandler = prg.dboBuilder.dboMap.get(fileTableName);
+              if (!tableHandler) throw "FileManager tableHandler not found";
+
+              if (command === "update") {
+                const { newData } = await updateFile(tableHandler, fileTable, {
+                  filter: filter ?? {},
+                  localParams,
+                  newData: data,
+                });
+                return {
+                  row: newData,
+                };
+              }
+
+              const media = await uploadFile(fileTable, {
+                row: data,
+                localParams,
+                mediaId: data.id,
+              });
+
+              return {
+                row: media,
+              };
+            },
+          } satisfies BeforeEachTsTrigger<FileTableRow, {}>,
+          ...(userFileTableConfig?.hooks?.beforeEach || []),
+        ],
+        onInsteadOfDelete: async ({ dbx, tx, returningQuery, isOneOrNone, filterOpts }) => {
+          return onDeleteFromFileTable(fileTable, {
+            dbTX: dbx,
+            t: tx,
+            returningQuery,
+            isOneOrNone,
+            filterOpts,
+          });
+        },
+      },
+    },
+    ...omitKeys(tableConfig ?? {}, [fileTableName]),
+  };
+};