prostgles-server 4.2.567 → 4.2.570

package/lib/FileManager/FileManager.ts

@@ -1,40 +1,27 @@
-import * as fs from "fs";
 import type * as stream from "stream";
 
 // import * as sharp from "sharp";
-import checkDiskSpace from "check-disk-space";
 
-import type { ALLOWED_CONTENT_TYPE, ALLOWED_EXTENSION, ValidatedColumnInfo } from "prostgles-types";
+import type { ALLOWED_CONTENT_TYPE, ALLOWED_EXTENSION } from "prostgles-types";
 import { CONTENT_TYPE_TO_EXT, getKeys } from "prostgles-types";
-import type { DB, DBHandlerServer, Prostgles } from "../Prostgles";
 
-import * as path from "path";
-import { removeExpressRoute } from "../Auth/utils/removeExpressRoute";
-import type { ExpressApp } from "../RestApi";
-import { getValidatedFileType } from "./getValidatedFileType";
-import { initFileManager } from "./initFileManager";
-import { upload } from "./upload";
-import { uploadStream } from "./uploadStream";
-
-export const HOUR = 3600 * 1000;
+export type LocalConfig = {
+  /**
+   * example: path.join(__dirname+'/media')
+   * note that this location will be relative to the compiled file location
+   */
+  localFolderPath: string;
 
-export const asSQLIdentifier = async (name: string, db: DB): Promise<string> => {
-  return (await db.one<{ name: string }>("select format('%I', $1) as name", [name])).name;
+  /**
+   * Minimum amount of free bytes available to allow saving files
+   * Defaults to 100MB
+   */
+  minFreeBytes?: number;
 };
 
-export type OnProgress = (progress: { total: number; loaded: number }) => void;
-
-type ImageCompressionOptions =
-  | { inside: { width: number; height: number } }
-  | { contain: { width: number } | { height: number } };
+export const HOUR = 3600 * 1000;
 
-/**
- * Deprecated
- */
-export type ImageOptions = {
-  keepMetadata?: boolean;
-  compression?: ImageCompressionOptions;
-};
+export type OnProgress = (progress: { total: number; loaded: number }) => void;
 
 export type UploadedCloudFile = {
   cloud_url: string;
@@ -45,31 +32,22 @@ export type FileUploadArgs = {
   fileName: string;
   contentType: string;
   file: string | Buffer | stream.PassThrough;
-  onFinish: (
-    ...args: [error: Error, result: undefined] | [error: undefined, result: UploadedCloudFile]
-  ) => void;
   onProgress?: (bytesUploaded: number) => void;
 };
-export type CloudClient = {
-  upload: (file: FileUploadArgs) => Promise<void>;
+export type StorageClientBase = {
+  upload: (file: FileUploadArgs) => Promise<UploadedCloudFile>;
   downloadAsStream: (name: string) => Promise<stream.Readable>;
   delete: (fileName: string) => Promise<void>;
-  getSignedUrlForDownload: (fileName: string, expiresInSeconds: number) => Promise<string>;
 };
-
-export type LocalConfig = {
-  /**
-   * example: path.join(__dirname+'/media')
-   * note that this location will be relative to the compiled file location
-   */
+export type LocalStorageClient = StorageClientBase & {
+  type: "local";
   localFolderPath: string;
-
-  /**
-   * Minimum amount of free bytes available to allow saving files
-   * Defaults to 100MB
-   */
-  minFreeBytes?: number;
 };
+export type CloudStorageClient = StorageClientBase & {
+  type: "cloud";
+  getSignedUrlForDownload: (fileName: string, expiresInSeconds: number) => Promise<string>;
+};
+export type StorageClient = LocalStorageClient | CloudStorageClient;
 
 export type UploadItem = {
   name: string;
@@ -96,210 +74,6 @@ export type UploadedItem = {
   content_length: number;
 };
 
-export class FileManager {
-  cloudClient?: CloudClient;
-
-  config: CloudClient | LocalConfig;
-  imageOptions?: ImageOptions;
-
-  prostgles?: Prostgles;
-  get dbo(): DBHandlerServer {
-    if (!this.prostgles?.dbo) {
-      throw "this.prostgles.dbo missing";
-    }
-    return this.prostgles.dbo;
-  }
-
-  get db(): DB {
-    if (!this.prostgles?.db) throw "this.prostgles.db missing";
-    return this.prostgles.db;
-  }
-
-  tableName?: string;
-
-  path?: string;
-  get fileRouteExpress() {
-    return this.path + "/:name";
-  }
-
-  constructor(config: FileManager["config"], imageOptions?: ImageOptions) {
-    this.config = config;
-    this.imageOptions = imageOptions;
-
-    if ("upload" in config) {
-      this.cloudClient = config;
-    }
-
-    const fullConfig = this.prostgles?.opts.fileTable;
-    if (fullConfig?.delayedDelete) {
-      setInterval(
-        () => {
-          void (async () => {
-            const fileTable = fullConfig.tableName;
-            const daysDelay = fullConfig.delayedDelete?.deleteAfterNDays ?? 0;
-            if (fileTable && this.dbo[fileTable]?.delete && daysDelay) {
-              const filesToDelete = await this.dbo[fileTable].find({
-                deleted_from_storage: null,
-                deleted: { ">": Date.now() - daysDelay * HOUR * 24 },
-              });
-              for (const file of filesToDelete) {
-                await this.deleteFile(file.name);
-              }
-            } else {
-              console.error(
-                "FileManager checkInterval delayedDelete FAIL: Could not access file table tableHandler.delete()",
-              );
-            }
-          })();
-        },
-        Math.max(10000, (fullConfig.delayedDelete.checkIntervalHours || 0) * HOUR),
-      );
-    }
-  }
-
-  async getFileStream(name: string): Promise<stream.Readable> {
-    if (this.cloudClient) {
-      return this.cloudClient.downloadAsStream(name);
-    } else if ("localFolderPath" in this.config) {
-      const filePath = path.resolve(`${this.config.localFolderPath}/${name}`);
-      if (!fs.existsSync(filePath)) {
-        throw `File ${filePath} could not be found`;
-      }
-      return fs.createReadStream(filePath, { encoding: undefined });
-    } else throw new Error("Not expected");
-  }
-
-  async deleteFile(name: string) {
-    if (this.cloudClient) {
-      const res = await this.cloudClient.delete(name);
-      return res;
-    } else if ("localFolderPath" in this.config) {
-      const path = `${this.config.localFolderPath}/${name}`;
-      if (!fs.existsSync(path)) {
-        throw `File ${path} could not be found`;
-      }
-      fs.unlinkSync(path);
-      if (fs.existsSync(path)) throw new Error("Could not delete file");
-    }
-    return true;
-  }
-
-  getValidatedFileType = getValidatedFileType.bind(this);
-
-  getLocalFileUrl = (name: string) => (this.path ? `${this.path}/${name}` : "");
-
-  checkFreeSpace = async (folderPath: string, fileSize = 0) => {
-    if (!this.cloudClient && "localFolderPath" in this.config) {
-      const { minFreeBytes = 1.048e6 } = this.config;
-      const required = Math.max(fileSize, minFreeBytes);
-      if (required) {
-        const diskSpace = await checkDiskSpace(folderPath);
-        if (diskSpace.free < required) {
-          const err = `There is not enough space on the server to save files.\nTotal: ${bytesToSize(diskSpace.size)} \nRemaning: ${bytesToSize(diskSpace.free)} \nRequired: ${bytesToSize(required)}`;
-          throw new Error(err);
-        }
-      }
-    }
-  };
-
-  uploadStream = uploadStream.bind(this);
-
-  upload = upload.bind(this);
-
-  uploadAsMedia = async (params: {
-    item: UploadItem;
-    allowedExtensions?: Array<ALLOWED_EXTENSION>;
-    dissallowedExtensions?: Array<ALLOWED_EXTENSION>;
-    imageOptions?: ImageOptions;
-  }): Promise<UploadedItem> => {
-    const { item } = params;
-    const { name, data, content_type } = item;
-    // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
-    if (!data) throw "No file provided";
-    if (!name || typeof name !== "string") throw "Expecting a string name";
-
-    // const type = await this.getMIME(data, name, allowedExtensions, dissallowedExtensions);
-
-    const _data = data;
-
-    /** Resize/compress/remove exif from photos */
-    // if(content_type.startsWith("image") && extension.toLowerCase() !== "gif"){
-
-    //   const compression = imageOptions?.compression
-    //   if(compression){
-    //     console.log("Resizing image")
-    //     let opts;
-    //     if("contain" in compression){
-    //       opts = {
-    //         fit: sharp.fit.contain,
-    //         ...compression.contain
-    //       }
-    //     } else if("inside" in compression){
-    //       opts = {
-    //         fit: sharp.fit.inside,
-    //         ...compression.inside
-    //       }
-    //     }
-    //     _data = await sharp(data)
-    //       .resize(opts as any)
-    //       .withMetadata(Boolean(imageOptions?.keepMetadata) as any)
-    //       // .jpeg({ quality: 80 })
-    //       .toBuffer()
-    //   } else if(!imageOptions?.keepMetadata) {
-    //     /**
-    //      * Remove exif
-    //      */
-    //     // const metadata = await simg.metadata();
-    //     // const simg = await sharp(data);
-
-    //     _data = await sharp(data).clone().withMetadata({
-    //         exif: {}
-    //       })
-    //       .toBuffer()
-    //   }
-    // }
-
-    const res = await this.upload(_data, name, content_type);
-
-    return res;
-  };
-
-  async getFileCloudDownloadURL(fileName: string, expiresInSecondsRaw = 30 * 60) {
-    const expiresInSeconds = Math.max(1, Math.round(expiresInSecondsRaw));
-    return await this.cloudClient!.getSignedUrlForDownload(fileName, expiresInSeconds);
-  }
-
-  parseSQLIdentifier = async (name: string) => asSQLIdentifier(name, this.prostgles!.db!); //  this.prostgles.dbo.sql<"value">("select format('%I', $1)", [name], { returnType: "value" } )
-
-  getColInfo = (args: {
-    tableName: string;
-    colName: string;
-  }): ValidatedColumnInfo["file"] | undefined => {
-    const { colName, tableName } = args;
-    const tableConfig = this.prostgles?.opts.fileTable?.referencedTables?.[tableName];
-    const isReferencingFileTable = this.dbo[tableName]?.columns.some(
-      (c) =>
-        c.name === colName &&
-        c.references &&
-        c.references.some(({ ftable }) => ftable === this.tableName),
-    );
-    const allowAllFiles = { acceptedContent: "*" } as const;
-    if (isReferencingFileTable) {
-      if (tableConfig && typeof tableConfig !== "string") {
-        return tableConfig.referenceColumns[colName] ?? allowAllFiles;
-      }
-      return allowAllFiles;
-    }
-    return undefined;
-  };
-
-  init = initFileManager.bind(this);
-
-  destroy = () => {
-    removeExpressRoute(this.prostgles?.opts.fileTable?.expressApp, [this.fileRouteExpress]);
-  };
-}
-
 export const getFileTypeFromFilename = (
   fileName: string,
 ): { mime: ALLOWED_CONTENT_TYPE; ext: ALLOWED_EXTENSION } | undefined => {

package/lib/FileManager/getFileTableConfig.ts

@@ -0,0 +1,140 @@
+import { omitKeys } from "prostgles-types";
+import { updateFile } from "../DboBuilder/TableHandler/updateFile";
+import { uploadFile } from "../DboBuilder/uploadFile";
+import type { Prostgles } from "../Prostgles";
+import type { TableConfig } from "../TableConfig/TableConfigTypes";
+import { setupFileServeHandler } from "./setupFileServeHandler";
+import { onDeleteFromFileTable } from "../DboBuilder/TableHandler/onDeleteFromFileTable";
+import type { BeforeEachTsTrigger } from "../PublishParser/publishTypesAndUtils";
+
+const FILE_TABLE_COLUMN_DEFINITIONS = {
+  name: `TEXT NOT NULL UNIQUE`,
+  extension: `TEXT NOT NULL`,
+  content_type: `TEXT NOT NULL`,
+  content_length: `BIGINT NOT NULL DEFAULT 0`,
+  added: `TIMESTAMP NOT NULL DEFAULT NOW()`,
+  url: `TEXT NOT NULL`,
+  id: `UUID PRIMARY KEY DEFAULT gen_random_uuid()`,
+  original_name: `TEXT NOT NULL`,
+  description: `TEXT`,
+  cloud_url: `TEXT`,
+  signed_url: `TEXT`,
+  signed_url_expires: `BIGINT`,
+  etag: `TEXT`,
+  deleted: `BIGINT`,
+  deleted_from_storage: `BIGINT`,
+} as const;
+
+type FileTableColumnDefinitions = typeof FILE_TABLE_COLUMN_DEFINITIONS;
+
+export type FileTableRow = {
+  [K in keyof FileTableColumnDefinitions as K]: FileTableColumnDefinitions[K] extends (
+    `${string} NOT NULL${string}` | `${string}PRIMARY KEY${string}`
+  ) ?
+    string
+  : string | null;
+};
+
+export const getFileTableConfig = (prg: Prostgles): TableConfig | undefined => {
+  const { fileTable, tableConfig } = prg.opts;
+  if (!fileTable) {
+    return tableConfig;
+  }
+
+  const { expressApp } = fileTable;
+
+  const { tableName: fileTableName, storageClient } = fileTable;
+
+  const userFileTableConfig = tableConfig?.[fileTableName];
+  if (userFileTableConfig) {
+    if ("isLookupTable" in userFileTableConfig) {
+      throw new Error(
+        `FileManager table name (${fileTableName}) cannot have isLookupTable set in tableConfig`,
+      );
+    }
+    if (userFileTableConfig.dropIfExists || userFileTableConfig.dropIfExistsCascade) {
+      throw new Error(
+        `FileManager table name (${fileTableName}) cannot have dropIfExists or dropIfExistsCascade set in tableConfig`,
+      );
+    }
+    if ("columns" in userFileTableConfig && userFileTableConfig.columns) {
+      const userCols = new Set(Object.keys(userFileTableConfig.columns));
+      const clashingColumns = Object.keys(FILE_TABLE_COLUMN_DEFINITIONS).filter((col) =>
+        userCols.has(col),
+      );
+      if (clashingColumns.length) {
+        throw new Error(
+          `FileManager table name (${fileTableName}) has clashing column names in tableConfig: ${clashingColumns}`,
+        );
+      }
+    }
+  }
+
+  return {
+    [fileTableName]: {
+      ...userFileTableConfig,
+      columns: {
+        ...FILE_TABLE_COLUMN_DEFINITIONS,
+        ...userFileTableConfig?.columns,
+      },
+      onMount: ({ _db }) => {
+        const { destroy } = setupFileServeHandler(_db, fileTable, storageClient, expressApp, prg);
+
+        const maxBfSizeMB = (prg.opts.io?.engine.opts.maxHttpBufferSize || 1e6) / 1e6;
+        console.log(
+          `Prostgles: Initiated file manager. Max allowed file size: ${maxBfSizeMB}MB (maxHttpBufferSize = 1e6). To increase this set maxHttpBufferSize in socket.io server init options`,
+        );
+        return {
+          onUnmount: destroy,
+        };
+      },
+      hooks: {
+        ...userFileTableConfig?.hooks,
+        beforeEach: [
+          {
+            commands: {
+              insert: 1,
+              update: 1,
+            },
+            validate: async ({ data, localParams, command, filter }) => {
+              const tableHandler = prg.dboBuilder.dboMap.get(fileTableName);
+              if (!tableHandler) throw "FileManager tableHandler not found";
+
+              if (command === "update") {
+                const { newData } = await updateFile(tableHandler, fileTable, {
+                  filter: filter ?? {},
+                  localParams,
+                  newData: data,
+                });
+                return {
+                  row: newData,
+                };
+              }
+
+              const media = await uploadFile(fileTable, {
+                row: data,
+                localParams,
+                mediaId: data.id,
+              });
+
+              return {
+                row: media,
+              };
+            },
+          } satisfies BeforeEachTsTrigger<FileTableRow, {}>,
+          ...(userFileTableConfig?.hooks?.beforeEach || []),
+        ],
+        onInsteadOfDelete: async ({ dbx, tx, returningQuery, isOneOrNone, filterOpts }) => {
+          return onDeleteFromFileTable(fileTable, {
+            dbTX: dbx,
+            t: tx,
+            returningQuery,
+            isOneOrNone,
+            filterOpts,
+          });
+        },
+      },
+    },
+    ...omitKeys(tableConfig ?? {}, [fileTableName]),
+  };
+};

package/lib/FileManager/getLocalStorageClient.ts

@@ -0,0 +1,118 @@
+import crypto from "crypto";
+import fs from "fs";
+import path from "path";
+import stream from "stream";
+import type {
+  FileUploadArgs,
+  LocalConfig,
+  LocalStorageClient,
+  UploadedCloudFile,
+} from "./FileManager";
+
+export const getLocalStorageClient = (localConfig: LocalConfig): LocalStorageClient => {
+  const { localFolderPath, minFreeBytes = 100 * 1024 * 1024 } = localConfig; // Default 100MB
+
+  // Helper to ensure we have enough free disk space (Requires Node >= 18.17.0)
+  const checkFreeSpace = async () => {
+    try {
+      const stats = await fs.promises.statfs(localFolderPath);
+      const freeSpace = stats.bavail * stats.bsize;
+
+      if (freeSpace < minFreeBytes) {
+        throw new Error(
+          `Insufficient disk space. Required: ${minFreeBytes} bytes, Available: ${freeSpace} bytes`,
+        );
+      }
+    } catch (error: any) {
+      // If the directory doesn't exist yet, statfs will throw. We'll ignore and let mkdir handle it.
+      if (error.code !== "ENOENT") {
+        throw error;
+      }
+    }
+  };
+
+  return {
+    type: "local",
+    localFolderPath: localConfig.localFolderPath,
+    upload: async (args: FileUploadArgs): Promise<UploadedCloudFile> => {
+      // Ensure the target directory exists
+      await fs.promises.mkdir(localFolderPath, { recursive: true });
+      await checkFreeSpace();
+
+      const filePath = path.join(localFolderPath, args.fileName);
+      const hash = crypto.createHash("md5");
+      let contentLength = 0;
+
+      return new Promise((resolve, reject) => {
+        const writeStream = fs.createWriteStream(filePath);
+
+        writeStream.on("error", reject);
+        writeStream.on("finish", () => {
+          resolve({
+            cloud_url: filePath,
+            etag: hash.digest("hex"),
+            content_length: contentLength,
+          });
+        });
+
+        // Handle raw string or Buffer
+        if (typeof args.file === "string" || Buffer.isBuffer(args.file)) {
+          const buffer = Buffer.isBuffer(args.file) ? args.file : Buffer.from(args.file);
+
+          contentLength = buffer.length;
+          hash.update(buffer);
+
+          if (args.onProgress) {
+            args.onProgress(contentLength);
+          }
+
+          writeStream.write(buffer);
+          writeStream.end();
+        } else if (
+          args.file instanceof stream.Readable ||
+          (args.file as unknown) instanceof stream.PassThrough
+        ) {
+          let lastProgress = Date.now();
+          const throttle = 1000; // 1 second
+          args.file.on("data", (chunk: Buffer) => {
+            contentLength += chunk.length;
+            hash.update(chunk);
+            if (args.onProgress && Date.now() - lastProgress > throttle) {
+              lastProgress = Date.now();
+              args.onProgress(contentLength);
+            }
+          });
+
+          args.file.on("error", reject);
+          args.file.pipe(writeStream);
+        } else {
+          reject(new Error("Unsupported file type provided to upload."));
+        }
+      });
+    },
+
+    downloadAsStream: async (name: string) => {
+      const filePath = path.join(localFolderPath, name);
+
+      try {
+        await fs.promises.access(filePath, fs.constants.R_OK);
+      } catch (err) {
+        throw new Error(`File not found or unreadable: ${name}`);
+      }
+
+      return fs.createReadStream(filePath);
+    },
+
+    delete: async (fileName: string) => {
+      const filePath = path.join(localFolderPath, fileName);
+      try {
+        await fs.promises.unlink(filePath);
+      } catch (err: any) {
+        // Ignore error if file doesn't exist
+        if (err.code !== "ENOENT") {
+          throw err;
+        }
+      }
+    },
+  };
+};

package/lib/FileManager/getValidatedFileType.ts

@@ -1,13 +1,7 @@
-import type {
-  ALLOWED_CONTENT_TYPE} from "prostgles-types";
-import {
-  ALLOWED_EXTENSION,
-  CONTENT_TYPE_TO_EXT,
-  getKeys,
-  isObject,
-} from "prostgles-types";
+import type { ALLOWED_CONTENT_TYPE } from "prostgles-types";
+import { CONTENT_TYPE_TO_EXT, getKeys, isObject } from "prostgles-types";
 import { parseFieldFilter } from "../DboBuilder/ViewHandler/parseFieldFilter";
-import type { FileManager} from "./FileManager";
+import type { FileTableConfig } from "../ProstglesTypes";
 import { getFileType, getFileTypeFromFilename } from "./FileManager";
 
 type Args = {
@@ -16,19 +10,17 @@ type Args = {
   colName?: string;
   tableName?: string;
 };
-export async function getValidatedFileType(
-  this: FileManager,
-  args: Args
+export const getValidatedFileType = async (
+  config: FileTableConfig,
+  args: Args,
 ): Promise<{
   mime: ALLOWED_CONTENT_TYPE;
-  ext: string; //| ALLOWED_EXTENSION;
+  ext: string;
 
   /** File name is not returned because we fail if the extensions do not match */
   // fileName: string;
-}> {
+}> => {
   const { file, fileName, tableName, colName } = args;
-  const config = this.prostgles?.opts.fileTable;
-  if (!config) throw new Error("File table config missing");
 
   const buffer = typeof file === "string" ? Buffer.from(file, "utf8") : file;
 
@@ -55,7 +47,7 @@ export async function getValidatedFileType(
         const allowedContent = parseFieldFilter(colConfig.acceptedContent, false, CONTENTS);
         if (!allowedContent.some((c) => mime.mime.startsWith(c))) {
           throw new Error(
-            `Dissallowed content type provided: ${mime.mime.split("/")[0]}. Allowed content types: ${allowedContent} `
+            `Disallowed content type provided: ${mime.mime.split("/")[0]}. Allowed content types: ${allowedContent} `,
           );
         }
       } else if (
@@ -67,12 +59,12 @@ export async function getValidatedFileType(
         const allowedContentTypes = parseFieldFilter(
           colConfig.acceptedContentType,
           false,
-          getKeys(CONTENT_TYPE_TO_EXT)
+          getKeys(CONTENT_TYPE_TO_EXT),
         );
 
         if (!allowedContentTypes.some((c) => c === mime.mime)) {
           throw new Error(
-            `Dissallowed MIME provided: ${mime.mime}. Allowed MIME values: ${allowedContentTypes} `
+            `Disallowed MIME provided: ${mime.mime}. Allowed MIME values: ${allowedContentTypes} `,
           );
         }
       } else if (
@@ -84,12 +76,12 @@ export async function getValidatedFileType(
         const allowedExtensions = parseFieldFilter(
           colConfig.acceptedFileTypes,
           false,
-          Object.values(CONTENT_TYPE_TO_EXT).flat()
+          Object.values(CONTENT_TYPE_TO_EXT).flat(),
         );
 
         if (!allowedExtensions.some((c) => c === mime.ext)) {
           throw new Error(
-            `Dissallowed extension provided: ${mime.ext}. Allowed extension values: ${allowedExtensions} `
+            `Disallowed extension provided: ${mime.ext}. Allowed extension values: ${allowedExtensions} `,
           );
         }
       }
@@ -97,4 +89,4 @@ export async function getValidatedFileType(
   }
   if (!result?.mime) throw `File MIME type not found for the provided extension: ${result?.ext}`;
   return result;
-}
+};