payload 4.0.0-internal.e7d4ebc → 4.0.0-internal.ec73d44

package/dist/uploads/fileVersioning/cleanup.js

@@ -0,0 +1,300 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { deferFileCleanup } from './fileOperationManager.js';
+import { normalizeStorageKey } from './naming.js';
+import { collectStoredFiles as collectStoredFileLocations, getStoredFileIdentity, withLegacyCloudUploadFileData } from './storedFiles.js';
+const pageSize = 100;
+export const collectStoredFiles = async ({ collection, doc, req })=>{
+    const stored = await withLegacyCloudUploadFileData({
+        collection,
+        doc,
+        req
+    });
+    return collectStoredFileLocations({
+        collection,
+        doc: stored,
+        req
+    });
+};
+/** Reads all saved versions in batches before a parent or its history is deleted. */ export const collectVersionFiles = async ({ collection, parentID, req, where })=>{
+    const files = [];
+    let page = 1;
+    while(true){
+        const versions = await req.payload.db.findVersions({
+            collection: collection.slug,
+            limit: pageSize,
+            page,
+            req,
+            where: where ? {
+                and: [
+                    {
+                        parent: {
+                            equals: parentID
+                        }
+                    },
+                    where
+                ]
+            } : {
+                parent: {
+                    equals: parentID
+                }
+            }
+        });
+        for (const row of versions.docs){
+            files.push(...await collectStoredFiles({
+                collection,
+                doc: row.version,
+                req
+            }));
+        }
+        if (versions.docs.length < pageSize) {
+            return files;
+        }
+        page += 1;
+    }
+};
+/** Collects every stored file that can become unreferenced when a document is deleted. */ export const collectDocumentDeleteFileCandidates = async ({ collection, doc, parentID, req })=>[
+        ...await collectStoredFiles({
+            collection,
+            doc,
+            req
+        }),
+        ...collection.versions ? await collectVersionFiles({
+            collection,
+            parentID,
+            req
+        }) : []
+    ];
+/** Confirms that deleted references have no survivors before and after commit. */ export const scheduleUnreferencedFileCleanup = async ({ candidates, collection, req })=>{
+    const unique = new Map(candidates.map((file)=>[
+            getStoredFileIdentity(file),
+            file
+        ]));
+    if (!unique.size) {
+        return;
+    }
+    const unreferenced = await findUnreferenced({
+        candidates: [
+            ...unique.values()
+        ],
+        collection,
+        req
+    });
+    if (!unreferenced.length) {
+        return;
+    }
+    await deferFileCleanup({
+        cleanup: async ()=>{
+            const stillUnreferenced = await findUnreferenced({
+                candidates: unreferenced,
+                collection,
+                req
+            });
+            for (const file of stillUnreferenced){
+                try {
+                    await deleteStoredFile({
+                        collection,
+                        file,
+                        req
+                    });
+                } catch (err) {
+                    req.payload.logger.error({
+                        err,
+                        msg: `Failed to delete unreferenced upload file ${file.key}. Verify the object is unreferenced and remove it manually.`
+                    });
+                }
+            }
+        },
+        req,
+        shouldCleanupAfterFailure: true
+    });
+};
+/** Removes objects staged by a failed write unless a persisted document or version references them. */ export const removeUnreferencedStagedObjects = async ({ collection, objects, req })=>{
+    const unreferenced = await findUnreferenced({
+        candidates: objects.map(({ key })=>({
+                key,
+                roles: []
+            })),
+        collection,
+        req
+    });
+    const identities = new Set(unreferenced.map(getStoredFileIdentity));
+    for (const object of objects){
+        if (!identities.has(getStoredFileIdentity(object))) {
+            continue;
+        }
+        try {
+            await object.remove();
+        } catch (err) {
+            req.payload.logger.error({
+                err,
+                msg: `Failed to remove staged upload file ${object.key}`
+            });
+        }
+    }
+};
+const findUnreferenced = async ({ candidates, collection, req })=>{
+    const variantsField = collection.flattenedFields.find((field)=>field.name === 'variants');
+    const variantNames = variantsField?.type === 'group' ? variantsField.flattenedFields.filter((field)=>field.type === 'group' && field.flattenedFields.some((nested)=>nested.name === 'filename')).map((field)=>field.name) : [];
+    const filenamePaths = [
+        'filename',
+        'original.filename',
+        ...variantNames.map((name)=>`variants.${name}.filename`)
+    ];
+    // A removed schema field cannot prove the absence of historical references.
+    // Retain those variant objects until their saved location data is migrated.
+    const remaining = new Map(candidates.filter((file)=>file.roles.every((role)=>role.type !== 'size' || variantNames.includes(role.sizeKey))).map((file)=>[
+            getStoredFileIdentity(file),
+            file
+        ]));
+    const representationSelect = {
+        _objectKey: true,
+        filename: true,
+        prefix: true,
+        url: true
+    };
+    const select = {
+        ...representationSelect,
+        filesize: true,
+        mimeType: true,
+        original: representationSelect,
+        ...variantNames.length ? {
+            variants: Object.fromEntries(variantNames.map((name)=>[
+                    name,
+                    representationSelect
+                ]))
+        } : {}
+    };
+    // Legacy filenames may include folders. Match every possible filename suffix,
+    // then verify the adapter's complete key before accepting a reference.
+    const filenames = [
+        ...new Set([
+            ...remaining.values()
+        ].flatMap(({ key })=>{
+            const parts = normalizeStorageKey({
+                key
+            }).split('/');
+            return parts.map((_, index)=>parts.slice(index).join('/'));
+        }))
+    ];
+    for(let offset = 0; offset < filenames.length && remaining.size; offset += pageSize){
+        const batch = filenames.slice(offset, offset + pageSize);
+        const where = {
+            or: filenamePaths.map((field)=>({
+                    [field]: {
+                        in: batch
+                    }
+                }))
+        };
+        let page = 1;
+        while(remaining.size){
+            const docs = await req.payload.db.find({
+                collection: collection.slug,
+                limit: pageSize,
+                page,
+                pagination: true,
+                req,
+                select,
+                sort: 'id',
+                where
+            });
+            for (const doc of docs.docs){
+                await removeReferenced({
+                    collection,
+                    doc,
+                    remaining,
+                    req
+                });
+            }
+            if (docs.docs.length < pageSize) {
+                break;
+            }
+            page += 1;
+        }
+        if (collection.versions) {
+            page = 1;
+            const versionWhere = {
+                or: filenamePaths.map((field)=>({
+                        [`version.${field}`]: {
+                            in: batch
+                        }
+                    }))
+            };
+            while(remaining.size){
+                const versions = await req.payload.db.findVersions({
+                    collection: collection.slug,
+                    limit: pageSize,
+                    page,
+                    pagination: true,
+                    req,
+                    select: {
+                        version: select
+                    },
+                    sort: 'id',
+                    where: versionWhere
+                });
+                for (const row of versions.docs){
+                    await removeReferenced({
+                        collection,
+                        doc: row.version,
+                        remaining,
+                        req
+                    });
+                }
+                if (versions.docs.length < pageSize) {
+                    break;
+                }
+                page += 1;
+            }
+        }
+    }
+    return [
+        ...remaining.values()
+    ];
+};
+const removeReferenced = async ({ collection, doc, remaining, req })=>{
+    for (const file of (await collectStoredFiles({
+        collection,
+        doc,
+        req
+    }))){
+        remaining.delete(getStoredFileIdentity(file));
+    }
+};
+const deleteStoredFile = async ({ collection, file, req })=>{
+    const operations = collection.upload.fileOperations;
+    if (!operations) {
+        const staticDir = collection.upload.staticDir;
+        if (!staticDir) {
+            throw new Error(`No local storage directory is configured for ${collection.slug}`);
+        }
+        const key = normalizeStorageKey({
+            key: file.key
+        });
+        const directory = await fs.realpath(staticDir);
+        const target = path.resolve(directory, key);
+        const relative = path.relative(directory, target);
+        if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) {
+            throw new Error(`Invalid local storage key: ${key}`);
+        }
+        try {
+            const parent = await fs.realpath(path.dirname(target));
+            const parentRelative = path.relative(directory, parent);
+            if (parentRelative.startsWith('..') || path.isAbsolute(parentRelative)) {
+                throw new Error(`Local storage key escapes its directory: ${key}`);
+            }
+            await fs.unlink(target);
+        } catch (err) {
+            if (err.code !== 'ENOENT') {
+                throw err;
+            }
+        }
+        return;
+    }
+    await operations.delete({
+        key: file.key,
+        req
+    });
+};
+
+//# sourceMappingURL=cleanup.js.map

package/dist/uploads/fileVersioning/cleanup.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/uploads/fileVersioning/cleanup.ts"],"sourcesContent":["import fs from 'node:fs/promises'\nimport path from 'node:path'\n\nimport type { SanitizedCollectionConfig } from '../../collections/config/types.js'\nimport type { JsonObject, PayloadRequest, SelectType, Where } from '../../types/index.js'\nimport type { StagedObject } from './fileOperationManager.js'\nimport type { StoredFile, StoredFileList } from './types.js'\n\nimport { deferFileCleanup } from './fileOperationManager.js'\nimport { normalizeStorageKey } from './naming.js'\nimport {\n  collectStoredFiles as collectStoredFileLocations,\n  getStoredFileIdentity,\n  withLegacyCloudUploadFileData,\n} from './storedFiles.js'\n\nconst pageSize = 100\n\nexport const collectStoredFiles = async ({\n  collection,\n  doc,\n  req,\n}: {\n  collection: SanitizedCollectionConfig\n  doc: JsonObject\n  req: PayloadRequest\n}): Promise<StoredFileList> => {\n  const stored = await withLegacyCloudUploadFileData({\n    collection,\n    doc,\n    req,\n  })\n\n  return collectStoredFileLocations({ collection, doc: stored, req })\n}\n\n/** Reads all saved versions in batches before a parent or its history is deleted. */\nexport const collectVersionFiles = async ({\n  collection,\n  parentID,\n  req,\n  where,\n}: {\n  collection: SanitizedCollectionConfig\n  parentID: number | string\n  req: PayloadRequest\n  where?: Where\n}): Promise<StoredFileList> => {\n  const files: StoredFileList = []\n  let page = 1\n\n  while (true) {\n    const versions = await req.payload.db.findVersions<JsonObject>({\n      collection: collection.slug,\n      limit: pageSize,\n      page,\n      req,\n      where: where\n        ? { and: [{ parent: { equals: parentID } }, where] }\n        : { parent: { equals: parentID } },\n    })\n\n    for (const row of versions.docs) {\n      files.push(...(await collectStoredFiles({ collection, doc: row.version, req })))\n    }\n\n    if (versions.docs.length < pageSize) {\n      return files\n    }\n    page += 1\n  }\n}\n\n/** Collects every stored file that can become unreferenced when a document is deleted. */\nexport const collectDocumentDeleteFileCandidates = async ({\n  collection,\n  doc,\n  parentID,\n  req,\n}: {\n  collection: SanitizedCollectionConfig\n  doc: JsonObject\n  parentID: number | string\n  req: PayloadRequest\n}): Promise<StoredFileList> => [\n  ...(await collectStoredFiles({ collection, doc, req })),\n  ...(collection.versions ? await collectVersionFiles({ collection, parentID, req }) : []),\n]\n\n/** Confirms that deleted references have no survivors before and after commit. */\nexport const scheduleUnreferencedFileCleanup = async ({\n  candidates,\n  collection,\n  req,\n}: {\n  candidates: StoredFileList\n  collection: SanitizedCollectionConfig\n  req: PayloadRequest\n}): Promise<void> => {\n  const unique = new Map(candidates.map((file) => [getStoredFileIdentity(file), file]))\n\n  if (!unique.size) {\n    return\n  }\n\n  const unreferenced = await findUnreferenced({ candidates: [...unique.values()], collection, req })\n\n  if (!unreferenced.length) {\n    return\n  }\n\n  await deferFileCleanup({\n    cleanup: async () => {\n      const stillUnreferenced = await findUnreferenced({\n        candidates: unreferenced,\n        collection,\n        req,\n      })\n\n      for (const file of stillUnreferenced) {\n        try {\n          await deleteStoredFile({ collection, file, req })\n        } catch (err) {\n          req.payload.logger.error({\n            err,\n            msg: `Failed to delete unreferenced upload file ${file.key}. Verify the object is unreferenced and remove it manually.`,\n          })\n        }\n      }\n    },\n    req,\n    shouldCleanupAfterFailure: true,\n  })\n}\n\n/** Removes objects staged by a failed write unless a persisted document or version references them. */\nexport const removeUnreferencedStagedObjects = async ({\n  collection,\n  objects,\n  req,\n}: {\n  collection: SanitizedCollectionConfig\n  objects: StagedObject[]\n  req: PayloadRequest\n}): Promise<void> => {\n  const unreferenced = await findUnreferenced({\n    candidates: objects.map(({ key }) => ({ key, roles: [] })),\n    collection,\n    req,\n  })\n  const identities = new Set(unreferenced.map(getStoredFileIdentity))\n\n  for (const object of objects) {\n    if (!identities.has(getStoredFileIdentity(object))) {\n      continue\n    }\n\n    try {\n      await object.remove()\n    } catch (err) {\n      req.payload.logger.error({\n        err,\n        msg: `Failed to remove staged upload file ${object.key}`,\n      })\n    }\n  }\n}\n\nconst findUnreferenced = async ({\n  candidates,\n  collection,\n  req,\n}: {\n  candidates: StoredFileList\n  collection: SanitizedCollectionConfig\n  req: PayloadRequest\n}): Promise<StoredFileList> => {\n  const variantsField = collection.flattenedFields.find((field) => field.name === 'variants')\n  const variantNames =\n    variantsField?.type === 'group'\n      ? variantsField.flattenedFields\n          .filter(\n            (field) =>\n              field.type === 'group' &&\n              field.flattenedFields.some((nested) => nested.name === 'filename'),\n          )\n          .map((field) => field.name)\n      : []\n  const filenamePaths = [\n    'filename',\n    'original.filename',\n    ...variantNames.map((name) => `variants.${name}.filename`),\n  ]\n  // A removed schema field cannot prove the absence of historical references.\n  // Retain those variant objects until their saved location data is migrated.\n  const remaining = new Map(\n    candidates\n      .filter((file) =>\n        file.roles.every((role) => role.type !== 'size' || variantNames.includes(role.sizeKey)),\n      )\n      .map((file) => [getStoredFileIdentity(file), file]),\n  )\n  const representationSelect: SelectType = {\n    _objectKey: true,\n    filename: true,\n    prefix: true,\n    url: true,\n  }\n  const select: SelectType = {\n    ...representationSelect,\n    filesize: true,\n    mimeType: true,\n    original: representationSelect,\n    ...(variantNames.length\n      ? { variants: Object.fromEntries(variantNames.map((name) => [name, representationSelect])) }\n      : {}),\n  }\n  // Legacy filenames may include folders. Match every possible filename suffix,\n  // then verify the adapter's complete key before accepting a reference.\n  const filenames = [\n    ...new Set(\n      [...remaining.values()].flatMap(({ key }) => {\n        const parts = normalizeStorageKey({ key }).split('/')\n        return parts.map((_, index) => parts.slice(index).join('/'))\n      }),\n    ),\n  ]\n\n  for (let offset = 0; offset < filenames.length && remaining.size; offset += pageSize) {\n    const batch = filenames.slice(offset, offset + pageSize)\n    const where: Where = { or: filenamePaths.map((field) => ({ [field]: { in: batch } })) }\n    let page = 1\n\n    while (remaining.size) {\n      const docs = await req.payload.db.find<JsonObject>({\n        collection: collection.slug,\n        limit: pageSize,\n        page,\n        pagination: true,\n        req,\n        select,\n        sort: 'id',\n        where,\n      })\n\n      for (const doc of docs.docs) {\n        await removeReferenced({ collection, doc, remaining, req })\n      }\n      if (docs.docs.length < pageSize) {\n        break\n      }\n      page += 1\n    }\n\n    if (collection.versions) {\n      page = 1\n      const versionWhere: Where = {\n        or: filenamePaths.map((field) => ({ [`version.${field}`]: { in: batch } })),\n      }\n\n      while (remaining.size) {\n        const versions = await req.payload.db.findVersions<JsonObject>({\n          collection: collection.slug,\n          limit: pageSize,\n          page,\n          pagination: true,\n          req,\n          select: { version: select },\n          sort: 'id',\n          where: versionWhere,\n        })\n\n        for (const row of versions.docs) {\n          await removeReferenced({ collection, doc: row.version, remaining, req })\n        }\n        if (versions.docs.length < pageSize) {\n          break\n        }\n        page += 1\n      }\n    }\n  }\n\n  return [...remaining.values()]\n}\n\nconst removeReferenced = async ({\n  collection,\n  doc,\n  remaining,\n  req,\n}: {\n  collection: SanitizedCollectionConfig\n  doc: JsonObject\n  remaining: Map<string, StoredFile>\n  req: PayloadRequest\n}): Promise<void> => {\n  for (const file of await collectStoredFiles({ collection, doc, req })) {\n    remaining.delete(getStoredFileIdentity(file))\n  }\n}\n\nconst deleteStoredFile = async ({\n  collection,\n  file,\n  req,\n}: {\n  collection: SanitizedCollectionConfig\n  file: StoredFile\n  req: PayloadRequest\n}): Promise<void> => {\n  const operations = collection.upload.fileOperations\n  if (!operations) {\n    const staticDir = collection.upload.staticDir\n\n    if (!staticDir) {\n      throw new Error(`No local storage directory is configured for ${collection.slug}`)\n    }\n\n    const key = normalizeStorageKey({ key: file.key })\n    const directory = await fs.realpath(staticDir)\n    const target = path.resolve(directory, key)\n    const relative = path.relative(directory, target)\n\n    if (!relative || relative.startsWith('..') || path.isAbsolute(relative)) {\n      throw new Error(`Invalid local storage key: ${key}`)\n    }\n\n    try {\n      const parent = await fs.realpath(path.dirname(target))\n      const parentRelative = path.relative(directory, parent)\n\n      if (parentRelative.startsWith('..') || path.isAbsolute(parentRelative)) {\n        throw new Error(`Local storage key escapes its directory: ${key}`)\n      }\n\n      await fs.unlink(target)\n    } catch (err) {\n      if ((err as NodeJS.ErrnoException).code !== 'ENOENT') {\n        throw err\n      }\n    }\n\n    return\n  }\n\n  await operations.delete({ key: file.key, req })\n}\n"],"names":["fs","path","deferFileCleanup","normalizeStorageKey","collectStoredFiles","collectStoredFileLocations","getStoredFileIdentity","withLegacyCloudUploadFileData","pageSize","collection","doc","req","stored","collectVersionFiles","parentID","where","files","page","versions","payload","db","findVersions","slug","limit","and","parent","equals","row","docs","push","version","length","collectDocumentDeleteFileCandidates","scheduleUnreferencedFileCleanup","candidates","unique","Map","map","file","size","unreferenced","findUnreferenced","values","cleanup","stillUnreferenced","deleteStoredFile","err","logger","error","msg","key","shouldCleanupAfterFailure","removeUnreferencedStagedObjects","objects","roles","identities","Set","object","has","remove","variantsField","flattenedFields","find","field","name","variantNames","type","filter","some","nested","filenamePaths","remaining","every","role","includes","sizeKey","representationSelect","_objectKey","filename","prefix","url","select","filesize","mimeType","original","variants","Object","fromEntries","filenames","flatMap","parts","split","_","index","slice","join","offset","batch","or","in","pagination","sort","removeReferenced","versionWhere","delete","operations","upload","fileOperations","staticDir","Error","directory","realpath","target","resolve","relative","startsWith","isAbsolute","dirname","parentRelative","unlink","code"],"mappings":"AAAA,OAAOA,QAAQ,mBAAkB;AACjC,OAAOC,UAAU,YAAW;AAO5B,SAASC,gBAAgB,QAAQ,4BAA2B;AAC5D,SAASC,mBAAmB,QAAQ,cAAa;AACjD,SACEC,sBAAsBC,0BAA0B,EAChDC,qBAAqB,EACrBC,6BAA6B,QACxB,mBAAkB;AAEzB,MAAMC,WAAW;AAEjB,OAAO,MAAMJ,qBAAqB,OAAO,EACvCK,UAAU,EACVC,GAAG,EACHC,GAAG,EAKJ;IACC,MAAMC,SAAS,MAAML,8BAA8B;QACjDE;QACAC;QACAC;IACF;IAEA,OAAON,2BAA2B;QAAEI;QAAYC,KAAKE;QAAQD;IAAI;AACnE,EAAC;AAED,mFAAmF,GACnF,OAAO,MAAME,sBAAsB,OAAO,EACxCJ,UAAU,EACVK,QAAQ,EACRH,GAAG,EACHI,KAAK,EAMN;IACC,MAAMC,QAAwB,EAAE;IAChC,IAAIC,OAAO;IAEX,MAAO,KAAM;QACX,MAAMC,WAAW,MAAMP,IAAIQ,OAAO,CAACC,EAAE,CAACC,YAAY,CAAa;YAC7DZ,YAAYA,WAAWa,IAAI;YAC3BC,OAAOf;YACPS;YACAN;YACAI,OAAOA,QACH;gBAAES,KAAK;oBAAC;wBAAEC,QAAQ;4BAAEC,QAAQZ;wBAAS;oBAAE;oBAAGC;iBAAM;YAAC,IACjD;gBAAEU,QAAQ;oBAAEC,QAAQZ;gBAAS;YAAE;QACrC;QAEA,KAAK,MAAMa,OAAOT,SAASU,IAAI,CAAE;YAC/BZ,MAAMa,IAAI,IAAK,MAAMzB,mBAAmB;gBAAEK;gBAAYC,KAAKiB,IAAIG,OAAO;gBAAEnB;YAAI;QAC9E;QAEA,IAAIO,SAASU,IAAI,CAACG,MAAM,GAAGvB,UAAU;YACnC,OAAOQ;QACT;QACAC,QAAQ;IACV;AACF,EAAC;AAED,wFAAwF,GACxF,OAAO,MAAMe,sCAAsC,OAAO,EACxDvB,UAAU,EACVC,GAAG,EACHI,QAAQ,EACRH,GAAG,EAMJ,GAA8B;WACzB,MAAMP,mBAAmB;YAAEK;YAAYC;YAAKC;QAAI;WAChDF,WAAWS,QAAQ,GAAG,MAAML,oBAAoB;YAAEJ;YAAYK;YAAUH;QAAI,KAAK,EAAE;KACxF,CAAA;AAED,gFAAgF,GAChF,OAAO,MAAMsB,kCAAkC,OAAO,EACpDC,UAAU,EACVzB,UAAU,EACVE,GAAG,EAKJ;IACC,MAAMwB,SAAS,IAAIC,IAAIF,WAAWG,GAAG,CAAC,CAACC,OAAS;YAAChC,sBAAsBgC;YAAOA;SAAK;IAEnF,IAAI,CAACH,OAAOI,IAAI,EAAE;QAChB;IACF;IAEA,MAAMC,eAAe,MAAMC,iBAAiB;QAAEP,YAAY;eAAIC,OAAOO,MAAM;SAAG;QAAEjC;QAAYE;IAAI;IAEhG,IAAI,CAAC6B,aAAaT,MAAM,EAAE;QACxB;IACF;IAEA,MAAM7B,iBAAiB;QACrByC,SAAS;YACP,MAAMC,oBAAoB,MAAMH,iBAAiB;gBAC/CP,YAAYM;gBACZ/B;gBACAE;YACF;YAEA,KAAK,MAAM2B,QAAQM,kBAAmB;gBACpC,IAAI;oBACF,MAAMC,iBAAiB;wBAAEpC;wBAAY6B;wBAAM3B;oBAAI;gBACjD,EAAE,OAAOmC,KAAK;oBACZnC,IAAIQ,OAAO,CAAC4B,MAAM,CAACC,KAAK,CAAC;wBACvBF;wBACAG,KAAK,CAAC,0CAA0C,EAAEX,KAAKY,GAAG,CAAC,2DAA2D,CAAC;oBACzH;gBACF;YACF;QACF;QACAvC;QACAwC,2BAA2B;IAC7B;AACF,EAAC;AAED,qGAAqG,GACrG,OAAO,MAAMC,kCAAkC,OAAO,EACpD3C,UAAU,EACV4C,OAAO,EACP1C,GAAG,EAKJ;IACC,MAAM6B,eAAe,MAAMC,iBAAiB;QAC1CP,YAAYmB,QAAQhB,GAAG,CAAC,CAAC,EAAEa,GAAG,EAAE,GAAM,CAAA;gBAAEA;gBAAKI,OAAO,EAAE;YAAC,CAAA;QACvD7C;QACAE;IACF;IACA,MAAM4C,aAAa,IAAIC,IAAIhB,aAAaH,GAAG,CAAC/B;IAE5C,KAAK,MAAMmD,UAAUJ,QAAS;QAC5B,IAAI,CAACE,WAAWG,GAAG,CAACpD,sBAAsBmD,UAAU;YAClD;QACF;QAEA,IAAI;YACF,MAAMA,OAAOE,MAAM;QACrB,EAAE,OAAOb,KAAK;YACZnC,IAAIQ,OAAO,CAAC4B,MAAM,CAACC,KAAK,CAAC;gBACvBF;gBACAG,KAAK,CAAC,oCAAoC,EAAEQ,OAAOP,GAAG,EAAE;YAC1D;QACF;IACF;AACF,EAAC;AAED,MAAMT,mBAAmB,OAAO,EAC9BP,UAAU,EACVzB,UAAU,EACVE,GAAG,EAKJ;IACC,MAAMiD,gBAAgBnD,WAAWoD,eAAe,CAACC,IAAI,CAAC,CAACC,QAAUA,MAAMC,IAAI,KAAK;IAChF,MAAMC,eACJL,eAAeM,SAAS,UACpBN,cAAcC,eAAe,CAC1BM,MAAM,CACL,CAACJ,QACCA,MAAMG,IAAI,KAAK,WACfH,MAAMF,eAAe,CAACO,IAAI,CAAC,CAACC,SAAWA,OAAOL,IAAI,KAAK,aAE1D3B,GAAG,CAAC,CAAC0B,QAAUA,MAAMC,IAAI,IAC5B,EAAE;IACR,MAAMM,gBAAgB;QACpB;QACA;WACGL,aAAa5B,GAAG,CAAC,CAAC2B,OAAS,CAAC,SAAS,EAAEA,KAAK,SAAS,CAAC;KAC1D;IACD,4EAA4E;IAC5E,4EAA4E;IAC5E,MAAMO,YAAY,IAAInC,IACpBF,WACGiC,MAAM,CAAC,CAAC7B,OACPA,KAAKgB,KAAK,CAACkB,KAAK,CAAC,CAACC,OAASA,KAAKP,IAAI,KAAK,UAAUD,aAAaS,QAAQ,CAACD,KAAKE,OAAO,IAEtFtC,GAAG,CAAC,CAACC,OAAS;YAAChC,sBAAsBgC;YAAOA;SAAK;IAEtD,MAAMsC,uBAAmC;QACvCC,YAAY;QACZC,UAAU;QACVC,QAAQ;QACRC,KAAK;IACP;IACA,MAAMC,SAAqB;QACzB,GAAGL,oBAAoB;QACvBM,UAAU;QACVC,UAAU;QACVC,UAAUR;QACV,GAAIX,aAAalC,MAAM,GACnB;YAAEsD,UAAUC,OAAOC,WAAW,CAACtB,aAAa5B,GAAG,CAAC,CAAC2B,OAAS;oBAACA;oBAAMY;iBAAqB;QAAG,IACzF,CAAC,CAAC;IACR;IACA,8EAA8E;IAC9E,uEAAuE;IACvE,MAAMY,YAAY;WACb,IAAIhC,IACL;eAAIe,UAAU7B,MAAM;SAAG,CAAC+C,OAAO,CAAC,CAAC,EAAEvC,GAAG,EAAE;YACtC,MAAMwC,QAAQvF,oBAAoB;gBAAE+C;YAAI,GAAGyC,KAAK,CAAC;YACjD,OAAOD,MAAMrD,GAAG,CAAC,CAACuD,GAAGC,QAAUH,MAAMI,KAAK,CAACD,OAAOE,IAAI,CAAC;QACzD;KAEH;IAED,IAAK,IAAIC,SAAS,GAAGA,SAASR,UAAUzD,MAAM,IAAIwC,UAAUhC,IAAI,EAAEyD,UAAUxF,SAAU;QACpF,MAAMyF,QAAQT,UAAUM,KAAK,CAACE,QAAQA,SAASxF;QAC/C,MAAMO,QAAe;YAAEmF,IAAI5B,cAAcjC,GAAG,CAAC,CAAC0B,QAAW,CAAA;oBAAE,CAACA,MAAM,EAAE;wBAAEoC,IAAIF;oBAAM;gBAAE,CAAA;QAAI;QACtF,IAAIhF,OAAO;QAEX,MAAOsD,UAAUhC,IAAI,CAAE;YACrB,MAAMX,OAAO,MAAMjB,IAAIQ,OAAO,CAACC,EAAE,CAAC0C,IAAI,CAAa;gBACjDrD,YAAYA,WAAWa,IAAI;gBAC3BC,OAAOf;gBACPS;gBACAmF,YAAY;gBACZzF;gBACAsE;gBACAoB,MAAM;gBACNtF;YACF;YAEA,KAAK,MAAML,OAAOkB,KAAKA,IAAI,CAAE;gBAC3B,MAAM0E,iBAAiB;oBAAE7F;oBAAYC;oBAAK6D;oBAAW5D;gBAAI;YAC3D;YACA,IAAIiB,KAAKA,IAAI,CAACG,MAAM,GAAGvB,UAAU;gBAC/B;YACF;YACAS,QAAQ;QACV;QAEA,IAAIR,WAAWS,QAAQ,EAAE;YACvBD,OAAO;YACP,MAAMsF,eAAsB;gBAC1BL,IAAI5B,cAAcjC,GAAG,CAAC,CAAC0B,QAAW,CAAA;wBAAE,CAAC,CAAC,QAAQ,EAAEA,OAAO,CAAC,EAAE;4BAAEoC,IAAIF;wBAAM;oBAAE,CAAA;YAC1E;YAEA,MAAO1B,UAAUhC,IAAI,CAAE;gBACrB,MAAMrB,WAAW,MAAMP,IAAIQ,OAAO,CAACC,EAAE,CAACC,YAAY,CAAa;oBAC7DZ,YAAYA,WAAWa,IAAI;oBAC3BC,OAAOf;oBACPS;oBACAmF,YAAY;oBACZzF;oBACAsE,QAAQ;wBAAEnD,SAASmD;oBAAO;oBAC1BoB,MAAM;oBACNtF,OAAOwF;gBACT;gBAEA,KAAK,MAAM5E,OAAOT,SAASU,IAAI,CAAE;oBAC/B,MAAM0E,iBAAiB;wBAAE7F;wBAAYC,KAAKiB,IAAIG,OAAO;wBAAEyC;wBAAW5D;oBAAI;gBACxE;gBACA,IAAIO,SAASU,IAAI,CAACG,MAAM,GAAGvB,UAAU;oBACnC;gBACF;gBACAS,QAAQ;YACV;QACF;IACF;IAEA,OAAO;WAAIsD,UAAU7B,MAAM;KAAG;AAChC;AAEA,MAAM4D,mBAAmB,OAAO,EAC9B7F,UAAU,EACVC,GAAG,EACH6D,SAAS,EACT5D,GAAG,EAMJ;IACC,KAAK,MAAM2B,QAAQ,CAAA,MAAMlC,mBAAmB;QAAEK;QAAYC;QAAKC;IAAI,EAAC,EAAG;QACrE4D,UAAUiC,MAAM,CAAClG,sBAAsBgC;IACzC;AACF;AAEA,MAAMO,mBAAmB,OAAO,EAC9BpC,UAAU,EACV6B,IAAI,EACJ3B,GAAG,EAKJ;IACC,MAAM8F,aAAahG,WAAWiG,MAAM,CAACC,cAAc;IACnD,IAAI,CAACF,YAAY;QACf,MAAMG,YAAYnG,WAAWiG,MAAM,CAACE,SAAS;QAE7C,IAAI,CAACA,WAAW;YACd,MAAM,IAAIC,MAAM,CAAC,6CAA6C,EAAEpG,WAAWa,IAAI,EAAE;QACnF;QAEA,MAAM4B,MAAM/C,oBAAoB;YAAE+C,KAAKZ,KAAKY,GAAG;QAAC;QAChD,MAAM4D,YAAY,MAAM9G,GAAG+G,QAAQ,CAACH;QACpC,MAAMI,SAAS/G,KAAKgH,OAAO,CAACH,WAAW5D;QACvC,MAAMgE,WAAWjH,KAAKiH,QAAQ,CAACJ,WAAWE;QAE1C,IAAI,CAACE,YAAYA,SAASC,UAAU,CAAC,SAASlH,KAAKmH,UAAU,CAACF,WAAW;YACvE,MAAM,IAAIL,MAAM,CAAC,2BAA2B,EAAE3D,KAAK;QACrD;QAEA,IAAI;YACF,MAAMzB,SAAS,MAAMzB,GAAG+G,QAAQ,CAAC9G,KAAKoH,OAAO,CAACL;YAC9C,MAAMM,iBAAiBrH,KAAKiH,QAAQ,CAACJ,WAAWrF;YAEhD,IAAI6F,eAAeH,UAAU,CAAC,SAASlH,KAAKmH,UAAU,CAACE,iBAAiB;gBACtE,MAAM,IAAIT,MAAM,CAAC,yCAAyC,EAAE3D,KAAK;YACnE;YAEA,MAAMlD,GAAGuH,MAAM,CAACP;QAClB,EAAE,OAAOlE,KAAK;YACZ,IAAI,AAACA,IAA8B0E,IAAI,KAAK,UAAU;gBACpD,MAAM1E;YACR;QACF;QAEA;IACF;IAEA,MAAM2D,WAAWD,MAAM,CAAC;QAAEtD,KAAKZ,KAAKY,GAAG;QAAEvC;IAAI;AAC/C"}

package/dist/uploads/fileVersioning/cloudStorage.d.ts

@@ -0,0 +1,24 @@
+import type { SanitizedCollectionConfig } from '../../collections/config/types.js';
+import type { JsonObject, PayloadRequest } from '../../types/index.js';
+import type { FileToSave } from '../types.js';
+/** Preserve the caller's cloud flags; nested Local API calls can replace req.context. */
+export declare const captureCloudHookState: ({ req }: {
+    req: PayloadRequest;
+}) => (() => void);
+export declare const runCloudFileCreation: <T>({ collection, data, files, req, write, }: {
+    collection: SanitizedCollectionConfig;
+    data: JsonObject;
+    files: FileToSave[];
+    req: PayloadRequest;
+    write: () => Promise<T>;
+}) => Promise<T>;
+export declare const runCloudFileUpdate: <T>({ id, collection, current, data, files, req, write, }: {
+    collection: SanitizedCollectionConfig;
+    current: JsonObject;
+    data: JsonObject;
+    files: FileToSave[];
+    id: number | string;
+    req: PayloadRequest;
+    write: () => Promise<T>;
+}) => Promise<T>;
+//# sourceMappingURL=cloudStorage.d.ts.map

package/dist/uploads/fileVersioning/cloudStorage.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cloudStorage.d.ts","sourceRoot":"","sources":["../../../src/uploads/fileVersioning/cloudStorage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAA;AAClF,OAAO,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AACtE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AAuB7C,yFAAyF;AACzF,eAAO,MAAM,qBAAqB,YAAa;IAAE,GAAG,EAAE,cAAc,CAAA;CAAE,KAAG,CAAC,MAAM,IAAI,CAmBnF,CAAA;AAED,eAAO,MAAM,oBAAoB,GAAU,CAAC,4CAMzC;IACD,UAAU,EAAE,yBAAyB,CAAA;IACrC,IAAI,EAAE,UAAU,CAAA;IAChB,KAAK,EAAE,UAAU,EAAE,CAAA;IACnB,GAAG,EAAE,cAAc,CAAA;IACnB,KAAK,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,CAAA;CACxB,KAAG,OAAO,CAAC,CAAC,CAqCZ,CAAA;AAED,eAAO,MAAM,kBAAkB,GAAU,CAAC,yDAQvC;IACD,UAAU,EAAE,yBAAyB,CAAA;IACrC,OAAO,EAAE,UAAU,CAAA;IACnB,IAAI,EAAE,UAAU,CAAA;IAChB,KAAK,EAAE,UAAU,EAAE,CAAA;IACnB,EAAE,EAAE,MAAM,GAAG,MAAM,CAAA;IACnB,GAAG,EAAE,cAAc,CAAA;IACnB,KAAK,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,CAAA;CACxB,KAAG,OAAO,CAAC,CAAC,CAqEZ,CAAA"}

package/dist/uploads/fileVersioning/cloudStorage.js

@@ -0,0 +1,275 @@
+import { resolveBranchRowID } from '../../branching/resolveBranchRowID.js';
+import { branchField, branchParentField } from '../../branching/types.js';
+import { saveVersion } from '../../versions/saveVersion.js';
+import { collectStoredFiles as collectSavedFiles, removeUnreferencedStagedObjects, scheduleUnreferencedFileCleanup } from './cleanup.js';
+import { runFileCreationPlan, runFileOperationPlan } from './fileOperationManager.js';
+import { collectStoredFiles, getStoredFileIdentity, withLegacyCloudUploadFileData } from './storedFiles.js';
+/** Preserve the caller's cloud flags; nested Local API calls can replace req.context. */ export const captureCloudHookState = ({ req })=>{
+    const previousState = [
+        '_payloadManagedCloudStorage',
+        '_payloadManagedCloudMetadata'
+    ].map((key)=>({
+            hasProperty: Object.hasOwn(req.context ?? {}, key),
+            key,
+            value: req.context?.[key]
+        }));
+    return ()=>{
+        req.context ??= {};
+        for (const { hasProperty, key, value } of previousState){
+            if (hasProperty) {
+                req.context[key] = value;
+            } else {
+                delete req.context[key];
+            }
+        }
+    };
+};
+export const runCloudFileCreation = async ({ collection, data, files, req, write })=>{
+    const operations = collection.upload.fileOperations;
+    if (!operations || req.context?.skipCloudStorage || files.length === 0 && !req.context?._payloadVerifiedProviderOriginal) {
+        return write();
+    }
+    let metadata = {};
+    return runFileCreationPlan({
+        cleanupStagedAfterWriteFailure: (objects)=>removeUnreferencedStagedObjects({
+                collection,
+                objects,
+                req
+            }),
+        req,
+        stage: async ({ trackStagedObject })=>{
+            trackClientUploadOriginal({
+                collection,
+                data,
+                req,
+                trackStagedObject
+            });
+            const staged = await operations.stage({
+                data,
+                files,
+                req,
+                trackStagedObject
+            });
+            metadata = staged.metadata;
+            Object.assign(data, metadata);
+        },
+        // Create's version and afterChange hooks run after the database insert. The
+        // outer create operation restores the previous guard after those hooks finish.
+        write: ()=>{
+            req.context ??= {};
+            req.context._payloadManagedCloudStorage = true;
+            req.context._payloadManagedCloudMetadata = metadata;
+            return write();
+        }
+    });
+};
+export const runCloudFileUpdate = async ({ id, collection, current, data, files, req, write })=>{
+    const operations = collection.upload.fileOperations;
+    const storedCurrent = operations ? await withLegacyCloudUploadFileData({
+        collection,
+        doc: current,
+        req
+    }) : current;
+    const nextFiles = collectStoredFiles({
+        collection,
+        doc: {
+            ...storedCurrent,
+            ...data
+        },
+        req,
+        trustGenerated: true
+    });
+    const currentFiles = collectStoredFiles({
+        collection,
+        doc: storedCurrent,
+        req
+    });
+    const nextIdentities = new Set(nextFiles.map(getStoredFileIdentity));
+    const hasStoredFileChange = nextIdentities.size !== currentFiles.length || currentFiles.some((file)=>!nextIdentities.has(getStoredFileIdentity(file)));
+    if (!operations || req.context?.skipCloudStorage || files.length === 0 && !req.context?._payloadVerifiedProviderOriginal && !hasStoredFileChange) {
+        return write();
+    }
+    let metadata = {};
+    return runFileOperationPlan({
+        cleanupStagedAfterWriteFailure: (objects)=>removeUnreferencedStagedObjects({
+                collection,
+                objects,
+                req
+            }),
+        req,
+        stage: async ({ trackStagedObject })=>{
+            if (files.length === 0 && !req.context?._payloadVerifiedProviderOriginal) {
+                return;
+            }
+            trackClientUploadOriginal({
+                collection,
+                data,
+                req,
+                trackStagedObject
+            });
+            const staged = await operations.stage({
+                data,
+                files,
+                req,
+                trackStagedObject
+            });
+            metadata = staged.metadata;
+            Object.assign(data, metadata);
+        },
+        write: async ()=>{
+            if (collection.versions && !hasStoredOriginal({
+                doc: current
+            }) && hasStoredOriginal({
+                doc: storedCurrent
+            })) {
+                await persistLegacyCloudVersions({
+                    id,
+                    collection,
+                    current: storedCurrent,
+                    req
+                });
+            }
+            const result = await withCloudHookGuard({
+                metadata,
+                req,
+                write
+            });
+            await scheduleUnreferencedFileCleanup({
+                candidates: await collectSavedFiles({
+                    collection,
+                    doc: storedCurrent,
+                    req
+                }),
+                collection,
+                req
+            });
+            return result;
+        }
+    });
+};
+/** Track a verified client-uploaded source, retaining it if another document still references it. */ const trackClientUploadOriginal = ({ collection, data, req, trackStagedObject })=>{
+    const verifiedOriginal = req.context?._payloadVerifiedProviderOriginal;
+    if (!verifiedOriginal) {
+        return;
+    }
+    const original = collectStoredFiles({
+        collection,
+        doc: data,
+        req,
+        trustGenerated: true
+    }).find(({ key, roles })=>key === verifiedOriginal.key && roles.some((role)=>role.type === 'original'));
+    if (!original) {
+        return;
+    }
+    const object = {
+        key: original.key,
+        remove: ()=>collection.upload.fileOperations.delete({
+                key: original.key,
+                req
+            })
+    };
+    trackStagedObject({
+        key: object.key,
+        remove: ()=>removeUnreferencedStagedObjects({
+                collection,
+                objects: [
+                    object
+                ],
+                req
+            })
+    });
+};
+const persistLegacyCloudVersions = async ({ id, collection, current, req })=>{
+    const versionParent = await resolveBranchRowID({
+        id,
+        collectionSlug: collection.slug,
+        req
+    });
+    let page = 1;
+    let hasVersions = false;
+    while(true){
+        const versions = await req.payload.db.findVersions({
+            branch: false,
+            collection: collection.slug,
+            limit: 100,
+            page,
+            req,
+            where: {
+                parent: {
+                    equals: versionParent
+                }
+            }
+        });
+        for (const row of versions.docs){
+            const branchVersion = row;
+            hasVersions = true;
+            if (hasStoredOriginal({
+                doc: row.version
+            })) {
+                continue;
+            }
+            const version = await withLegacyCloudUploadFileData({
+                collection,
+                doc: row.version,
+                req
+            });
+            if (!hasStoredOriginal({
+                doc: version
+            })) {
+                continue;
+            }
+            await req.payload.db.updateVersion({
+                id: row.id,
+                collection: collection.slug,
+                req,
+                versionData: {
+                    ...branchVersion[branchField] !== undefined ? {
+                        [branchField]: branchVersion[branchField]
+                    } : {},
+                    ...branchVersion[branchParentField] !== undefined ? {
+                        [branchParentField]: branchVersion[branchParentField]
+                    } : {},
+                    createdAt: row.createdAt,
+                    latest: row.latest,
+                    parent: row.parent,
+                    publishedLocale: row.publishedLocale,
+                    updatedAt: row.updatedAt,
+                    version
+                }
+            });
+        }
+        if (versions.docs.length < 100) {
+            break;
+        }
+        page += 1;
+    }
+    if (!hasVersions && hasStoredOriginal({
+        doc: current
+    })) {
+        await saveVersion({
+            id,
+            collection,
+            docWithLocales: current,
+            draft: false,
+            operation: 'update',
+            payload: req.payload,
+            req
+        });
+    }
+};
+const hasStoredOriginal = ({ doc })=>Boolean(doc.original && typeof doc.original === 'object' && typeof doc.original.filename === 'string');
+const withCloudHookGuard = async ({ metadata, req, write })=>{
+    const restoreCloudHookState = captureCloudHookState({
+        req
+    });
+    req.context ??= {};
+    req.context._payloadManagedCloudStorage = true;
+    req.context._payloadManagedCloudMetadata = metadata;
+    try {
+        return await write();
+    } finally{
+        restoreCloudHookState();
+    }
+};
+
+//# sourceMappingURL=cloudStorage.js.map