piper-utils 1.1.70 → 1.1.72

package/src/database/dbUtils/queryStringUtils/getPagination.test.js

@@ -0,0 +1,83 @@
+import { getPagination } from './getPagination.js';
+
+describe('getPagination', function () {
+    it('should default to limit 10 offset 0 when nothing is supplied', function () {
+        // Act
+        const result = getPagination({});
+
+        // Assert
+        expect(result).toEqual({ limit: 10, offset: 0 });
+    });
+
+    it('should tolerate a missing event', function () {
+        // Act
+        const result = getPagination(undefined);
+
+        // Assert
+        expect(result).toEqual({ limit: 10, offset: 0 });
+    });
+
+    it('should use the supplied limit and offset', function () {
+        // Arrange
+        const event = { queryStringParameters: { limit: '25', offset: '50' } };
+
+        // Act
+        const result = getPagination(event);
+
+        // Assert
+        expect(result).toEqual({ limit: 25, offset: 50 });
+    });
+
+    it('should fall back to the default when limit is not a number', function () {
+        // Arrange - parseInt('abc') is NaN, and findAll turns a falsy limit into
+        // Sequelize `limit: 1`, so this used to return exactly one row.
+        const event = { queryStringParameters: { limit: 'abc' } };
+
+        // Act
+        const result = getPagination(event);
+
+        // Assert
+        expect(result.limit).toBe(10);
+    });
+
+    it('should fall back to the default when limit is empty or zero', function () {
+        // Act + Assert
+        expect(getPagination({ queryStringParameters: { limit: '' } }).limit).toBe(10);
+        expect(getPagination({ queryStringParameters: { limit: '0' } }).limit).toBe(10);
+    });
+
+    it('should clamp an oversized limit rather than running it', function () {
+        // Arrange
+        const event = { queryStringParameters: { limit: '100000' } };
+
+        // Act
+        const result = getPagination(event);
+
+        // Assert
+        expect(result.limit).toBe(500);
+    });
+
+    it('should never return a negative offset', function () {
+        // Arrange
+        const event = { queryStringParameters: { offset: '-20' } };
+
+        // Act
+        const result = getPagination(event);
+
+        // Assert
+        expect(result.offset).toBe(0);
+    });
+
+    it('should honor caller supplied defaultLimit and maxLimit', function () {
+        // Arrange
+        const event = { queryStringParameters: { limit: '999' } };
+
+        // Act
+        const clamped = getPagination(event, { maxLimit: 50 });
+        const defaulted = getPagination({}, { defaultLimit: 20 });
+
+        // Assert
+        expect(clamped.limit).toBe(50);
+        expect(defaulted.limit).toBe(20);
+    });
+});

package/src/eventManager/handleFile.js

@@ -1,115 +1,129 @@
-import _ from 'lodash';
-import S3Util from '../s3/S3Utils/S3Utils.js';
-
-export async function handleFile(path, s3Bucket, transformer, options) {
-    const shouldSkipFailedFolders = _.get(options, 'shouldSkipFailedFolders');
-    const userImportTypes = _.get(options, 'userImportTypes');
-    const params = {
-        Bucket: s3Bucket,
-        Key: path
-    };
-    let body;
-
-    try {
-        body = await S3Util.getObject(params);
-    } catch (e) {
-        if (e.Code === 'NoSuchKey' || e.code === 'NoSuchKey' || e.name === 'NoSuchKey') {
-            return;
-        }
-        console.error('S3 getObject error:', e);
-        throw e;
-    }
-
-    try {
-        // exit early for null/undefined or blank string bodies
-        if (_.isNil(body) || (_.isString(body) && _.isEmpty(body.trim()))) {
-            await S3Util.deleteObject(params);
-            return;
-        }
-
-        const parsedBody = JSON.parse(body);
-
-        // exit early for objects like {} or []
-        if (_.isEmpty(parsedBody)) {
-            await S3Util.deleteObject(params);
-            return;
-        }
-
-        const returnedValue = await transformer(parsedBody);
-        await S3Util.deleteObject(params);
-
-        return returnedValue;
-    } catch (error) {
-        console.error('HANDLE-FILE-ERROR', error);
-
-        if (!body) {
-            return;
-        }
-
-        let newPath = '';
-
-        if (userImportTypes) {
-            console.log('USER IMPORT');
-            // look for items in the userImportTypes
-            const usersImports = Object.values(userImportTypes);
-            const items = path.split('/');
-            const fileName = _.last(items);
-            const folderName = _.first(items);
-
-            if (usersImports.indexOf(folderName) > -1) {
-                if (shouldSkipFailedFolders) {
-                    newPath = `${folderName}/error/${fileName}`;
-                } else if (_.startsWith(path, `${folderName}/failed-once/`)) {
-                    newPath = _.replace(path, `${folderName}/failed-once/`, `${folderName}/failed-twice/`);
-                } else if (_.startsWith(path, `${folderName}/failed-twice/`)) {
-                    newPath = _.replace(path, `${folderName}/failed-twice/`, `${folderName}/error/`);
-                } else {
-                    newPath = `${folderName}/failed-once/${fileName}`;
-                }
-            } else {
-                newPath = `pathAndEventMisMatchError/${path}`;
-            }
-
-        } else {
-            if (shouldSkipFailedFolders) {
-                newPath = `error/${path}`;
-            } else if (_.startsWith(path, 'failed-once/')) {
-                newPath = _.replace(path, 'failed-once/', 'failed-twice/');
-            } else if (_.startsWith(path, 'failed-twice/')) {
-                newPath = _.replace(path, 'failed-twice/', 'error/');
-            } else {
-                if (_.includes(path, 'delayUntil/')) {
-                    path = _.last(path.split('/'));
-                }
-
-                newPath = `failed-once/${path}`;
-            }
-        }
-
-        const newParams = {
-            Bucket: s3Bucket,
-            Key: newPath,
-            Body: body
-        };
-
-        if (
-            containsError(newPath, 'failed-once') ||
-            containsError(newPath, 'failed-twice') ||
-            containsError(newPath, 'error')
-        ) {
-            throw new Error('NESTING ERROR');
-        }
-
-        await S3Util.putObject(newParams);
-        await S3Util.deleteObject(params);
-
-        throw error;
-    }
-}
-
-export function containsError(str, val) {
-    const firstIndex = str.toLowerCase().indexOf(val);
-    if (firstIndex === -1) return false;
-    const secondIndex = str.toLowerCase().indexOf(val, firstIndex + 1);
-    return secondIndex !== -1;
-}
+import _ from 'lodash';
+import S3Util from '../s3/S3Utils/S3Utils.js';
+
+export async function handleFile(path, s3Bucket, transformer, options) {
+    const shouldSkipFailedFolders = _.get(options, 'shouldSkipFailedFolders');
+    const userImportTypes = _.get(options, 'userImportTypes');
+    const params = {
+        Bucket: s3Bucket,
+        Key: path
+    };
+    let body;
+
+    try {
+        body = await S3Util.getObject(params);
+    } catch (e) {
+        if (e.Code === 'NoSuchKey' || e.code === 'NoSuchKey' || e.name === 'NoSuchKey') {
+            return;
+        }
+        console.error('S3 getObject error:', e);
+        throw e;
+    }
+
+    try {
+        // exit early for null/undefined or blank string bodies
+        if (_.isNil(body) || (_.isString(body) && _.isEmpty(body.trim()))) {
+            await S3Util.deleteObject(params);
+            return;
+        }
+
+        const parsedBody = JSON.parse(body);
+
+        // exit early for objects like {} or []
+        if (_.isEmpty(parsedBody)) {
+            await S3Util.deleteObject(params);
+            return;
+        }
+
+        const returnedValue = await transformer(parsedBody);
+        await S3Util.deleteObject(params);
+
+        return returnedValue;
+    } catch (error) {
+        console.error('HANDLE-FILE-ERROR', error);
+
+        if (!body) {
+            return;
+        }
+
+        let newPath = '';
+
+        if (userImportTypes) {
+            console.log('USER IMPORT');
+            // look for items in the userImportTypes
+            const usersImports = Object.values(userImportTypes);
+            const items = path.split('/');
+            const fileName = _.last(items);
+            const folderName = _.first(items);
+
+            if (usersImports.indexOf(folderName) > -1) {
+                if (shouldSkipFailedFolders) {
+                    newPath = `${folderName}/error/${fileName}`;
+                } else if (_.startsWith(path, `${folderName}/failed-once/`)) {
+                    newPath = _.replace(path, `${folderName}/failed-once/`, `${folderName}/failed-twice/`);
+                } else if (_.startsWith(path, `${folderName}/failed-twice/`)) {
+                    newPath = _.replace(path, `${folderName}/failed-twice/`, `${folderName}/error/`);
+                } else {
+                    newPath = `${folderName}/failed-once/${fileName}`;
+                }
+            } else {
+                newPath = `pathAndEventMisMatchError/${path}`;
+            }
+
+        } else {
+            if (shouldSkipFailedFolders) {
+                newPath = `error/${path}`;
+            } else if (_.startsWith(path, 'failed-once/')) {
+                newPath = _.replace(path, 'failed-once/', 'failed-twice/');
+            } else if (_.startsWith(path, 'failed-twice/')) {
+                newPath = _.replace(path, 'failed-twice/', 'error/');
+            } else {
+                if (_.includes(path, 'delayUntil/')) {
+                    path = _.last(path.split('/'));
+                }
+
+                newPath = `failed-once/${path}`;
+            }
+        }
+
+        const newParams = {
+            Bucket: s3Bucket,
+            Key: newPath,
+            Body: body
+        };
+
+        // Guard against genuine prefix nesting (e.g. `failed-once/error/<file>`) — a file that has
+        // already traversed the failure ladder. Detect this on the FOLDER SEGMENTS only, never the
+        // filename, so a user-uploaded file named e.g. `errors.csv` or `failed-once-list.csv` is
+        // not mistaken for a nested path. The previous check counted the token anywhere in the key
+        // (including the filename) and threw BEFORE the move/delete below, so the file was neither
+        // dead-lettered nor removed — publishEvents.loop() then re-listed it every tick, an
+        // infinite reprocess loop. If the destination really is nested, still DELETE the source so
+        // it dead-letters instead of looping forever. See audit finding #8.
+        if (isNestedFailurePath(newPath)) {
+            await S3Util.deleteObject(params);
+            throw new Error('NESTING ERROR');
+        }
+
+        await S3Util.putObject(newParams);
+        await S3Util.deleteObject(params);
+
+        throw error;
+    }
+}
+
+/**
+ * True when a key has already traversed the failure ladder more than once, i.e. its FOLDER
+ * portion contains more than one of the ladder stages (failed-once / failed-twice / error).
+ * Only whole path SEGMENTS are considered, never the filename, so a file literally named
+ * `errors.csv` is not treated as nested.
+ *
+ * @param {string} key - The destination S3 key
+ * @returns {boolean}
+ */
+export function isNestedFailurePath(key) {
+    const ladderStages = [ 'failed-once', 'failed-twice', 'error' ];
+    const folderSegments = String(key).split('/').slice(0, -1);
+    const ladderCount = folderSegments.filter((segment) => ladderStages.includes(segment)).length;
+    return ladderCount > 1;
+}