piper-utils 1.0.0

package/src/database/dbUtils/queryStringUtils/mocks/mocks.js

@@ -0,0 +1,127 @@
+import DB from 'sequelize';
+
+export const sequilizeSchema = {
+    id: { type: DB.INTEGER, allowNull: true, field: 'id' },
+    char: { type: DB.CHAR, allowNull: true, field: 'char' },
+    decimal: { type: DB.DECIMAL, allowNull: true, field: 'decimal' },
+    boolean: { type: DB.BOOLEAN, allowNull: true, field: 'boolean' },
+    date: { type: DB.DATE, allowNull: true, field: 'authCode' },
+    dateOnly: { type: DB.DATEONLY, allowNull: true, field: 'authCode' },
+    string: { type: DB.STRING(20), allowNull: true, field: 'string' },
+    instantiatedDecimal: { type: DB.DECIMAL(16, 2), allowNull: true, field: 'instantiatedDecimal' },
+    transactionStatus: { type: DB.INTEGER, filterType: DB.Op.in, allowNull: true }
+};
+
+export const defaultFilters = {
+    id: { filterType: DB.Op.eq },
+    char: { filterType: DB.Op.iLike },
+    decimal: { filterType: DB.Op.eq },
+    boolean: { filterType: DB.Op.eq }
+};
+
+export const defaultFiltersWithSubSchema = {
+    'snowman.snowmanType': { filterType: DB.Op.eq },
+    'gearMonkey.type': { filterType: DB.Op.eq },
+    'FLURB.status': { filterType: DB.Op.eq },
+    id: { filterType: DB.Op.eq },
+    char: { filterType: DB.Op.iLike },
+    decimal: { filterType: DB.Op.or },
+    boolean: { type: DB.BOOLEAN, filterType: DB.Op.eq },
+    array: { filterType: DB.Op.in }
+};
+
+export const event = {
+    'pathParameters': {
+        'id': '1'
+    },
+    'body': '{}',
+    'requestContext': {
+        'authorizer': {
+            'claims': {
+                'custom:businessIds': '[\'1\',\'2\',\'3\',\'4\']'
+            }
+        }
+    },
+    'queryStringParameters': {
+        'snowman.snowmanType': 10,
+        'monkey.foo': 10,
+        'gearMonkey.type':10,
+        'limit': 22,
+        'offset': 14,
+        'endDate': '2017-01-01',
+        'startDate': '2001-01-01',
+        'businessIds': '1,2,3,4',
+        'id': '213235',
+        'char': 'StringToSearch',
+        'decimal': '4565',
+        'boolean': 'true',
+        'array': '10,20,30',
+        'date': '2001-9-9',
+        'notOnObject': 'no',
+        'sort': 'decimal,-id,boolean,notOnObject,-date,FLURB.id'
+    }
+};
+
+export const eventWithSingleElementInArray = {
+    'pathParameters': {
+        'id': '1'
+    },
+    'body': '{}',
+    'requestContext': {
+        'authorizer': {
+            'claims': {
+                'custom:businessIds': '[\'1\',\'2\',\'3\',\'4\']'
+            }
+        }
+    },
+    'queryStringParameters': {
+        'snowman.snowmanType': 10,
+        'monkey.foo': 10,
+        'gearMonkey.type':10,
+        'limit': 22,
+        'offset': 14,
+        'endDate': '2017-01-01',
+        'startDate': '2001-01-01',
+        'businessIds': '1,2,3,4',
+        'id': '213235',
+        'char': 'StringToSearch',
+        'decimal': '4565',
+        'boolean': 'true',
+        'array': '10',
+        'date': '2001-9-9',
+        'notOnObject': 'no',
+        'sort': 'decimal,-id,boolean,notOnObject,-date,FLURB.id'
+    }
+};
+
+export const eventWithSingleElementAsArray = {
+    'pathParameters': {
+        'id': '1'
+    },
+    'body': '{}',
+    'requestContext': {
+        'authorizer': {
+            'claims': {
+                'custom:businessIds': '[\'1\',\'2\',\'3\',\'4\']'
+            }
+        }
+    },
+    'queryStringParameters': {
+        'snowman.snowmanType': 10,
+        'monkey.foo': 10,
+        'gearMonkey.type':10,
+        'limit': 22,
+        'offset': 14,
+        'endDate': '2017-01-01',
+        'startDate': '2001-01-01',
+        'businessIds': '1,2,3,4',
+        'id': '213235',
+        'char': 'StringToSearch',
+        'decimal': '4565',
+        'boolean': 'true',
+        'array': '10',
+        'date': '2001-9-9',
+        'notOnObject': 'no',
+        'sort': 'decimal,-id,boolean,notOnObject,-date,FLURB.id'
+    }
+};

package/src/dynamo/dynamoUtils.js

@@ -0,0 +1,16 @@
+import { DynamoDBDocument } from '@aws-sdk/lib-dynamodb';
+import { DynamoDB } from '@aws-sdk/client-dynamodb';
+
+const dynamoUtil = {
+    get: (params) => get(params)
+};
+
+/**
+ * @param {{TableName:string, Key:{key:string}}} params
+ */
+async function get(params) {
+    const dynamoDb = DynamoDBDocument.from(new DynamoDB());
+    return dynamoDb.get(params);
+}
+
+export default dynamoUtil;

package/src/eventManager/handleEvents.js

@@ -0,0 +1,32 @@
+import { handleFile } from './handleFile.js';
+import _ from 'lodash';
+import Promise from 'bluebird';
+
+export function handleEvents(event, transformer, errorHandlerPerFile, shouldSkipFailedFolders = false, userImportTypes) {
+    if (!event || !event.Records) {
+        return Promise.resolve();
+    }
+
+    let hasErrors = false;
+
+    return Promise.map(event.Records, (record) => {
+        const snsInfo = JSON.parse(_.get(record, 'Sns.Message', '{}'));
+        const s3Bucket = snsInfo.bucket;
+        const files = snsInfo.files || [];
+
+        return Promise.map(files, (file) => {
+            let path = decodeURIComponent(file);
+            if (path) {
+                return handleFile(path, s3Bucket, transformer, { shouldSkipFailedFolders, userImportTypes }).catch((err) => {
+                    if (errorHandlerPerFile && !errorHandlerPerFile(err, path)) {
+                        hasErrors = true;
+                    }
+                });
+            }
+        });
+    }).then(() => {
+        if (hasErrors) {
+            throw 'Errors processing one or more files. See error logs for detail on each error.';
+        }
+    });
+}

package/src/eventManager/handleEvents.test.js

@@ -0,0 +1,92 @@
+import * as handleEvents from './handleEvents.js';
+import * as handleFile from './handleFile.js';
+
+describe('handleEvents.js', () => {
+    describe('handleEvents', () => {
+        it('should run functions correctly for one file', function () {
+            const handleFileSpy = spyOn(handleFile, 'handleFile').and.returnValue(Promise.resolve());
+
+            return handleEvents.handleEvents({
+                Records: [{
+                    'EventVersion': '1.0',
+                    'EventSubscriptionArn': 'arn:aws:sns:EXAMPLE',
+                    'EventSource': 'aws:sns',
+                    'Sns': {
+                        'SignatureVersion': '1',
+                        'Timestamp': '1970-01-01T00:00:00.000Z',
+                        'Signature': 'EXAMPLE',
+                        'SigningCertUrl': 'EXAMPLE',
+                        'MessageId': '95df01b4-ee98-5cb9-9903-4c221d41eb5e',
+                        'Message': '{"bucket": "etl-input-transactions-development","files": ["failed-twice/A-emvio-plugin-transaction-6bbd68be-af30-49da-8e75-20d8d569ad22-2019-07-08T15_29_48.099Z.json"]}',
+                        'MessageAttributes': {
+                            'Test': {
+                                'Type': 'String',
+                                'Value': 'TestString'
+                            },
+                            'TestBinary': {
+                                'Type': 'Binary',
+                                'Value': 'TestBinary'
+                            }
+                        },
+                        'Type': 'Notification',
+                        'UnsubscribeUrl': 'EXAMPLE',
+                        'TopicArn': 'arn:aws:sns:EXAMPLE',
+                        'Subject': 'TestInvoke'
+                    }
+                }]
+            }, {}, () => {
+            }).then(() => {
+                expect(handleFileSpy).toHaveBeenCalled();
+            });
+        });
+
+        it('should run errorHandlerPerFile if file errors', function () {
+            const handleFileSpy = spyOn(handleFile, 'handleFile').and.rejectWith();
+            const spy = jasmine.createSpy('errorSpy');
+
+            return handleEvents.handleEvents({
+                Records: [{
+                    'EventVersion': '1.0',
+                    'EventSubscriptionArn': 'arn:aws:sns:EXAMPLE',
+                    'EventSource': 'aws:sns',
+                    'Sns': {
+                        'SignatureVersion': '1',
+                        'Timestamp': '1970-01-01T00:00:00.000Z',
+                        'Signature': 'EXAMPLE',
+                        'SigningCertUrl': 'EXAMPLE',
+                        'MessageId': '95df01b4-ee98-5cb9-9903-4c221d41eb5e',
+                        'Message': '{"bucket": "etl-input-transactions-development","files": ["failed-twice/A-emvio-plugin-transaction-6bbd68be-af30-49da-8e75-20d8d569ad22-2019-07-08T15_29_48.099Z.json"]}',
+                        'MessageAttributes': {
+                            'Test': {
+                                'Type': 'String',
+                                'Value': 'TestString'
+                            },
+                            'TestBinary': {
+                                'Type': 'Binary',
+                                'Value': 'TestBinary'
+                            }
+                        },
+                        'Type': 'Notification',
+                        'UnsubscribeUrl': 'EXAMPLE',
+                        'TopicArn': 'arn:aws:sns:EXAMPLE',
+                        'Subject': 'TestInvoke'
+                    }
+                }]
+            }, () => {}, spy).then(() => {
+                fail();
+            }).catch(() => {
+                expect(handleFileSpy).toHaveBeenCalled();
+                expect(spy).toHaveBeenCalled();
+            });
+        });
+
+        it('should return immediately if there is no Record on the event', () => {
+            try {
+                handleEvents.handleEvents({}, {}, () => {
+                });
+            } catch (error) {
+                expect(error).toBeUndefined();
+            }
+        });
+    });
+});

package/src/eventManager/handleFile.js

@@ -0,0 +1,76 @@
+import _ from 'lodash';
+import S3Util from '../s3/S3Utils.js/S3Utils.js';
+
+export async function handleFile(path, s3Bucket, transformer, options) {
+    const shouldSkipFailedFolders = _.get(options, 'shouldSkipFailedFolders');
+    const userImportTypes = _.get(options, 'userImportTypes');
+    const params = {
+        Bucket: s3Bucket,
+        Key: path
+    };
+    let body = '{}'
+    try {
+        body = await S3Util.getObject(params);
+        const parsedBody = JSON.parse(body);
+        const returnedValue = await transformer(parsedBody);
+        await S3Util.deleteObject(params);
+
+        return returnedValue;
+    } catch (error) {
+        console.error('handle-file-error', error);
+
+        if (error.code === 'NoSuchKey' || !body) {
+            return;
+        }
+
+        let newPath = '';
+
+        if (userImportTypes) {
+            // look for items in the userImportTypes
+            const usersImports = Object.values(userImportTypes);
+            const items = path.split('/');
+            const fileName = _.last(items);
+            const folderName = _.first(items);
+
+            if (usersImports.indexOf(folderName) > -1) {
+                if (shouldSkipFailedFolders) {
+                    newPath = `${folderName}/error/${fileName}`;
+                } else if (_.startsWith(path, `${folderName}/failed-once/`)) {
+                    newPath = _.replace(path, `${folderName}/failed-once/`, `${folderName}/failed-twice/`);
+                } else if (_.startsWith(path, `${folderName}/failed-twice/`)) {
+                    newPath = _.replace(path, `${folderName}/failed-twice/`, `${folderName}/error/`);
+                } else {
+                    newPath = `${folderName}/failed-once/${fileName}`;
+                }
+            } else {
+                newPath = `pathAndEventMisMatchError/${path}`;
+            }
+
+        } else {
+            if (shouldSkipFailedFolders) {
+                newPath = `error/${path}`;
+            } else if (_.startsWith(path, 'failed-once/')) {
+                newPath = _.replace(path, 'failed-once/', 'failed-twice/');
+            } else if (_.startsWith(path, 'failed-twice/')) {
+                newPath = _.replace(path, 'failed-twice/', 'error/');
+            } else {
+                if (_.includes(path, 'delayUntil/')) {
+                    path = _.last(path.split('/'));
+                }
+
+                newPath = `failed-once/${path}`;
+            }
+        }
+
+        const newParams = {
+            Bucket: s3Bucket,
+            Key: newPath,
+            Body: body
+        };
+
+        await S3Util.putObject(newParams);
+        await S3Util.deleteObject(params);
+
+        throw error;
+    }
+}