quidproquo-actionprocessor-awslambda 0.0.255 → 0.0.257
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/commonjs/getActionProcessor/core/config/getConfigListParametersActionProcessor.d.ts +2 -0
- package/lib/commonjs/getActionProcessor/core/config/getConfigListParametersActionProcessor.js +25 -0
- package/lib/commonjs/getActionProcessor/core/config/index.js +2 -1
- package/lib/commonjs/getActionProcessor/core/file/getFileGenerateTemporaryUploadSecureUrlActionProcessor.d.ts +2 -0
- package/lib/commonjs/getActionProcessor/core/file/getFileGenerateTemporaryUploadSecureUrlActionProcessor.js +35 -0
- package/lib/commonjs/getActionProcessor/core/file/index.js +2 -1
- package/lib/commonjs/getActionProcessor/webserver/extract/getExtractExpenseActionProcessor.d.ts +2 -0
- package/lib/commonjs/getActionProcessor/webserver/extract/getExtractExpenseActionProcessor.js +56 -0
- package/lib/commonjs/getActionProcessor/webserver/extract/index.d.ts +2 -0
- package/lib/commonjs/getActionProcessor/webserver/extract/index.js +17 -0
- package/lib/commonjs/getActionProcessor/webserver/index.d.ts +1 -0
- package/lib/commonjs/getActionProcessor/webserver/index.js +3 -1
- package/lib/commonjs/logic/dynamo/qpqDynamoOrm/buildDynamoUpdate.d.ts +1 -2
- package/lib/commonjs/logic/dynamo/qpqDynamoOrm/buildDynamoUpdate.js +43 -21
- package/lib/commonjs/logic/s3/generatePresignedUploadUrl.d.ts +1 -0
- package/lib/commonjs/logic/s3/generatePresignedUploadUrl.js +59 -0
- package/lib/commonjs/logic/textract/analyzeExpense.d.ts +51 -0
- package/lib/commonjs/logic/textract/analyzeExpense.js +31 -0
- package/lib/commonjs/logic/textract/index.d.ts +2 -0
- package/lib/commonjs/logic/textract/index.js +18 -0
- package/lib/commonjs/logic/textract/transformExpenseResponse.d.ts +29 -0
- package/lib/commonjs/logic/textract/transformExpenseResponse.js +180 -0
- package/lib/esm/getActionProcessor/core/config/getConfigListParametersActionProcessor.d.ts +2 -0
- package/lib/esm/getActionProcessor/core/config/getConfigListParametersActionProcessor.js +10 -0
- package/lib/esm/getActionProcessor/core/config/index.js +2 -0
- package/lib/esm/getActionProcessor/core/file/getFileGenerateTemporaryUploadSecureUrlActionProcessor.d.ts +2 -0
- package/lib/esm/getActionProcessor/core/file/getFileGenerateTemporaryUploadSecureUrlActionProcessor.js +20 -0
- package/lib/esm/getActionProcessor/core/file/index.js +2 -0
- package/lib/esm/getActionProcessor/webserver/extract/getExtractExpenseActionProcessor.d.ts +2 -0
- package/lib/esm/getActionProcessor/webserver/extract/getExtractExpenseActionProcessor.js +40 -0
- package/lib/esm/getActionProcessor/webserver/extract/index.d.ts +2 -0
- package/lib/esm/getActionProcessor/webserver/extract/index.js +4 -0
- package/lib/esm/getActionProcessor/webserver/index.d.ts +1 -0
- package/lib/esm/getActionProcessor/webserver/index.js +3 -0
- package/lib/esm/logic/dynamo/qpqDynamoOrm/buildDynamoUpdate.d.ts +1 -2
- package/lib/esm/logic/dynamo/qpqDynamoOrm/buildDynamoUpdate.js +42 -19
- package/lib/esm/logic/s3/generatePresignedUploadUrl.d.ts +1 -0
- package/lib/esm/logic/s3/generatePresignedUploadUrl.js +49 -0
- package/lib/esm/logic/textract/analyzeExpense.d.ts +51 -0
- package/lib/esm/logic/textract/analyzeExpense.js +18 -0
- package/lib/esm/logic/textract/index.d.ts +2 -0
- package/lib/esm/logic/textract/index.js +2 -0
- package/lib/esm/logic/textract/transformExpenseResponse.d.ts +29 -0
- package/lib/esm/logic/textract/transformExpenseResponse.js +173 -0
- package/package.json +8 -6
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.transformTextractExpenseResponse = transformTextractExpenseResponse;
|
|
4
|
+
const FIELD_TYPE_MAPPINGS = {
|
|
5
|
+
'VENDOR_NAME': 'merchantName',
|
|
6
|
+
'NAME': 'merchantName',
|
|
7
|
+
'MERCHANT_NAME': 'merchantName',
|
|
8
|
+
'VENDOR_ADDRESS': 'merchantAddress',
|
|
9
|
+
'ADDRESS': 'merchantAddress',
|
|
10
|
+
'MERCHANT_ADDRESS': 'merchantAddress',
|
|
11
|
+
'INVOICE_RECEIPT_DATE': 'date',
|
|
12
|
+
'DATE': 'date',
|
|
13
|
+
'CURRENCY_CODE': 'currency',
|
|
14
|
+
'PAYMENT_METHOD': 'paymentMethod',
|
|
15
|
+
'SUBTOTAL': 'subtotal',
|
|
16
|
+
'TAX': 'tax',
|
|
17
|
+
'TOTAL': 'total',
|
|
18
|
+
'AMOUNT_DUE': 'total',
|
|
19
|
+
'AMOUNT_PAID': 'amountPaid',
|
|
20
|
+
'INVOICE_RECEIPT_ID': 'receiptNumber',
|
|
21
|
+
'INVOICE_ID': 'invoiceNumber',
|
|
22
|
+
'RECEIPT_ID': 'receiptNumber',
|
|
23
|
+
'TAX_PAYER_ID': 'taxId',
|
|
24
|
+
'VENDOR_ABN_NUMBER': 'vendorAbn',
|
|
25
|
+
'VENDOR_PHONE': 'vendorPhone',
|
|
26
|
+
'VENDOR_URL': 'vendorUrl',
|
|
27
|
+
};
|
|
28
|
+
const LINE_ITEM_FIELD_MAPPINGS = {
|
|
29
|
+
'ITEM': 'description',
|
|
30
|
+
'DESCRIPTION': 'description',
|
|
31
|
+
'PRODUCT_NAME': 'description',
|
|
32
|
+
'QUANTITY': 'quantity',
|
|
33
|
+
'UNIT_PRICE': 'unitPrice',
|
|
34
|
+
'PRICE': 'total',
|
|
35
|
+
'AMOUNT': 'total',
|
|
36
|
+
};
|
|
37
|
+
function extractFieldValue(field, forceString = false) {
|
|
38
|
+
var _a, _b;
|
|
39
|
+
const text = ((_a = field === null || field === void 0 ? void 0 : field.ValueDetection) === null || _a === void 0 ? void 0 : _a.Text) || ((_b = field === null || field === void 0 ? void 0 : field.Type) === null || _b === void 0 ? void 0 : _b.Text);
|
|
40
|
+
if (!text)
|
|
41
|
+
return undefined;
|
|
42
|
+
// For certain fields, always return as string (dates, IDs, etc.)
|
|
43
|
+
if (forceString)
|
|
44
|
+
return text;
|
|
45
|
+
// Only try to parse as number if it looks like a price/amount (has $ or is purely numeric)
|
|
46
|
+
const cleanedText = text.replace(/[$,]/g, '').trim();
|
|
47
|
+
// Check if it's a pure number or price
|
|
48
|
+
if (/^\d+\.?\d*$/.test(cleanedText) && !text.includes('/') && !text.includes('-')) {
|
|
49
|
+
const numericValue = parseFloat(cleanedText);
|
|
50
|
+
if (!isNaN(numericValue))
|
|
51
|
+
return numericValue;
|
|
52
|
+
}
|
|
53
|
+
return text;
|
|
54
|
+
}
|
|
55
|
+
function extractSummaryFields(summaryFields) {
|
|
56
|
+
var _a, _b, _c;
|
|
57
|
+
const metadata = {};
|
|
58
|
+
if (!summaryFields)
|
|
59
|
+
return metadata;
|
|
60
|
+
// Fields that should always be strings
|
|
61
|
+
const stringFields = new Set([
|
|
62
|
+
'DATE', 'INVOICE_RECEIPT_DATE', 'CURRENCY_CODE', 'PAYMENT_METHOD',
|
|
63
|
+
'INVOICE_RECEIPT_ID', 'INVOICE_ID', 'RECEIPT_ID', 'TAX_PAYER_ID',
|
|
64
|
+
'VENDOR_ABN_NUMBER', 'VENDOR_PHONE', 'VENDOR_URL'
|
|
65
|
+
]);
|
|
66
|
+
// Track confidence scores for duplicate fields
|
|
67
|
+
const fieldConfidence = {};
|
|
68
|
+
for (const field of summaryFields) {
|
|
69
|
+
const fieldType = (_b = (_a = field === null || field === void 0 ? void 0 : field.Type) === null || _a === void 0 ? void 0 : _a.Text) === null || _b === void 0 ? void 0 : _b.toUpperCase();
|
|
70
|
+
if (!fieldType)
|
|
71
|
+
continue;
|
|
72
|
+
const confidence = ((_c = field === null || field === void 0 ? void 0 : field.Type) === null || _c === void 0 ? void 0 : _c.Confidence) || 0;
|
|
73
|
+
const forceString = stringFields.has(fieldType) || fieldType.includes('DATE') || fieldType.includes('ID') || fieldType.includes('NUMBER') || fieldType.includes('PHONE') || fieldType.includes('URL');
|
|
74
|
+
if (FIELD_TYPE_MAPPINGS[fieldType]) {
|
|
75
|
+
const mappedKey = FIELD_TYPE_MAPPINGS[fieldType];
|
|
76
|
+
const value = extractFieldValue(field, forceString);
|
|
77
|
+
// Only update if value exists and has higher confidence than existing
|
|
78
|
+
if (value !== undefined) {
|
|
79
|
+
const existingConfidence = fieldConfidence[mappedKey] || 0;
|
|
80
|
+
if (confidence > existingConfidence) {
|
|
81
|
+
metadata[mappedKey] = value;
|
|
82
|
+
fieldConfidence[mappedKey] = confidence;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
else {
|
|
87
|
+
// Skip fields that are clearly not useful
|
|
88
|
+
const fieldKey = fieldType.toLowerCase().replace(/_/g, '');
|
|
89
|
+
// Skip certain field types that aren't useful in our structure
|
|
90
|
+
if (['street', 'city', 'addressblock', 'expenserow', 'other'].includes(fieldKey)) {
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
// Store other unmapped fields for extensibility
|
|
94
|
+
const value = extractFieldValue(field, forceString);
|
|
95
|
+
if (value !== undefined && !fieldKey.startsWith('vendor_') && !fieldKey.startsWith('amount_')) {
|
|
96
|
+
// Use more readable field names
|
|
97
|
+
const cleanKey = fieldType.toLowerCase().replace(/_/g, '');
|
|
98
|
+
metadata[cleanKey] = value;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
// Clean up merchant name - take the first line if multiline
|
|
103
|
+
if (metadata.merchantName && metadata.merchantName.includes('\n')) {
|
|
104
|
+
metadata.merchantName = metadata.merchantName.split('\n')[0].trim();
|
|
105
|
+
}
|
|
106
|
+
return metadata;
|
|
107
|
+
}
|
|
108
|
+
function extractLineItems(lineItemGroups) {
|
|
109
|
+
var _a, _b;
|
|
110
|
+
if (!lineItemGroups || lineItemGroups.length === 0)
|
|
111
|
+
return undefined;
|
|
112
|
+
const items = [];
|
|
113
|
+
for (const group of lineItemGroups) {
|
|
114
|
+
if (!group.LineItems)
|
|
115
|
+
continue;
|
|
116
|
+
for (const lineItem of group.LineItems) {
|
|
117
|
+
if (!lineItem.LineItemExpenseFields)
|
|
118
|
+
continue;
|
|
119
|
+
const item = { description: '' };
|
|
120
|
+
for (const field of lineItem.LineItemExpenseFields) {
|
|
121
|
+
const fieldType = (_b = (_a = field === null || field === void 0 ? void 0 : field.Type) === null || _a === void 0 ? void 0 : _a.Text) === null || _b === void 0 ? void 0 : _b.toUpperCase();
|
|
122
|
+
if (!fieldType)
|
|
123
|
+
continue;
|
|
124
|
+
// Determine if this field should be a string
|
|
125
|
+
const forceString = fieldType === 'ITEM' || fieldType === 'DESCRIPTION' || fieldType === 'PRODUCT_NAME';
|
|
126
|
+
if (LINE_ITEM_FIELD_MAPPINGS[fieldType]) {
|
|
127
|
+
const mappedKey = LINE_ITEM_FIELD_MAPPINGS[fieldType];
|
|
128
|
+
const value = extractFieldValue(field, forceString);
|
|
129
|
+
if (value !== undefined) {
|
|
130
|
+
item[mappedKey] = value;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
else if (fieldType !== 'EXPENSE_ROW') {
|
|
134
|
+
// Store unmapped fields except expense_row which is just raw text
|
|
135
|
+
const value = extractFieldValue(field, forceString);
|
|
136
|
+
if (value !== undefined) {
|
|
137
|
+
const cleanKey = fieldType.toLowerCase().replace(/_/g, '');
|
|
138
|
+
item[cleanKey] = value;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
// Only add if we have at least a description or some meaningful data
|
|
143
|
+
if (item.description || (Object.keys(item).length > 1 && (item.total || item.quantity))) {
|
|
144
|
+
items.push(item);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
return items.length > 0 ? items : undefined;
|
|
149
|
+
}
|
|
150
|
+
function extractRawText(blocks) {
|
|
151
|
+
if (!blocks)
|
|
152
|
+
return undefined;
|
|
153
|
+
const textBlocks = blocks
|
|
154
|
+
.filter(block => block.BlockType === 'LINE' && block.Text)
|
|
155
|
+
.map(block => block.Text)
|
|
156
|
+
.join('\n');
|
|
157
|
+
return textBlocks || undefined;
|
|
158
|
+
}
|
|
159
|
+
function transformTextractExpenseResponse(textractResponse, storageDrive, filePath, includeRaw = true) {
|
|
160
|
+
var _a;
|
|
161
|
+
// Process the first expense document (most receipts/invoices have just one)
|
|
162
|
+
const expenseDoc = (_a = textractResponse.ExpenseDocuments) === null || _a === void 0 ? void 0 : _a[0];
|
|
163
|
+
const metadata = extractSummaryFields(expenseDoc === null || expenseDoc === void 0 ? void 0 : expenseDoc.SummaryFields);
|
|
164
|
+
const lineItems = extractLineItems(expenseDoc === null || expenseDoc === void 0 ? void 0 : expenseDoc.LineItemGroups);
|
|
165
|
+
const rawText = extractRawText(expenseDoc === null || expenseDoc === void 0 ? void 0 : expenseDoc.Blocks);
|
|
166
|
+
const result = {
|
|
167
|
+
metadata,
|
|
168
|
+
lineItems,
|
|
169
|
+
rawText,
|
|
170
|
+
source: {
|
|
171
|
+
storageDrive: storageDrive,
|
|
172
|
+
filePath: filePath,
|
|
173
|
+
},
|
|
174
|
+
};
|
|
175
|
+
// Include raw response for debugging (can be disabled in production)
|
|
176
|
+
if (includeRaw) {
|
|
177
|
+
result._raw = textractResponse;
|
|
178
|
+
}
|
|
179
|
+
return result;
|
|
180
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { actionResult, ConfigActionType, qpqCoreUtils, } from 'quidproquo-core';
|
|
2
|
+
const getProcessConfigListParameters = (qpqConfig) => {
|
|
3
|
+
const paramConfigs = qpqCoreUtils.getOwnedParameterConfigs(qpqConfig).map((pc) => pc.key);
|
|
4
|
+
return async () => {
|
|
5
|
+
return actionResult(paramConfigs);
|
|
6
|
+
};
|
|
7
|
+
};
|
|
8
|
+
export const getConfigListParametersActionProcessor = async (qpqConfig) => ({
|
|
9
|
+
[ConfigActionType.ListParameters]: getProcessConfigListParameters(qpqConfig),
|
|
10
|
+
});
|
|
@@ -2,11 +2,13 @@ import { getConfigGetGlobalActionProcessor } from './getConfigGetGlobalActionPro
|
|
|
2
2
|
import { getConfigGetParameterActionProcessor } from './getConfigGetParameterActionProcessor';
|
|
3
3
|
import { getConfigGetParametersActionProcessor } from './getConfigGetParametersActionProcessor';
|
|
4
4
|
import { getConfigGetSecretActionProcessor } from './getConfigGetSecretActionProcessor';
|
|
5
|
+
import { getConfigListParametersActionProcessor } from './getConfigListParametersActionProcessor';
|
|
5
6
|
import { getConfigSetParameterActionProcessor } from './getConfigSetParameterActionProcessor';
|
|
6
7
|
export const getConfigActionProcessor = async (qpqConfig, dynamicModuleLoader) => ({
|
|
7
8
|
...(await getConfigGetGlobalActionProcessor(qpqConfig, dynamicModuleLoader)),
|
|
8
9
|
...(await getConfigGetParameterActionProcessor(qpqConfig, dynamicModuleLoader)),
|
|
9
10
|
...(await getConfigGetParametersActionProcessor(qpqConfig, dynamicModuleLoader)),
|
|
10
11
|
...(await getConfigGetSecretActionProcessor(qpqConfig, dynamicModuleLoader)),
|
|
12
|
+
...(await getConfigListParametersActionProcessor(qpqConfig, dynamicModuleLoader)),
|
|
11
13
|
...(await getConfigSetParameterActionProcessor(qpqConfig, dynamicModuleLoader)),
|
|
12
14
|
});
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { qpqConfigAwsUtils } from 'quidproquo-config-aws';
|
|
2
|
+
import { actionResultError, ErrorTypeEnum } from 'quidproquo-core';
|
|
3
|
+
import { actionResult, FileActionType } from 'quidproquo-core';
|
|
4
|
+
import { generatePresignedUploadUrl } from '../../../logic/s3/generatePresignedUploadUrl';
|
|
5
|
+
import { resolveStorageDriveBucketName } from './utils';
|
|
6
|
+
const getProcessFileGenerateTemporaryUploadSecureUrl = (qpqConfig) => {
|
|
7
|
+
return async ({ drive, filepath, expirationMs, contentType }, session) => {
|
|
8
|
+
try {
|
|
9
|
+
const s3BucketName = resolveStorageDriveBucketName(drive, qpqConfig);
|
|
10
|
+
const url = await generatePresignedUploadUrl(s3BucketName, filepath, qpqConfigAwsUtils.getApplicationModuleDeployRegion(qpqConfig), expirationMs, session.correlation, contentType);
|
|
11
|
+
return actionResult(url);
|
|
12
|
+
}
|
|
13
|
+
catch (error) {
|
|
14
|
+
return actionResultError(ErrorTypeEnum.GenericError, 'Unable to generate temporary upload secure URL', error);
|
|
15
|
+
}
|
|
16
|
+
};
|
|
17
|
+
};
|
|
18
|
+
export const getFileGenerateTemporaryUploadSecureUrlActionProcessor = async (qpqConfig) => ({
|
|
19
|
+
[FileActionType.GenerateTemporaryUploadSecureUrl]: getProcessFileGenerateTemporaryUploadSecureUrl(qpqConfig),
|
|
20
|
+
});
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { getFileDeleteActionProcessor } from './getFileDeleteActionProcessor';
|
|
2
2
|
import { getFileExistsActionProcessor } from './getFileExistsActionProcessor';
|
|
3
3
|
import { getFileGenerateTemporarySecureUrlActionProcessor } from './getFileGenerateTemporarySecureUrlActionProcessor';
|
|
4
|
+
import { getFileGenerateTemporaryUploadSecureUrlActionProcessor } from './getFileGenerateTemporaryUploadSecureUrlActionProcessor';
|
|
4
5
|
import { getFileIsColdStorageActionProcessor } from './getFileIsColdStorageActionProcessor';
|
|
5
6
|
import { getFileListDirectoryActionProcessor } from './getFileListDirectoryActionProcessor';
|
|
6
7
|
import { getFileReadBinaryContentsActionProcessor } from './getFileReadBinaryContentsActionProcessor';
|
|
@@ -13,6 +14,7 @@ export const getFileActionProcessor = async (qpqConfig, dynamicModuleLoader) =>
|
|
|
13
14
|
...(await getFileDeleteActionProcessor(qpqConfig, dynamicModuleLoader)),
|
|
14
15
|
...(await getFileExistsActionProcessor(qpqConfig, dynamicModuleLoader)),
|
|
15
16
|
...(await getFileGenerateTemporarySecureUrlActionProcessor(qpqConfig, dynamicModuleLoader)),
|
|
17
|
+
...(await getFileGenerateTemporaryUploadSecureUrlActionProcessor(qpqConfig, dynamicModuleLoader)),
|
|
16
18
|
...(await getFileIsColdStorageActionProcessor(qpqConfig, dynamicModuleLoader)),
|
|
17
19
|
...(await getFileListDirectoryActionProcessor(qpqConfig, dynamicModuleLoader)),
|
|
18
20
|
...(await getFileReadTextContentsActionProcessor(qpqConfig, dynamicModuleLoader)),
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { qpqConfigAwsUtils } from 'quidproquo-config-aws';
|
|
2
|
+
import { actionResult, actionResultError, actionResultErrorFromCaughtError, qpqCoreUtils, } from 'quidproquo-core';
|
|
3
|
+
import { ExtractActionType, ExtractExpenseErrorTypeEnum } from 'quidproquo-webserver';
|
|
4
|
+
import { getConfigRuntimeResourceNameFromConfigWithServiceOverride } from '../../../awsNamingUtils';
|
|
5
|
+
import { analyzeExpenseDocument, transformTextractExpenseResponse } from '../../../logic/textract';
|
|
6
|
+
const resolveStorageDriveBucketName = (drive, qpqConfig) => {
|
|
7
|
+
const storageDriveConfig = qpqCoreUtils.getStorageDriveByName(drive, qpqConfig);
|
|
8
|
+
if (!storageDriveConfig) {
|
|
9
|
+
throw new Error(`Could not find storage drive config for [${drive}]`);
|
|
10
|
+
}
|
|
11
|
+
return getConfigRuntimeResourceNameFromConfigWithServiceOverride(storageDriveConfig.owner?.resourceNameOverride || drive, qpqConfig, storageDriveConfig.owner?.module);
|
|
12
|
+
};
|
|
13
|
+
const getProcessExtractExpense = (qpqConfig) => {
|
|
14
|
+
return async ({ storageDriveName, filePath }) => {
|
|
15
|
+
try {
|
|
16
|
+
const bucketName = resolveStorageDriveBucketName(storageDriveName, qpqConfig);
|
|
17
|
+
const region = qpqConfigAwsUtils.getApplicationModuleDeployRegion(qpqConfig);
|
|
18
|
+
// Call Textract to analyze the expense document
|
|
19
|
+
const textractResponse = await analyzeExpenseDocument(bucketName, filePath, region);
|
|
20
|
+
// Transform the response to our format
|
|
21
|
+
const extractedDocument = transformTextractExpenseResponse(textractResponse, storageDriveName, filePath, true);
|
|
22
|
+
return actionResult(extractedDocument);
|
|
23
|
+
}
|
|
24
|
+
catch (error) {
|
|
25
|
+
// Handle specific AWS errors with proper error mapping
|
|
26
|
+
return actionResultErrorFromCaughtError(error, {
|
|
27
|
+
InvalidS3ObjectException: () => actionResultError(ExtractExpenseErrorTypeEnum.FileNotFound, 'The specified file could not be found or accessed'),
|
|
28
|
+
NoSuchKey: () => actionResultError(ExtractExpenseErrorTypeEnum.FileNotFound, 'The specified file does not exist'),
|
|
29
|
+
UnsupportedDocumentException: () => actionResultError(ExtractExpenseErrorTypeEnum.UnsupportedFormat, 'The document format is not supported for expense analysis'),
|
|
30
|
+
InvalidParameterException: () => actionResultError(ExtractExpenseErrorTypeEnum.InvalidParameter, 'Invalid parameters provided to the extraction service'),
|
|
31
|
+
ThrottlingException: () => actionResultError(ExtractExpenseErrorTypeEnum.RateLimited, 'Too many requests, please try again later'),
|
|
32
|
+
InvalidObjectState: () => actionResultError(ExtractExpenseErrorTypeEnum.InvalidStorageClass, 'File is in the wrong storage class'),
|
|
33
|
+
AccessDenied: () => actionResultError(ExtractExpenseErrorTypeEnum.AccessDenied, 'Access denied to the specified file'),
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
};
|
|
37
|
+
};
|
|
38
|
+
export const getExtractExpenseActionProcessor = async (qpqConfig) => ({
|
|
39
|
+
[ExtractActionType.Expense]: getProcessExtractExpense(qpqConfig),
|
|
40
|
+
});
|
|
@@ -1,10 +1,13 @@
|
|
|
1
|
+
import { getExtractActionProcessor } from './extract';
|
|
1
2
|
import { getServiceFunctionActionProcessor } from './serviceFunction';
|
|
2
3
|
import { getWebEntryActionProcessor } from './webEntry';
|
|
3
4
|
import { getWebsocketActionProcessor } from './websocket';
|
|
5
|
+
export * from './extract';
|
|
4
6
|
export * from './serviceFunction';
|
|
5
7
|
export * from './webEntry';
|
|
6
8
|
export * from './websocket';
|
|
7
9
|
export const getWebserverActionProcessor = async (qpqConfig, dynamicModuleLoader) => ({
|
|
10
|
+
...(await getExtractActionProcessor(qpqConfig, dynamicModuleLoader)),
|
|
8
11
|
...(await getWebEntryActionProcessor(qpqConfig, dynamicModuleLoader)),
|
|
9
12
|
...(await getServiceFunctionActionProcessor(qpqConfig, dynamicModuleLoader)),
|
|
10
13
|
...(await getWebsocketActionProcessor(qpqConfig, dynamicModuleLoader)),
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { KvsUpdate
|
|
1
|
+
import { KvsUpdate } from 'quidproquo-core';
|
|
2
2
|
import { AttributeValue } from '@aws-sdk/client-dynamodb';
|
|
3
3
|
interface ExpressionAttributeNameMap {
|
|
4
4
|
[key: string]: string;
|
|
@@ -7,6 +7,5 @@ export declare const buildUpdateExpressionAttributeNames: (updates: KvsUpdate) =
|
|
|
7
7
|
export declare const buildUpdateExpressionAttributeValues: (updates: KvsUpdate) => {
|
|
8
8
|
[key: string]: AttributeValue;
|
|
9
9
|
} | undefined;
|
|
10
|
-
export declare const buildDynamoUpdateExpressionForType: (type: KvsUpdateActionType, kvsUpdate: KvsUpdate) => string;
|
|
11
10
|
export declare const buildDynamoUpdateExpression: (updates: KvsUpdate) => string;
|
|
12
11
|
export {};
|
|
@@ -31,6 +31,11 @@ export const buildUpdateExpressionAttributeValues = (updates) => {
|
|
|
31
31
|
const valuePlaceholder = getValueName(update.value);
|
|
32
32
|
attributeValues[valuePlaceholder] = buildAttributeValue(update.value);
|
|
33
33
|
}
|
|
34
|
+
// Include defaultValue for Increment actions
|
|
35
|
+
if (update.defaultValue !== undefined && update.defaultValue !== null) {
|
|
36
|
+
const defaultPlaceholder = getValueName(update.defaultValue);
|
|
37
|
+
attributeValues[defaultPlaceholder] = buildAttributeValue(update.defaultValue);
|
|
38
|
+
}
|
|
34
39
|
}
|
|
35
40
|
return Object.keys(attributeValues).length > 0 ? attributeValues : undefined;
|
|
36
41
|
};
|
|
@@ -57,6 +62,23 @@ const buildDynamoUpdateExpressionDelete = (update) => {
|
|
|
57
62
|
return `${getNestedItemName(update.attributePath)}`;
|
|
58
63
|
}
|
|
59
64
|
};
|
|
65
|
+
const buildDynamoUpdateExpressionSetIfNotExists = (update) => {
|
|
66
|
+
if (update.value === undefined || update.value === null) {
|
|
67
|
+
throw new Error("Value must be provided for 'SetIfNotExists' action");
|
|
68
|
+
}
|
|
69
|
+
const attrPath = getNestedItemName(update.attributePath);
|
|
70
|
+
return `${attrPath} = if_not_exists(${attrPath}, ${getValueName(update.value)})`;
|
|
71
|
+
};
|
|
72
|
+
const buildDynamoUpdateExpressionIncrement = (update) => {
|
|
73
|
+
if (update.value === undefined || update.value === null) {
|
|
74
|
+
throw new Error("Increment value must be provided for 'Increment' action");
|
|
75
|
+
}
|
|
76
|
+
if (update.defaultValue === undefined || update.defaultValue === null) {
|
|
77
|
+
throw new Error("Default value must be provided for 'Increment' action");
|
|
78
|
+
}
|
|
79
|
+
const attrPath = getNestedItemName(update.attributePath);
|
|
80
|
+
return `${attrPath} = if_not_exists(${attrPath}, ${getValueName(update.defaultValue)}) + ${getValueName(update.value)}`;
|
|
81
|
+
};
|
|
60
82
|
const getNestedItemName = (attributePath) => {
|
|
61
83
|
if (Array.isArray(attributePath)) {
|
|
62
84
|
let path = '';
|
|
@@ -84,35 +106,36 @@ const buildDynamoUpdateExpressionPart = (update, updateIndex) => {
|
|
|
84
106
|
return buildDynamoUpdateExpressionAdd(update);
|
|
85
107
|
case KvsUpdateActionType.Delete:
|
|
86
108
|
return buildDynamoUpdateExpressionDelete(update);
|
|
109
|
+
case KvsUpdateActionType.SetIfNotExists:
|
|
110
|
+
return buildDynamoUpdateExpressionSetIfNotExists(update);
|
|
111
|
+
case KvsUpdateActionType.Increment:
|
|
112
|
+
return buildDynamoUpdateExpressionIncrement(update);
|
|
87
113
|
default:
|
|
88
114
|
throw new Error(`Invalid update action type: ${update.action}`);
|
|
89
115
|
}
|
|
90
116
|
};
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
117
|
+
// Types that generate SET expressions
|
|
118
|
+
const SET_ACTION_TYPES = [
|
|
119
|
+
KvsUpdateActionType.Set,
|
|
120
|
+
KvsUpdateActionType.SetIfNotExists,
|
|
121
|
+
KvsUpdateActionType.Increment
|
|
122
|
+
];
|
|
123
|
+
const buildDynamoUpdateExpressionForClause = (clause, actionTypes, kvsUpdate) => {
|
|
124
|
+
const actions = kvsUpdate.filter((update) => actionTypes.includes(update.action));
|
|
94
125
|
if (actions.length === 0) {
|
|
95
126
|
return '';
|
|
96
127
|
}
|
|
97
128
|
const expressions = actions.map((update, index) => buildDynamoUpdateExpressionPart(update, index)).join(', ');
|
|
98
|
-
|
|
99
|
-
case KvsUpdateActionType.Set:
|
|
100
|
-
return `SET ${expressions}`;
|
|
101
|
-
case KvsUpdateActionType.Remove:
|
|
102
|
-
return `REMOVE ${expressions}`;
|
|
103
|
-
case KvsUpdateActionType.Add:
|
|
104
|
-
return `ADD ${expressions}`;
|
|
105
|
-
case KvsUpdateActionType.Delete:
|
|
106
|
-
return `DELETE ${expressions}`;
|
|
107
|
-
default:
|
|
108
|
-
throw new Error(`Invalid update action type: ${type}`);
|
|
109
|
-
}
|
|
129
|
+
return `${clause} ${expressions}`;
|
|
110
130
|
};
|
|
111
131
|
export const buildDynamoUpdateExpression = (updates) => {
|
|
112
|
-
const
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
132
|
+
const clauses = [
|
|
133
|
+
buildDynamoUpdateExpressionForClause('SET', SET_ACTION_TYPES, updates),
|
|
134
|
+
buildDynamoUpdateExpressionForClause('REMOVE', [KvsUpdateActionType.Remove], updates),
|
|
135
|
+
buildDynamoUpdateExpressionForClause('ADD', [KvsUpdateActionType.Add], updates),
|
|
136
|
+
buildDynamoUpdateExpressionForClause('DELETE', [KvsUpdateActionType.Delete], updates),
|
|
137
|
+
].filter((expression) => !!expression);
|
|
138
|
+
const result = clauses.join(' ');
|
|
116
139
|
console.log('Update Expression: ', result);
|
|
117
140
|
return result;
|
|
118
141
|
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const generatePresignedUploadUrl: (bucketName: string, objectKey: string, region: string, expirationMs: number, correlationId?: string, contentType?: string) => Promise<string>;
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
|
|
2
|
+
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
|
|
3
|
+
// import { createPresignedPost } from '@aws-sdk/s3-presigned-post';
|
|
4
|
+
import { createAwsClient } from '../createAwsClient';
|
|
5
|
+
// export const generatePresignedUploadFormUrl = async (
|
|
6
|
+
// bucketName: string,
|
|
7
|
+
// objectKey: string,
|
|
8
|
+
// region: string,
|
|
9
|
+
// expirationMs: number,
|
|
10
|
+
// contentType: string | undefined,
|
|
11
|
+
// maxSizeBytes?: number | undefined
|
|
12
|
+
// ): Promise<{ url: string; fields: Record<string, string> }> => {
|
|
13
|
+
// const s3Client = createAwsClient(S3Client, { region });
|
|
14
|
+
// const conditions: any[] = [];
|
|
15
|
+
// // Add content type condition if provided
|
|
16
|
+
// if (contentType) {
|
|
17
|
+
// conditions.push(['eq', '$Content-Type', contentType]);
|
|
18
|
+
// }
|
|
19
|
+
// // Add file size limit condition if provided
|
|
20
|
+
// if (maxSizeBytes) {
|
|
21
|
+
// conditions.push(['content-length-range', 0, maxSizeBytes]);
|
|
22
|
+
// }
|
|
23
|
+
// const presignedPost = await createPresignedPost(s3Client, {
|
|
24
|
+
// Bucket: bucketName,
|
|
25
|
+
// Key: objectKey,
|
|
26
|
+
// Conditions: conditions,
|
|
27
|
+
// Expires: Math.floor(expirationMs / 1000),
|
|
28
|
+
// });
|
|
29
|
+
// return {
|
|
30
|
+
// url: presignedPost.url,
|
|
31
|
+
// fields: presignedPost.fields
|
|
32
|
+
// };
|
|
33
|
+
// };
|
|
34
|
+
export const generatePresignedUploadUrl = async (bucketName, objectKey, region, expirationMs, correlationId, contentType) => {
|
|
35
|
+
const s3Client = createAwsClient(S3Client, { region });
|
|
36
|
+
const putObjectCommand = new PutObjectCommand({
|
|
37
|
+
Bucket: bucketName,
|
|
38
|
+
Key: objectKey,
|
|
39
|
+
ContentType: contentType,
|
|
40
|
+
Metadata: {
|
|
41
|
+
...(correlationId && { 'correlation-id': correlationId }),
|
|
42
|
+
'upload-timestamp': Date.now().toString()
|
|
43
|
+
}
|
|
44
|
+
});
|
|
45
|
+
const url = await getSignedUrl(s3Client, putObjectCommand, {
|
|
46
|
+
expiresIn: expirationMs / 1000,
|
|
47
|
+
});
|
|
48
|
+
return url;
|
|
49
|
+
};
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
export interface TextractExpenseAnalysis {
|
|
2
|
+
DocumentMetadata?: {
|
|
3
|
+
Pages?: number;
|
|
4
|
+
};
|
|
5
|
+
ExpenseDocuments?: Array<{
|
|
6
|
+
ExpenseIndex?: number;
|
|
7
|
+
SummaryFields?: Array<{
|
|
8
|
+
Type?: {
|
|
9
|
+
Text?: string;
|
|
10
|
+
Confidence?: number;
|
|
11
|
+
};
|
|
12
|
+
ValueDetection?: {
|
|
13
|
+
Text?: string;
|
|
14
|
+
Confidence?: number;
|
|
15
|
+
};
|
|
16
|
+
LabelDetection?: {
|
|
17
|
+
Text?: string;
|
|
18
|
+
Confidence?: number;
|
|
19
|
+
};
|
|
20
|
+
PageNumber?: number;
|
|
21
|
+
}>;
|
|
22
|
+
LineItemGroups?: Array<{
|
|
23
|
+
LineItemGroupIndex?: number;
|
|
24
|
+
LineItems?: Array<{
|
|
25
|
+
LineItemExpenseFields?: Array<{
|
|
26
|
+
Type?: {
|
|
27
|
+
Text?: string;
|
|
28
|
+
Confidence?: number;
|
|
29
|
+
};
|
|
30
|
+
ValueDetection?: {
|
|
31
|
+
Text?: string;
|
|
32
|
+
Confidence?: number;
|
|
33
|
+
};
|
|
34
|
+
LabelDetection?: {
|
|
35
|
+
Text?: string;
|
|
36
|
+
Confidence?: number;
|
|
37
|
+
};
|
|
38
|
+
PageNumber?: number;
|
|
39
|
+
}>;
|
|
40
|
+
}>;
|
|
41
|
+
}>;
|
|
42
|
+
Blocks?: Array<{
|
|
43
|
+
BlockType?: string;
|
|
44
|
+
Text?: string;
|
|
45
|
+
Confidence?: number;
|
|
46
|
+
Page?: number;
|
|
47
|
+
Id?: string;
|
|
48
|
+
}>;
|
|
49
|
+
}>;
|
|
50
|
+
}
|
|
51
|
+
export declare const analyzeExpenseDocument: (bucketName: string, documentKey: string, region: string) => Promise<TextractExpenseAnalysis>;
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { AnalyzeExpenseCommand, TextractClient } from '@aws-sdk/client-textract';
|
|
2
|
+
import { createAwsClient } from '../createAwsClient';
|
|
3
|
+
export const analyzeExpenseDocument = async (bucketName, documentKey, region) => {
|
|
4
|
+
const textractClient = createAwsClient(TextractClient, { region });
|
|
5
|
+
const command = new AnalyzeExpenseCommand({
|
|
6
|
+
Document: {
|
|
7
|
+
S3Object: {
|
|
8
|
+
Bucket: bucketName,
|
|
9
|
+
Name: documentKey,
|
|
10
|
+
},
|
|
11
|
+
},
|
|
12
|
+
});
|
|
13
|
+
const response = await textractClient.send(command);
|
|
14
|
+
return {
|
|
15
|
+
DocumentMetadata: response.DocumentMetadata,
|
|
16
|
+
ExpenseDocuments: response.ExpenseDocuments,
|
|
17
|
+
};
|
|
18
|
+
};
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { TextractExpenseAnalysis } from './analyzeExpense';
|
|
2
|
+
export interface ExtractedExpenseDocument {
|
|
3
|
+
metadata: {
|
|
4
|
+
merchantName?: string;
|
|
5
|
+
merchantAddress?: string;
|
|
6
|
+
date?: string;
|
|
7
|
+
currency?: string;
|
|
8
|
+
paymentMethod?: string;
|
|
9
|
+
subtotal?: number;
|
|
10
|
+
tax?: number;
|
|
11
|
+
total?: number;
|
|
12
|
+
[key: string]: any;
|
|
13
|
+
};
|
|
14
|
+
lineItems?: Array<{
|
|
15
|
+
description: string;
|
|
16
|
+
quantity?: number;
|
|
17
|
+
unitPrice?: number;
|
|
18
|
+
total?: number;
|
|
19
|
+
[key: string]: any;
|
|
20
|
+
}>;
|
|
21
|
+
rawText?: string;
|
|
22
|
+
source: {
|
|
23
|
+
storageDrive: string;
|
|
24
|
+
filePath: string;
|
|
25
|
+
textractJobId?: string;
|
|
26
|
+
};
|
|
27
|
+
_raw?: any;
|
|
28
|
+
}
|
|
29
|
+
export declare function transformTextractExpenseResponse(textractResponse: TextractExpenseAnalysis, storageDrive: string, filePath: string, includeRaw?: boolean): ExtractedExpenseDocument;
|