triangle-utils 1.0.33 → 1.0.35

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "triangle-utils",
-    "version": "1.0.33",
+    "version": "1.0.35",
     "main": "Utils.js",
     "author": "",
     "license": "ISC",
@@ -13,6 +13,7 @@
         "@aws-sdk/client-s3vectors": "^3.953.0",
         "@aws-sdk/s3-request-presigner": "^3.953.0",
         "nodemailer": "^7.0.11",
-        "scrapingbee": "^1.7.6"
+        "scrapingbee": "^1.7.6",
+        "triangle-utils": "^1.0.34"
     }
 }

package/test.js

@@ -1,8 +1,9 @@
 import Utils from "./Utils.js";
 
 
-const utils = new Utils({ region : "us-east-2"})
+const utils = new Utils({ region : "us-east-1"})
 
-const items = await utils.s3vectors.batch_get("doersnetwork", "candidate-document-chunks", ["yeeerp", "text|2025-12-10T21:34:37.429Z|0|1761", "text|2025-12-10T21:37:00.719Z|0|2413"])
+const items = await utils.dynamodb.scan("candidates", { concurrency : 1, undefined_attribute_names : ["district"] })
+    // .then(items => items.filter(item => item.district === undefined))
 
-console.log(items)
+console.log(items.length)

package/utils/Utils_DynamoDB.js

@@ -62,15 +62,34 @@ export default class Utils_DynamoDB {
         this.dynamodb = new DynamoDB({ region : config.region })
     }
 
-    async scan(table, attribute_names = [], filters = {}) {
-        const request = Object.fromEntries(Object.entries({
-            TableName : table,
-            ExpressionAttributeNames : Object.fromEntries((Object.keys(filters).concat(attribute_names)).map(attribute_name => ["#" + attribute_name, attribute_name])),
-            ExpressionAttributeValues : Object.fromEntries(Object.entries(filters).map(([attribute_name, attribute_value]) => [":" + attribute_name, convert_input(attribute_value)])),
-            FilterExpression : Object.keys(filters).map(attribute_name => "#" + attribute_name + " = :" + attribute_name).join(","),
-            ProjectionExpression : attribute_names.map(attribute_name => "#" + attribute_name).join(", ")
-        }).filter(([field, value]) => value !== undefined && Object.keys(value).length !== 0))
-        return await paginate(request, (request) => this.dynamodb.scan(request))
+    async scan(table, options = {}) {
+        const filters = options.filters !== undefined ? options.filters : {}
+        const undefined_attribute_names = options.undefined_attribute_names !== undefined ? options.undefined_attribute_names : []
+        const defined_attribute_names = options.defined_attribute_names !== undefined ? options.defined_attribute_names : []
+        const concurrency = options.concurrency !== undefined ? options.concurrency : 1
+        const attribute_names = options.attribute_names !== undefined ? options.attribute_names : []
+        const iterators = []
+        for (let i = 0; i < concurrency; i++) {
+            const request = Object.fromEntries(Object.entries({
+                TableName : table,
+                ExpressionAttributeNames : Object.fromEntries(
+                    [...Object.keys(filters), ...attribute_names, ...undefined_attribute_names, ...defined_attribute_names]
+                        .map(attribute_name => ["#" + attribute_name, attribute_name])
+                ),
+                ExpressionAttributeValues : Object.fromEntries(Object.entries(filters).map(([attribute_name, attribute_value]) => [":" + attribute_name, convert_input(attribute_value)])),
+                FilterExpression : [
+                    ...Object.keys(filters).map(attribute_name => "#" + attribute_name + " = :" + attribute_name),
+                    ...undefined_attribute_names.map(attribute_name => "attribute_not_exists(#" + attribute_name + ")"),
+                    ...defined_attribute_names.map(attribute_name => "attribute_exists(#" + attribute_name + ")")
+                ].join(","),
+                ProjectionExpression : attribute_names.map(attribute_name => "#" + attribute_name).join(", "),
+                Segment : i,
+                TotalSegments : concurrency
+            }).filter(([field, value]) => value !== undefined && (typeof value === typeof 3 || Object.keys(value).length !== 0)))
+            iterators.push(paginate(request, (request) => this.dynamodb.scan(request)))
+        }
+        const segments = await Promise.all(iterators)
+        return segments.flat()
     }
 
     async get(table, key, consistent=false) {

package/utils/Utils_S3.js

@@ -17,7 +17,6 @@ export default class Utils_S3 {
         this.s3 = new S3({ region : config.region })
     }
 
-
     async file_exists(s3_filename) {
         const s3_key = parse_s3_filename(s3_filename)
         if (s3_key === undefined) {
@@ -31,7 +30,6 @@ export default class Utils_S3 {
         }
     }
 
-
     async get_file(s3_filename, json=false) {
         const s3_key = parse_s3_filename(s3_filename)
         if (s3_key === undefined) {