triangle-utils 1.0.34 → 1.0.36

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "triangle-utils",
-    "version": "1.0.34",
+    "version": "1.0.36",
     "main": "Utils.js",
     "author": "",
     "license": "ISC",
@@ -13,6 +13,7 @@
         "@aws-sdk/client-s3vectors": "^3.953.0",
         "@aws-sdk/s3-request-presigner": "^3.953.0",
         "nodemailer": "^7.0.11",
-        "scrapingbee": "^1.7.6"
+        "scrapingbee": "^1.7.6",
+        "triangle-utils": "^1.0.34"
     }
 }

package/test.js

@@ -1,8 +1,11 @@
 import Utils from "./Utils.js";
 
 
-const utils = new Utils({ region : "us-east-1"})
+const utils = new Utils({ region : "us-east-2"})
 
-const items = await utils.dynamodb.scan("article_hashes", { concurrency : 1 })
+const jurisdiction_documents = await utils.dynamodb.scan("jurisdiction_documents", {
+    defined_attribute_names : ["s3_filename"],
+    undefined_attribute_names : ["vector_keys"]
+})
 
-console.log(items.length)
+console.log(jurisdiction_documents.length)

package/utils/Utils_DynamoDB.js

@@ -64,15 +64,24 @@ export default class Utils_DynamoDB {
 
     async scan(table, options = {}) {
         const filters = options.filters !== undefined ? options.filters : {}
+        const undefined_attribute_names = options.undefined_attribute_names !== undefined ? options.undefined_attribute_names : []
+        const defined_attribute_names = options.defined_attribute_names !== undefined ? options.defined_attribute_names : []
         const concurrency = options.concurrency !== undefined ? options.concurrency : 1
         const attribute_names = options.attribute_names !== undefined ? options.attribute_names : []
         const iterators = []
         for (let i = 0; i < concurrency; i++) {
             const request = Object.fromEntries(Object.entries({
                 TableName : table,
-                ExpressionAttributeNames : Object.fromEntries((Object.keys(filters).concat(attribute_names)).map(attribute_name => ["#" + attribute_name, attribute_name])),
+                ExpressionAttributeNames : Object.fromEntries(
+                    [...Object.keys(filters), ...attribute_names, ...undefined_attribute_names, ...defined_attribute_names]
+                        .map(attribute_name => ["#" + attribute_name, attribute_name])
+                ),
                 ExpressionAttributeValues : Object.fromEntries(Object.entries(filters).map(([attribute_name, attribute_value]) => [":" + attribute_name, convert_input(attribute_value)])),
-                FilterExpression : Object.keys(filters).map(attribute_name => "#" + attribute_name + " = :" + attribute_name).join(","),
+                FilterExpression : [
+                    ...Object.keys(filters).map(attribute_name => "#" + attribute_name + " = :" + attribute_name),
+                    ...undefined_attribute_names.map(attribute_name => "attribute_not_exists(#" + attribute_name + ")"),
+                    ...defined_attribute_names.map(attribute_name => "attribute_exists(#" + attribute_name + ")")
+                ].join(" AND "),
                 ProjectionExpression : attribute_names.map(attribute_name => "#" + attribute_name).join(", "),
                 Segment : i,
                 TotalSegments : concurrency