triangle-utils 1.0.35 → 1.0.37

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "triangle-utils",
-    "version": "1.0.35",
+    "version": "1.0.37",
     "main": "Utils.js",
     "author": "",
     "license": "ISC",

package/test.js

@@ -1,9 +1,11 @@
 import Utils from "./Utils.js";
 
 
-const utils = new Utils({ region : "us-east-1"})
+const utils = new Utils({ region : "us-east-2"})
 
-const items = await utils.dynamodb.scan("candidates", { concurrency : 1, undefined_attribute_names : ["district"] })
-    // .then(items => items.filter(item => item.district === undefined))
+const jurisdiction_documents = await utils.dynamodb.scan("jurisdiction_documents", {
+    defined_attribute_names : ["s3_filename"],
+    undefined_attribute_names : ["vector_keys"]
+})
 
-console.log(items.length)
+console.log(jurisdiction_documents.length)

package/utils/Utils_DynamoDB.js

@@ -38,7 +38,7 @@ function convert_input(input) {
     return undefined
 }
 
-async function paginate(request, f) {
+async function compile_pages(request, f, compile = true) {
     const items = []
     let last_eval_key = undefined
     while (true) {
@@ -49,7 +49,7 @@ async function paginate(request, f) {
         )
         const new_items = response.Items.map(item => convert_output({ M : item }))
         items.push(...new_items)
-        if (response.LastEvaluatedKey === undefined) {
+        if (response.LastEvaluatedKey === undefined || !compile) {
             return items
         }
         last_eval_key = response.LastEvaluatedKey
@@ -81,12 +81,12 @@ export default class Utils_DynamoDB {
                     ...Object.keys(filters).map(attribute_name => "#" + attribute_name + " = :" + attribute_name),
                     ...undefined_attribute_names.map(attribute_name => "attribute_not_exists(#" + attribute_name + ")"),
                     ...defined_attribute_names.map(attribute_name => "attribute_exists(#" + attribute_name + ")")
-                ].join(","),
+                ].join(" AND "),
                 ProjectionExpression : attribute_names.map(attribute_name => "#" + attribute_name).join(", "),
                 Segment : i,
                 TotalSegments : concurrency
             }).filter(([field, value]) => value !== undefined && (typeof value === typeof 3 || Object.keys(value).length !== 0)))
-            iterators.push(paginate(request, (request) => this.dynamodb.scan(request)))
+            iterators.push(compile_pages(request, (request) => this.dynamodb.scan(request)))
         }
         const segments = await Promise.all(iterators)
         return segments.flat()
@@ -131,7 +131,9 @@ export default class Utils_DynamoDB {
 
 
 
-    async query(table, primary_key, reverse=false) {
+    async query(table, primary_key, options = {}) {
+        const reverse = options.reverse !== undefined ? options.reverse : false
+        const compile = options.reverse !== undefined ? options.compile : true
         if (Object.keys(primary_key).length !== 1) {
             return undefined
         }
@@ -146,7 +148,7 @@ export default class Utils_DynamoDB {
             KeyConditionExpression: "#a = :a",
             ScanIndexForward : !reverse
         }
-        return await paginate(request, (request) => this.dynamodb.query(request))
+        return await compile_pages(request, (request) => this.dynamodb.query(request), compile)
     }
 
     async query_prefix(table, primary_key, secondary_key_prefix, reverse=false) {
@@ -166,7 +168,7 @@ export default class Utils_DynamoDB {
             KeyConditionExpression: "#a = :a AND begins_with(#b, :b)",
             ScanIndexForward : !reverse
         }
-        return await paginate(request, (request) => this.dynamodb.query(request))
+        return await compile_pages(request, (request) => this.dynamodb.query(request))
     }
 
     async query_range(table, primary_key, secondary_key_range, reverse=false) {
@@ -187,7 +189,7 @@ export default class Utils_DynamoDB {
             KeyConditionExpression: "#a = :a AND (#b BETWEEN :b1 AND :b2)",
             ScanIndexForward : !reverse
         }
-        return await paginate(request, (request) => this.dynamodb.query(request))
+        return await compile_pages(request, (request) => this.dynamodb.query(request))
     }
 
     async set(table, key, attributes) {