triangle-utils 1.4.106 → 1.4.108

package/dist/src/UtilsDynamoDB.d.ts

@@ -58,7 +58,9 @@ export declare class UtilsDynamoDB {
     remove(table_name: string, key: Record<string, any>, attributes: string[]): Promise<void>;
     create(table_name: string, key: Record<string, any>, attributes?: Record<string, any>): Promise<void>;
     delete(table_name: string, key: Record<string, any>): Promise<void>;
-    batch_put(table_name: string, items: Record<string, any>[]): Promise<void>;
+    batch_put(table_name: string, items: Record<string, any>[], options?: {
+        is_verbose?: boolean;
+    }): Promise<void>;
     batch_delete(table_name: string, keys: Record<string, any>[]): Promise<void>;
     duplicate_attribute(table_name: string, attribute_name: string, new_attribute_name: string): Promise<void>;
     remove_attribute(table_name: string, attribute_name: string): Promise<undefined>;

package/dist/src/UtilsDynamoDB.js

@@ -1,4 +1,6 @@
+import { ThrottlingException } from "@aws-sdk/client-bedrock-runtime";
 import { DynamoDB } from "@aws-sdk/client-dynamodb";
+import { UtilsMisc } from "./UtilsMisc.js";
 function convert_output(dynamodb_output) {
     if (dynamodb_output.S !== undefined) {
         return dynamodb_output.S;
@@ -498,7 +500,8 @@ export class UtilsDynamoDB {
             Key: converted_key
         });
     }
-    async batch_put(table_name, items) {
+    async batch_put(table_name, items, options = {}) {
+        const is_verbose = options.is_verbose !== undefined ? options.is_verbose : false;
         const dynamodb_items = items.map(item => convert_input(item)?.M)
             .filter(dynamodb_item => dynamodb_item !== undefined);
         for (let i = 0; i < dynamodb_items.length; i += 25) {
@@ -508,12 +511,24 @@ export class UtilsDynamoDB {
                 }
             }));
             while (next_dynamodb_requests.length > 0) {
-                const response = await this.dynamodb.batchWriteItem({
-                    RequestItems: {
-                        [table_name]: next_dynamodb_requests
+                try {
+                    const response = await this.dynamodb.batchWriteItem({
+                        RequestItems: {
+                            [table_name]: next_dynamodb_requests
+                        }
+                    });
+                    next_dynamodb_requests = response.UnprocessedItems?.[table_name] || [];
+                }
+                catch (error) {
+                    if (error instanceof ThrottlingException) {
+                        if (is_verbose) {
+                            console.log("Throttled.");
+                        }
+                        await UtilsMisc.wait(1000);
+                        continue;
                     }
-                });
-                next_dynamodb_requests = response.UnprocessedItems?.[table_name] || [];
+                    throw error;
+                }
             }
         }
     }

package/dist/src/UtilsMisc.d.ts

@@ -7,7 +7,15 @@ export declare class UtilsMisc {
     readonly config: GlobalConfig;
     constructor(config: GlobalConfig);
     safe_run<T>(f: () => Promise<T>): Promise<T | undefined>;
-    iterate<T, U>(inputs: T[], f: (input: T, iterator_id: number, index: number) => Promise<U>, num_iterators?: number, print_option?: number | boolean): Promise<(U | undefined)[]>;
+    iterate<T, U>(inputs: T[], f: (input: T, iterator_id: number, index: number) => Promise<U>, num_iterators?: number, options?: {
+        num_iterators?: number;
+        verbosity?: number | boolean;
+    }): Promise<(U | undefined)[]>;
+    batch_iterate<T, U>(inputs: T[], f: (inputs: T[], iterator_id: number, index: number) => Promise<U>, options: {
+        batch_size?: number;
+        num_iterators?: number;
+        verbosity?: number | boolean;
+    }): Promise<(U | undefined)[]>;
     static wait(duration: number): Promise<unknown>;
     static get_current_time(): string;
     static get_current_milliseconds(): number;

package/dist/src/UtilsMisc.js

@@ -21,8 +21,8 @@ export class UtilsMisc {
         }
         return undefined;
     }
-    async iterate(inputs, f, num_iterators = 1, print_option) {
-        const print_freq = typeof print_option === "boolean" ? (print_option === true ? 1 : 0) : print_option;
+    async iterate(inputs, f, num_iterators = 1, options = {}) {
+        const verbosity = typeof options.verbosity === "boolean" ? (options.verbosity === true ? 1 : 0) : (options.verbosity || 0);
         let index = 0;
         let iterators = [];
         const outputs = new Array(inputs.length).fill(undefined);
@@ -31,7 +31,7 @@ export class UtilsMisc {
                 while (index < inputs.length) {
                     const local_index = index;
                     index += 1;
-                    if (print_freq !== undefined && (local_index % print_freq === 0)) {
+                    if (verbosity !== undefined && (local_index % verbosity === 0)) {
                         console.log(iterator_id + ":" + local_index + "/" + inputs.length);
                     }
                     const output = await this.safe_run(() => f(inputs[local_index], iterator_id, local_index));
@@ -42,6 +42,29 @@ export class UtilsMisc {
         await Promise.all(iterators);
         return outputs;
     }
+    async batch_iterate(inputs, f, options) {
+        const batch_size = options.batch_size || 1;
+        const num_iterators = options.num_iterators || 0;
+        const verbosity = typeof options.verbosity === "boolean" ? (options.verbosity === true ? 1 : 0) : (options.verbosity || 0);
+        let index = 0;
+        let iterators = [];
+        const outputs = new Array(inputs.length).fill(undefined);
+        for (let iterator_id = 0; iterator_id < num_iterators; iterator_id++) {
+            iterators.push((async () => {
+                while (index < inputs.length) {
+                    const local_index = index;
+                    index += batch_size;
+                    if (verbosity !== undefined && (local_index % verbosity === 0)) {
+                        console.log(iterator_id + ":" + local_index + "/" + inputs.length);
+                    }
+                    const output = await this.safe_run(() => f(inputs.slice(local_index, local_index + batch_size), iterator_id, local_index));
+                    outputs[local_index] = output;
+                }
+            })());
+        }
+        await Promise.all(iterators);
+        return outputs;
+    }
     static async wait(duration) {
         return new Promise(resolve => setTimeout(resolve, duration));
     }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "triangle-utils",
-    "version": "1.4.106",
+    "version": "1.4.108",
     "main": "dist/src/index.js",
     "types": "dist/src/index.d.ts",
     "directories": {

package/src/UtilsDynamoDB.ts

@@ -1,4 +1,6 @@
+import { ThrottlingException } from "@aws-sdk/client-bedrock-runtime"
 import { AttributeValue, DynamoDB, QueryCommandInput, QueryCommandOutput, ScanCommandInput, ScanCommandOutput, UpdateItemCommandInput, WriteRequest } from "@aws-sdk/client-dynamodb"
+import { UtilsMisc } from "./UtilsMisc"
 
 function convert_output(dynamodb_output : AttributeValue) : string | boolean | number | any[] | Set<any> | Record<string, any> | undefined {
     if (dynamodb_output.S !== undefined) {
@@ -615,8 +617,12 @@ export class UtilsDynamoDB {
 
     async batch_put(
         table_name : string,
-        items : Record<string, any>[]
+        items : Record<string, any>[],
+        options : {
+            is_verbose? : boolean
+        } = {}
     ) : Promise<void> {
+        const is_verbose = options.is_verbose !== undefined ? options.is_verbose : false
         const dynamodb_items = items.map(item => convert_input(item)?.M)
             .filter(dynamodb_item => dynamodb_item !== undefined)
             
@@ -627,12 +633,23 @@ export class UtilsDynamoDB {
                 }
             }))
             while (next_dynamodb_requests.length > 0) {
-                const response = await this.dynamodb.batchWriteItem({
-                    RequestItems : {
-                        [table_name] : next_dynamodb_requests
+                try {
+                    const response = await this.dynamodb.batchWriteItem({
+                        RequestItems : {
+                            [table_name] : next_dynamodb_requests
+                        }
+                    })
+                    next_dynamodb_requests = response.UnprocessedItems?.[table_name] || []
+                } catch (error : any) {
+                    if (error instanceof ThrottlingException) {
+                        if (is_verbose) {
+                            console.log("Throttled.")
+                        }
+                        await UtilsMisc.wait(1000)
+                        continue
                     }
-                })
-                next_dynamodb_requests = response.UnprocessedItems?.[table_name] || []
+                    throw error
+                }
             }
         }
     }

package/src/UtilsMisc.ts

@@ -31,8 +31,8 @@ export class UtilsMisc {
         return undefined
     }
 
-    async iterate<T, U>(inputs : T[], f : (input : T, iterator_id : number, index : number) => Promise<U>, num_iterators : number = 1, print_option? : number | boolean) {
-        const print_freq : number | undefined = typeof print_option === "boolean" ? (print_option === true ? 1 : 0) : print_option
+    async iterate<T, U>(inputs : T[], f : (input : T, iterator_id : number, index : number) => Promise<U>, num_iterators : number = 1, options : { num_iterators? : number, verbosity? : number | boolean } = {}) {
+        const verbosity : number = typeof options.verbosity === "boolean" ? (options.verbosity === true ? 1 : 0) : (options.verbosity || 0)
         let index = 0
         let iterators = []
         const outputs : (U | undefined)[] = new Array(inputs.length).fill(undefined)
@@ -41,7 +41,7 @@ export class UtilsMisc {
                 while (index < inputs.length) {
                     const local_index = index
                     index += 1
-                    if (print_freq !== undefined && (local_index % print_freq === 0)) {
+                    if (verbosity !== undefined && (local_index % verbosity === 0)) {
                         console.log(iterator_id + ":" + local_index + "/" + inputs.length)
                     }
                     const output = await this.safe_run(() => f(inputs[local_index], iterator_id, local_index))
@@ -53,6 +53,31 @@ export class UtilsMisc {
         return outputs
     }
 
+    async batch_iterate<T, U>(inputs : T[], f : (inputs : T[], iterator_id : number, index : number) => Promise<U>, 
+    options : { batch_size? : number, num_iterators? : number, verbosity? : number | boolean }) {
+        const batch_size = options.batch_size || 1
+        const num_iterators = options.num_iterators || 0
+        const verbosity : number = typeof options.verbosity === "boolean" ? (options.verbosity === true ? 1 : 0) : (options.verbosity || 0)
+        let index = 0
+        let iterators = []
+        const outputs : (U | undefined)[] = new Array(inputs.length).fill(undefined)
+        for (let iterator_id = 0; iterator_id < num_iterators; iterator_id++) {
+            iterators.push((async () => {
+                while (index < inputs.length) {
+                    const local_index = index
+                    index += batch_size
+                    if (verbosity !== undefined && (local_index % verbosity === 0)) {
+                        console.log(iterator_id + ":" + local_index + "/" + inputs.length)
+                    }
+                    const output = await this.safe_run(() => f(inputs.slice(local_index, local_index + batch_size), iterator_id, local_index))
+                    outputs[local_index] = output
+                }
+            })())
+        }
+        await Promise.all(iterators)
+        return outputs
+    }
+
 
     static async wait(duration : number) {
         return new Promise(resolve => setTimeout(resolve, duration))