triangle-utils 1.4.184 → 1.4.186

package/dist/src/UtilsMisc.d.ts

@@ -7,12 +7,12 @@ interface Encryption {
 export declare class UtilsMisc {
     readonly config: GlobalConfig;
     constructor(config: GlobalConfig);
-    safe_run<T>(f: () => Promise<T>): Promise<T | undefined>;
-    iterate<T, U>(items: T[], f: (item: T, iterator_id: number, index: number) => Promise<U>, options?: {
+    static safe_run<T>(f: () => Promise<T>): Promise<T | undefined>;
+    static iterate<T, U>(items: T[], f: (item: T, iterator_id: number, index: number) => Promise<U>, options?: {
         num_iterators?: number;
         verbosity?: number | boolean;
     }): Promise<(U | undefined)[]>;
-    batch_iterate<T, U>(items: T[], f: (items: T[], iterator_id: number, index: number) => Promise<U>, options: {
+    static batch_iterate<T, U>(items: T[], f: (items: T[], iterator_id: number, index: number) => Promise<U>, options: {
         batch_num_items?: number;
         num_iterators?: number;
         verbosity?: number | boolean;

package/dist/src/UtilsMisc.js

@@ -5,7 +5,7 @@ export class UtilsMisc {
     constructor(config) {
         this.config = config;
     }
-    async safe_run(f) {
+    static async safe_run(f) {
         try {
             return await f();
         }
@@ -22,7 +22,7 @@ export class UtilsMisc {
         }
         return undefined;
     }
-    async iterate(items, f, options = {}) {
+    static async iterate(items, f, options = {}) {
         const num_iterators = options.num_iterators || 1;
         const verbosity = typeof options.verbosity === "boolean" ? (options.verbosity === true ? 1 : 0) : (options.verbosity || 0);
         let index = 0;
@@ -44,7 +44,7 @@ export class UtilsMisc {
         await Promise.all(iterators);
         return outputs;
     }
-    async batch_iterate(items, f, options) {
+    static async batch_iterate(items, f, options) {
         const batch_num_items = options.batch_num_items || 1;
         const num_iterators = options.num_iterators || 1;
         const verbosity = typeof options.verbosity === "boolean" ? (options.verbosity === true ? 1 : 0) : (options.verbosity || 0);

package/dist/src/UtilsRDS.d.ts

@@ -25,11 +25,12 @@ export declare class UtilsRDS {
         is_read?: boolean;
     }): Promise<Record<string, any> | undefined>;
     batch_get(table_id: string, primary_keys: Record<string, any>[], options?: {
+        num_iterators?: number;
         batch_num_items?: number;
         attribute_names?: string[];
         is_verbose?: boolean;
         is_read?: boolean;
-    }): Promise<any[]>;
+    }): Promise<any[][]>;
     create(table_id: string, item: Record<string, any>, options?: {
         is_verbose?: boolean;
         is_read?: boolean;

package/dist/src/UtilsRDS.js

@@ -1,5 +1,6 @@
 import { Pool } from "pg";
 import wkx from "wkx";
+import { UtilsMisc } from "./UtilsMisc.js";
 function convert_postgres_input(input) {
     if (input === undefined) {
         return "NULL";
@@ -96,8 +97,13 @@ export class UtilsRDS {
             console.log(sql);
         }
         const pool = is_read ? this.reader_pool : this.cluster_pool;
-        const response = await pool.query(sql);
-        const items = response.rows.map(item => Object.fromEntries(response.fields.map(field => [field.name, convert_postgres_output(item, field)])
+        const response = await pool.query(sql)
+            .then(response => Array.isArray(response) ? response.at(-1) : response);
+        if (response === undefined) {
+            return [];
+        }
+        const rows = response.rows;
+        const items = rows.map((item) => Object.fromEntries(response.fields.map(field => [field.name, convert_postgres_output(item, field)])
             .filter(([attribute_name, attribute_value]) => attribute_value !== null && attribute_value !== undefined)));
         if (!Array.isArray(items)) {
             if (is_verbose) {
@@ -134,13 +140,13 @@ export class UtilsRDS {
     async batch_get(table_id, primary_keys, options = {}) {
         const batch_num_items = options.batch_num_items || 100;
         const attribute_names = Array.from(new Set(primary_keys.map(primary_key => Object.keys(primary_key)).flat())).sort();
-        const items = [];
-        for (let i = 0; i < primary_keys.length; i += batch_num_items) {
-            const sql = "SELECT " + (options.attribute_names === undefined ? "*" : options.attribute_names.join(", ")) + " FROM " + table_id + " WHERE " + "(" + attribute_names.join(", ") + ")" + " IN " + "(" + primary_keys.slice(i, i + batch_num_items).map(primary_key => "(" + attribute_names.map(attribute_name => convert_postgres_input(primary_key[attribute_name])).join(", ") + ")").join(", ") + ")";
-            const new_items = await this.exec(sql, options);
-            items.push(...new_items);
-        }
-        return items;
+        const new_items = await UtilsMisc.batch_iterate(primary_keys, async (batch_primary_keys) => {
+            const sql = "SELECT " + (options.attribute_names === undefined ? "*" : options.attribute_names.join(", ")) + " FROM " + table_id + " WHERE " + "(" + attribute_names.join(", ") + ")" + " IN " + "(" + batch_primary_keys.map(primary_key => "(" + attribute_names.map(attribute_name => convert_postgres_input(primary_key[attribute_name])).join(", ") + ")").join(", ") + ")";
+            const batch_new_items = await this.exec(sql, options);
+            return batch_new_items;
+        }, { num_iterators: options.num_iterators, batch_num_items: batch_num_items })
+            .then(new_items => new_items.filter(new_item => new_item !== undefined));
+        return new_items;
     }
     async create(table_id, item, options = {}) {
         const attribute_names = Object.keys(item).sort();

package/dist/src/f.js

@@ -83,5 +83,6 @@ const utils = new TriangleUtils(config);
 //     .then(response => console.log(response))
 // await utils.s3.get_download_url("s3://triangle-675045717295-us-east-1-an/tenant_documents/a3a9627f-a94d-4d47-a110-307a6f5563af/a146d789-da40-4e3c-a132-8b7110f3f7df.pdf", { name : "hye.pdf" })
 //     .then(response => console.log(response))
-await utils.rds.scan("federal_census_counties", { max_num_items: 1 })
+const prisms = await utils.rds.scan("prisms");
+await utils.rds.batch_get("prisms", prisms.map(prism => ({ prism_id: prism.prism_id })), { is_verbose: true })
     .then(console.log);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "triangle-utils",
-    "version": "1.4.184",
+    "version": "1.4.186",
     "main": "dist/src/index.js",
     "types": "dist/src/index.d.ts",
     "directories": {

package/src/UtilsMisc.ts

@@ -17,7 +17,7 @@ export class UtilsMisc {
         this.config = config
     }
 
-    async safe_run<T>(f : () => Promise<T>) : Promise<T | undefined> {
+    static async safe_run<T>(f : () => Promise<T>) : Promise<T | undefined> {
         try {
             return await f()
         } catch (error) {
@@ -33,7 +33,7 @@ export class UtilsMisc {
         return undefined
     }
 
-    async iterate<T, U>(items : T[], f : (item : T, iterator_id : number, index : number) => Promise<U>, options : { num_iterators? : number, verbosity? : number | boolean } = {}) {
+    static async iterate<T, U>(items : T[], f : (item : T, iterator_id : number, index : number) => Promise<U>, options : { num_iterators? : number, verbosity? : number | boolean } = {}) {
         const num_iterators = options.num_iterators || 1
         const verbosity : number = typeof options.verbosity === "boolean" ? (options.verbosity === true ? 1 : 0) : (options.verbosity || 0)
         let index = 0
@@ -56,7 +56,7 @@ export class UtilsMisc {
         return outputs
     }
 
-    async batch_iterate<T, U>(items : T[], f : (items : T[], iterator_id : number, index : number) => Promise<U>, 
+    static async batch_iterate<T, U>(items : T[], f : (items : T[], iterator_id : number, index : number) => Promise<U>, 
     options : { batch_num_items? : number, num_iterators? : number, verbosity? : number | boolean }) {
         const batch_num_items = options.batch_num_items || 1
         const num_iterators = options.num_iterators || 1

package/src/UtilsRDS.ts

@@ -1,5 +1,7 @@
-import { FieldDef, Pool } from "pg"
+import { FieldDef, Pool, QueryResult } from "pg"
 import wkx from "wkx"
+import { TriangleUtils } from "."
+import { UtilsMisc } from "./UtilsMisc"
 
 function convert_postgres_input(input : string | boolean | number | any[] | Record<string, any> | undefined) : string {
     if (input === undefined) {
@@ -109,8 +111,13 @@ export class UtilsRDS {
             console.log(sql)
         }
         const pool = is_read ? this.reader_pool : this.cluster_pool
-        const response = await pool.query(sql)
-        const items = response.rows.map(item => Object.fromEntries(
+        const response : QueryResult<any> | undefined = await pool.query(sql)
+            .then(response => Array.isArray(response) ? response.at(-1) : response)
+        if (response === undefined) {
+            return []
+        }
+        const rows = response.rows
+        const items = rows.map((item : any) => Object.fromEntries(
             response.fields.map(field => [field.name, convert_postgres_output(item, field)])
                 .filter(([attribute_name, attribute_value]) => attribute_value !== null && attribute_value !== undefined)
         ))
@@ -172,6 +179,7 @@ export class UtilsRDS {
         table_id : string,
         primary_keys : Record<string, any>[],
         options : {
+            num_iterators? : number,
             batch_num_items? : number,
             attribute_names? : string[],
             is_verbose? : boolean,
@@ -180,13 +188,13 @@ export class UtilsRDS {
     ) {
         const batch_num_items = options.batch_num_items || 100
         const attribute_names = Array.from(new Set(primary_keys.map(primary_key => Object.keys(primary_key)).flat())).sort()
-        const items = []
-        for (let i = 0; i < primary_keys.length; i += batch_num_items) {
-            const sql = "SELECT " + (options.attribute_names === undefined ? "*" : options.attribute_names.join(", ")) + " FROM " + table_id + " WHERE " + "(" + attribute_names.join(", ") + ")" + " IN " + "(" + primary_keys.slice(i, i + batch_num_items).map(primary_key => "(" + attribute_names.map(attribute_name => convert_postgres_input(primary_key[attribute_name])).join(", ") + ")" ).join(", ") + ")"
-            const new_items = await this.exec(sql, options)
-            items.push(...new_items)
-        }
-        return items
+        const new_items = await UtilsMisc.batch_iterate(primary_keys, async (batch_primary_keys) => {
+            const sql = "SELECT " + (options.attribute_names === undefined ? "*" : options.attribute_names.join(", ")) + " FROM " + table_id + " WHERE " + "(" + attribute_names.join(", ") + ")" + " IN " + "(" + batch_primary_keys.map(primary_key => "(" + attribute_names.map(attribute_name => convert_postgres_input(primary_key[attribute_name])).join(", ") + ")" ).join(", ") + ")"
+            const batch_new_items = await this.exec(sql, options)
+            return batch_new_items
+        }, { num_iterators : options.num_iterators, batch_num_items : batch_num_items })
+            .then(new_items => new_items.filter(new_item => new_item !== undefined))
+        return new_items
     }
 
     async create(

package/src/f.ts

@@ -106,6 +106,7 @@ const utils = new TriangleUtils(config)
 // await utils.s3.get_download_url("s3://triangle-675045717295-us-east-1-an/tenant_documents/a3a9627f-a94d-4d47-a110-307a6f5563af/a146d789-da40-4e3c-a132-8b7110f3f7df.pdf", { name : "hye.pdf" })
 //     .then(response => console.log(response))
 
+const prisms = await utils.rds.scan("prisms")
 
-await utils.rds.scan("federal_census_counties", { max_num_items : 1 })
+await utils.rds.batch_get("prisms", prisms.map(prism => ({ prism_id : prism.prism_id })), { is_verbose : true })
     .then(console.log)