triangle-utils 1.4.162 → 1.4.163

package/dist/src/UtilsMisc.d.ts

@@ -1,4 +1,5 @@
 import { GlobalConfig } from "./types/GlobalConfig.js";
+import { Readable } from "stream";
 interface Encryption {
     iv: string;
     ciphertext: string;
@@ -25,5 +26,10 @@ export declare class UtilsMisc {
     static encrypt(secret_key: string, text: string): Encryption;
     static decrypt(secret_key: string, encryption: Encryption): string;
     static get_secret_hash(username: string, cognito_client_id: string, cognito_client_secret: string): string;
+    static batch_import_csv<T>(stream: Readable, import_new_items: (new_items: any[]) => Promise<void>, options: {
+        delimiter?: string;
+        batch_num_items: number;
+        max_num_items?: number;
+    }): Promise<void>;
 }
 export {};

package/dist/src/UtilsMisc.js

@@ -1,4 +1,5 @@
 import * as crypto from "crypto";
+import csv_parser from "csv-parser";
 export class UtilsMisc {
     config;
     constructor(config) {
@@ -114,4 +115,35 @@ export class UtilsMisc {
             .digest("base64");
         return secret_hash;
     }
+    static async batch_import_csv(stream, import_new_items, options) {
+        const delimiter = options.delimiter || ",";
+        const max_num_items = options.max_num_items;
+        const batch_num_items = options.batch_num_items;
+        let new_items = [];
+        let i = 0;
+        const csv_stream = stream.pipe(csv_parser({ separator: delimiter }));
+        await new Promise(resolve => {
+            csv_stream
+                .on("data", async (item) => {
+                i += 1;
+                if (max_num_items !== undefined && i > max_num_items) {
+                    csv_stream.end();
+                    return;
+                }
+                new_items.push(item);
+                if (i % batch_num_items === 0) {
+                    csv_stream.pause();
+                    console.log(i);
+                    await import_new_items(await Promise.all(new_items).then(new_items => new_items.flat()));
+                    new_items = [];
+                    csv_stream.resume();
+                }
+            })
+                .on("end", async () => {
+                await import_new_items(await Promise.all(new_items).then(new_items => new_items.flat()));
+                new_items = [];
+                resolve();
+            });
+        });
+    }
 }

package/dist/src/index.d.ts

@@ -32,7 +32,6 @@ export declare class TriangleUtils extends UtilsMisc {
 }
 export * from "./types/ApplicationConfig.js";
 export * from "./types/GlobalConfig.js";
-export * from "./UtilsMisc.js";
 export * from "./UtilsAnthropic.js";
 export * from "./UtilsDynamoDB.js";
 export * from "./UtilsRDS.js";

package/dist/src/index.js

@@ -45,7 +45,6 @@ export class TriangleUtils extends UtilsMisc {
 }
 export * from "./types/ApplicationConfig.js";
 export * from "./types/GlobalConfig.js";
-export * from "./UtilsMisc.js";
 export * from "./UtilsAnthropic.js";
 export * from "./UtilsDynamoDB.js";
 export * from "./UtilsRDS.js";

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "triangle-utils",
-    "version": "1.4.162",
+    "version": "1.4.163",
     "main": "dist/src/index.js",
     "types": "dist/src/index.d.ts",
     "directories": {
@@ -34,6 +34,7 @@
         "@types/nodemailer": "^7.0.9",
         "@types/pg": "^8.21.0",
         "@xdevplatform/xdk": "^0.5.0",
+        "csv-parser": "^3.2.1",
         "googleapis": "^170.0.0",
         "jsdom": "^29.0.1",
         "pg": "^8.23.0",

package/src/UtilsMisc.ts

@@ -1,6 +1,8 @@
 import * as crypto from "crypto"
 
 import { GlobalConfig } from "./types/GlobalConfig"
+import csv_parser from "csv-parser"
+import { Readable } from "stream"
 
 interface Encryption {
     iv : string,
@@ -140,4 +142,36 @@ export class UtilsMisc {
         return secret_hash
     }
 
+    static async batch_import_csv<T>(stream : Readable, import_new_items : (new_items : any[]) => Promise<void>, options : { delimiter? : string, batch_num_items : number, max_num_items? : number }) {
+        const delimiter = options.delimiter || ","
+        const max_num_items = options.max_num_items
+        const batch_num_items = options.batch_num_items
+        let new_items : Promise<T[]>[] = []
+        let i = 0
+        const csv_stream = stream.pipe(csv_parser({ separator: delimiter }))
+        await new Promise<void>(resolve => {
+            csv_stream
+                .on("data", async (item) => {
+                    i += 1
+                    if (max_num_items !== undefined && i > max_num_items) {
+                        csv_stream.end()
+                        return
+                    }
+                    new_items.push(item)
+                    if (i % batch_num_items === 0) {
+                        csv_stream.pause()
+                        console.log(i)
+                        await import_new_items(await Promise.all(new_items).then(new_items => new_items.flat()))
+                        new_items = []
+                        csv_stream.resume()
+                    }
+                })
+                .on("end", async () => {
+                    await import_new_items(await Promise.all(new_items).then(new_items => new_items.flat()))
+                    new_items = []
+                    resolve()
+                })
+        })
+    }
+
 }

package/src/index.ts

@@ -53,7 +53,6 @@ export class TriangleUtils extends UtilsMisc {
 
 export * from "./types/ApplicationConfig"
 export * from "./types/GlobalConfig"
-export * from "./UtilsMisc"
 export * from "./UtilsAnthropic"
 export * from "./UtilsDynamoDB"
 export * from "./UtilsRDS"