triangle-utils 1.4.161 → 1.4.163

package/dist/src/UtilsMisc.d.ts

@@ -1,4 +1,5 @@
 import { GlobalConfig } from "./types/GlobalConfig.js";
+import { Readable } from "stream";
 interface Encryption {
     iv: string;
     ciphertext: string;
@@ -25,5 +26,10 @@ export declare class UtilsMisc {
     static encrypt(secret_key: string, text: string): Encryption;
     static decrypt(secret_key: string, encryption: Encryption): string;
     static get_secret_hash(username: string, cognito_client_id: string, cognito_client_secret: string): string;
+    static batch_import_csv<T>(stream: Readable, import_new_items: (new_items: any[]) => Promise<void>, options: {
+        delimiter?: string;
+        batch_num_items: number;
+        max_num_items?: number;
+    }): Promise<void>;
 }
 export {};

package/dist/src/UtilsMisc.js

@@ -1,4 +1,5 @@
 import * as crypto from "crypto";
+import csv_parser from "csv-parser";
 export class UtilsMisc {
     config;
     constructor(config) {
@@ -114,4 +115,35 @@ export class UtilsMisc {
             .digest("base64");
         return secret_hash;
     }
+    static async batch_import_csv(stream, import_new_items, options) {
+        const delimiter = options.delimiter || ",";
+        const max_num_items = options.max_num_items;
+        const batch_num_items = options.batch_num_items;
+        let new_items = [];
+        let i = 0;
+        const csv_stream = stream.pipe(csv_parser({ separator: delimiter }));
+        await new Promise(resolve => {
+            csv_stream
+                .on("data", async (item) => {
+                i += 1;
+                if (max_num_items !== undefined && i > max_num_items) {
+                    csv_stream.end();
+                    return;
+                }
+                new_items.push(item);
+                if (i % batch_num_items === 0) {
+                    csv_stream.pause();
+                    console.log(i);
+                    await import_new_items(await Promise.all(new_items).then(new_items => new_items.flat()));
+                    new_items = [];
+                    csv_stream.resume();
+                }
+            })
+                .on("end", async () => {
+                await import_new_items(await Promise.all(new_items).then(new_items => new_items.flat()));
+                new_items = [];
+                resolve();
+            });
+        });
+    }
 }

package/dist/src/index.d.ts

@@ -32,7 +32,6 @@ export declare class TriangleUtils extends UtilsMisc {
 }
 export * from "./types/ApplicationConfig.js";
 export * from "./types/GlobalConfig.js";
-export * from "./UtilsMisc.js";
 export * from "./UtilsAnthropic.js";
 export * from "./UtilsDynamoDB.js";
 export * from "./UtilsRDS.js";

package/dist/src/index.js

@@ -45,7 +45,6 @@ export class TriangleUtils extends UtilsMisc {
 }
 export * from "./types/ApplicationConfig.js";
 export * from "./types/GlobalConfig.js";
-export * from "./UtilsMisc.js";
 export * from "./UtilsAnthropic.js";
 export * from "./UtilsDynamoDB.js";
 export * from "./UtilsRDS.js";

package/dist/src/types/GlobalConfig.d.ts

@@ -21,6 +21,7 @@ export declare class GlobalConfig {
     readonly twitter_api_key: string;
     readonly youtube_api_key: string;
     readonly xai_api_key: string;
+    readonly ballotpedia_api_key: string;
     readonly [x: string]: any;
     constructor(global_config: GlobalConfig);
     static is(global_config: any): global_config is GlobalConfig;

package/dist/src/types/GlobalConfig.js

@@ -21,6 +21,7 @@ export class GlobalConfig {
     twitter_api_key;
     youtube_api_key;
     xai_api_key;
+    ballotpedia_api_key;
     constructor(global_config) {
         if (!GlobalConfig.is(global_config)) {
             throw Error("Invalid input.");
@@ -47,6 +48,7 @@ export class GlobalConfig {
         this.twitter_api_key = global_config.twitter_api_key;
         this.youtube_api_key = global_config.youtube_api_key;
         this.xai_api_key = global_config.xai_api_key;
+        this.ballotpedia_api_key = global_config.ballotpedia_api_key;
     }
     static is(global_config) {
         return (global_config !== undefined &&
@@ -71,6 +73,7 @@ export class GlobalConfig {
             typeof global_config.scraping_bee_api_key === "string" &&
             typeof global_config.twitter_api_key === "string" &&
             typeof global_config.youtube_api_key === "string" &&
-            typeof global_config.xai_api_key === "string");
+            typeof global_config.xai_api_key === "string" &&
+            typeof global_config.ballotpedia_api_key === "string");
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "triangle-utils",
-    "version": "1.4.161",
+    "version": "1.4.163",
     "main": "dist/src/index.js",
     "types": "dist/src/index.d.ts",
     "directories": {
@@ -34,6 +34,7 @@
         "@types/nodemailer": "^7.0.9",
         "@types/pg": "^8.21.0",
         "@xdevplatform/xdk": "^0.5.0",
+        "csv-parser": "^3.2.1",
         "googleapis": "^170.0.0",
         "jsdom": "^29.0.1",
         "pg": "^8.23.0",

package/src/UtilsMisc.ts

@@ -1,6 +1,8 @@
 import * as crypto from "crypto"
 
 import { GlobalConfig } from "./types/GlobalConfig"
+import csv_parser from "csv-parser"
+import { Readable } from "stream"
 
 interface Encryption {
     iv : string,
@@ -140,4 +142,36 @@ export class UtilsMisc {
         return secret_hash
     }
 
+    static async batch_import_csv<T>(stream : Readable, import_new_items : (new_items : any[]) => Promise<void>, options : { delimiter? : string, batch_num_items : number, max_num_items? : number }) {
+        const delimiter = options.delimiter || ","
+        const max_num_items = options.max_num_items
+        const batch_num_items = options.batch_num_items
+        let new_items : Promise<T[]>[] = []
+        let i = 0
+        const csv_stream = stream.pipe(csv_parser({ separator: delimiter }))
+        await new Promise<void>(resolve => {
+            csv_stream
+                .on("data", async (item) => {
+                    i += 1
+                    if (max_num_items !== undefined && i > max_num_items) {
+                        csv_stream.end()
+                        return
+                    }
+                    new_items.push(item)
+                    if (i % batch_num_items === 0) {
+                        csv_stream.pause()
+                        console.log(i)
+                        await import_new_items(await Promise.all(new_items).then(new_items => new_items.flat()))
+                        new_items = []
+                        csv_stream.resume()
+                    }
+                })
+                .on("end", async () => {
+                    await import_new_items(await Promise.all(new_items).then(new_items => new_items.flat()))
+                    new_items = []
+                    resolve()
+                })
+        })
+    }
+
 }

package/src/index.ts

@@ -53,7 +53,6 @@ export class TriangleUtils extends UtilsMisc {
 
 export * from "./types/ApplicationConfig"
 export * from "./types/GlobalConfig"
-export * from "./UtilsMisc"
 export * from "./UtilsAnthropic"
 export * from "./UtilsDynamoDB"
 export * from "./UtilsRDS"

package/src/types/GlobalConfig.ts

@@ -21,6 +21,7 @@ export class GlobalConfig {
     readonly twitter_api_key : string
     readonly youtube_api_key : string
     readonly xai_api_key : string
+    readonly ballotpedia_api_key : string
     readonly [x : string] : any
 
     constructor(global_config : GlobalConfig) {
@@ -49,6 +50,7 @@ export class GlobalConfig {
         this.twitter_api_key = global_config.twitter_api_key
         this.youtube_api_key = global_config.youtube_api_key
         this.xai_api_key = global_config.xai_api_key
+        this.ballotpedia_api_key = global_config.ballotpedia_api_key
     }
     
     static is(global_config : any) : global_config is GlobalConfig {
@@ -75,7 +77,8 @@ export class GlobalConfig {
             typeof global_config.scraping_bee_api_key === "string" &&
             typeof global_config.twitter_api_key === "string" &&
             typeof global_config.youtube_api_key === "string" &&
-            typeof global_config.xai_api_key === "string"
+            typeof global_config.xai_api_key === "string" &&
+            typeof global_config.ballotpedia_api_key === "string"
         )
     }
 }