triangle-utils 1.2.12 → 1.3.1

package/{utils/Utils_S3.js → src/Utils_S3.ts}

@@ -1,7 +1,13 @@
 import { S3, NoSuchKey, GetObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3"
 import { getSignedUrl } from "@aws-sdk/s3-request-presigner"
+import { Config } from "./types/Config"
 
-function parse_s3_filename(s3_filename) {
+interface S3Key {
+    Bucket : string,
+    Key : string
+}
+
+function parse_s3_filename(s3_filename : string) : S3Key | undefined {
     if (s3_filename.substring(0, 5) !== "s3://") {
         return undefined
     }
@@ -11,13 +17,15 @@ function parse_s3_filename(s3_filename) {
     return { Bucket : bucket, Key : filename }
 }
 
-export default class Utils_S3 {
+export class Utils_S3 {
+
+    readonly s3 : S3
 
-    constructor(config) {
+    constructor(config : Config) {
         this.s3 = new S3({ region : config.region })
     }
 
-    async file_exists(s3_filename) {
+    async file_exists(s3_filename : string) {
         const s3_key = parse_s3_filename(s3_filename)
         if (s3_key === undefined) {
             return undefined
@@ -30,15 +38,19 @@ export default class Utils_S3 {
         }
     }
 
-    async get_file(s3_filename, json=false) {
+    async get_file(s3_filename : string) : Promise<string | undefined> {
         const s3_key = parse_s3_filename(s3_filename)
         if (s3_key === undefined) {
             return undefined
         }
         try {
-            return await this.s3.getObject(s3_key)
-                .then(response => response.Body.transformToString("utf-8"))
-                .then(text => json ? JSON.parse(text) : text)
+            const response = await this.s3.getObject(s3_key)
+            const body = response.Body
+            if (body === undefined) {
+                return undefined
+            }
+            const text = body.transformToString("utf-8")
+            return text
         } catch (error) {
             if (error instanceof NoSuchKey) {
                 return undefined
@@ -47,8 +59,10 @@ export default class Utils_S3 {
         }
     }
 
-    async create_file(s3_filename, content, options = {}) {
-        const content_type = options.content_type !== undefined ? options.content_type : {}
+    async create_file(s3_filename : string, content : string | Buffer, options : {
+        content_type? : string
+    } = {}) {
+        const content_type = options.content_type !== undefined ? options.content_type : undefined
 
         const exists = await this.file_exists(s3_filename)
         if (exists) {
@@ -65,27 +79,33 @@ export default class Utils_S3 {
         })
     }
 
-    async delete_file(s3_filename) {
+    async delete_file(s3_filename : string) : Promise<void>{
         const s3_key = parse_s3_filename(s3_filename)
         if (s3_key === undefined) {
             return undefined
         }
-        return await this.s3.deleteObject(s3_key)
+        await this.s3.deleteObject(s3_key)
     }
 
-    async generate_download_url(s3_filename) {
+    async generate_download_url(s3_filename : string) : Promise<string | undefined> {
         const exists = await this.file_exists(s3_filename)
         if (!exists) {
             return undefined
         }
         const s3_key = parse_s3_filename(s3_filename)
+        if (s3_key === undefined) {
+            return undefined
+        }
         const command = new GetObjectCommand(s3_key)
         const url = await getSignedUrl(this.s3, command)
         return url
     }
 
-    async generate_upload_url(s3_filename) {
+    async generate_upload_url(s3_filename : string) : Promise<string | undefined> {
         const s3_key = parse_s3_filename(s3_filename)
+        if (s3_key === undefined) {
+            return undefined
+        }
         const command = new PutObjectCommand(s3_key)
         const url = await getSignedUrl(this.s3, command)
         return url

package/{utils/Utils_S3Vectors.js → src/Utils_S3Vectors.ts}

@@ -1,43 +1,52 @@
-import { S3Vectors } from "@aws-sdk/client-s3vectors"
+import { ListVectorsCommandInput, S3Vectors } from "@aws-sdk/client-s3vectors"
+import { Config } from "./types/Config"
 
-export default class Utils_S3Vectors {
+interface Vector {
+    key : string,
+    data : number[],
+    metadata : any
+}
 
-    constructor(config) {
+export class Utils_S3Vectors {
+
+    readonly s3vectors : S3Vectors
+
+    constructor(config : Config) {
         this.s3vectors = new S3Vectors({ region : config.region })
     }
 
-    async scan(vector_bucket, vector_index) {
-        const vectors = []
-        let next_token = undefined
+    async scan(vector_bucket : string, vector_index : string) : Promise<string[]> {
+        const vector_keys = []
+        let next_token : string | undefined = undefined
         while (true) {
-            const response = (next_token === undefined) ? (await this.s3vectors.listVectors({
-                vectorBucketName : vector_bucket,
-                indexName : vector_index,
-                maxResults : 1000
-            })) : (await this.s3vectors.listVectors({
+            const request : ListVectorsCommandInput = {
                 vectorBucketName : vector_bucket,
                 indexName : vector_index,
                 maxResults : 1000,
                 nextToken : next_token
-            }))
-            vectors.push(...response.vectors)
+            }
+            const response = await this.s3vectors.listVectors(request)
+            if (response === undefined || response.vectors === undefined) {
+                return []
+            }
+            vector_keys.push(...response.vectors.map(vector => vector.key))
             next_token = response.nextToken
             if (next_token === undefined) {
-                return vectors
+                return vector_keys.filter(vector_key => vector_key !== undefined)
             }
         }
     }
 
-    async get(vector_bucket, vector_index, vector_key) {
-        const vector = await this.s3vectors.getVectors({
+    async get(vector_bucket : string, vector_index : string, vector_key : string) : Promise<Vector | undefined> {
+        const response = await this.s3vectors.getVectors({
             vectorBucketName : vector_bucket,
             indexName : vector_index,
             keys : [vector_key],
             returnData : true,
             returnMetadata : true
         })
-            .then(response => response.vectors[0])
-        if (vector === undefined) {
+        const vector = response.vectors?.[0]
+        if (vector === undefined || vector.key === undefined || vector.data === undefined || vector.data.float32 === undefined || vector.metadata === undefined) {
             return undefined
         }
         return {
@@ -47,10 +56,10 @@ export default class Utils_S3Vectors {
         }
     }
 
-    async batch_get(vector_bucket, vector_index, vector_keys) {
+    async batch_get(vector_bucket : string, vector_index : string, vector_keys : string[]) {
         const vectors = []
         for (let i = 0; i < vector_keys.length; i += 100) {
-            const vectors_by_key = await this.s3vectors.getVectors({
+            const new_vectors = await this.s3vectors.getVectors({
                 vectorBucketName : vector_bucket,
                 indexName : vector_index,
                 keys : vector_keys.slice(i, i + 100),
@@ -58,20 +67,25 @@ export default class Utils_S3Vectors {
                 returnMetadata : true
             })
                 .then(response => response.vectors)
-                .then(vectors => Object.fromEntries(vectors.map(vector => [
+            if (new_vectors === undefined) {
+                return []
+            }
+            const vectors_by_key : Record<string, Vector | undefined> = Object.fromEntries(new_vectors
+                .map(vector => [
                     vector.key,
-                    {
+                    vector.key !== undefined && vector.data !== undefined && vector.data.float32 !== undefined && vector.metadata !== undefined ? {
                         key : vector.key,
                         data : vector.data.float32,
                         metadata : vector.metadata
-                    }
-                ])))
+                    } : undefined
+                ])
+            )
             vectors.push(...vector_keys.map(vector_key => vectors_by_key[vector_key]))
         }
         return vectors
     }
 
-    async create(vector_bucket, vector_index, vector) {
+    async create(vector_bucket : string, vector_index : string, vector : Vector) : Promise<void> {
         await this.s3vectors.putVectors({
             vectorBucketName : vector_bucket,
             indexName : vector_index,
@@ -85,7 +99,7 @@ export default class Utils_S3Vectors {
         })
     }
 
-    async batch_create(vector_bucket, vector_index, vectors) {
+    async batch_create(vector_bucket : string, vector_index : string, vectors : Vector[]) : Promise<void> {
         for (let i = 0; i < vectors.length; i += 100) {
             await this.s3vectors.putVectors({
                 vectorBucketName : vector_bucket,
@@ -102,7 +116,7 @@ export default class Utils_S3Vectors {
         }
     }
 
-    async delete(vector_bucket, vector_index, vector_key) {
+    async delete(vector_bucket : string, vector_index : string, vector_key : string) : Promise<void> {
         await this.s3vectors.deleteVectors({
             vectorBucketName : vector_bucket,
             indexName : vector_index,
@@ -110,7 +124,7 @@ export default class Utils_S3Vectors {
         })
     }
 
-    async batch_delete(vector_bucket, vector_index, vector_keys) {
+    async batch_delete(vector_bucket : string, vector_index : string, vector_keys : string[]) : Promise<void> {
         for (let i = 0; i < vector_keys.length; i += 100) {
             await this.s3vectors.deleteVectors({
                 vectorBucketName : vector_bucket,
@@ -120,7 +134,7 @@ export default class Utils_S3Vectors {
         }
     }
 
-    async query(vector_bucket, vector_index, data, num_vectors, filters) {
+    async query(vector_bucket : string, vector_index : string, data : number[], num_vectors : number, filters : Record<string, any> = {}) {
         const response = await this.s3vectors.queryVectors({
             vectorBucketName : vector_bucket,
             indexName : vector_index,

package/{utils/Utils_Youtube.js → src/Utils_Youtube.ts}

@@ -1,15 +1,19 @@
 import { youtube_v3 } from "googleapis"
+import { Config } from "./types/Config"
 
-export default class Utils_Youtube {
+export class Utils_Youtube {
 
-    constructor(config) {
+    readonly config : Config
+    readonly youtube : youtube_v3.Youtube
+
+    constructor(config : Config) {
         this.config = config
         this.youtube = new youtube_v3.Youtube({
             auth: config.youtube_api_key
         })
     }
 
-    async batch_get_videos(video_ids, attributes = ["snippet", "contentDetails"]) {
+    async batch_get_videos(video_ids : string[], attributes : string[] = ["snippet", "contentDetails"]) {
         for (let i = 0; i < 3; i++) {
             try {
                 const response = await this.youtube.videos.list({
@@ -20,9 +24,12 @@ export default class Utils_Youtube {
                     console.log("Status", response.status)
                     return undefined
                 }
-                return response.data.items
-            } catch (e) {
-                console.log(e.stack)
+                const items = response.data.items
+                return items
+            } catch (error) {
+                if (error instanceof Error) {
+                    console.log(error.stack)
+                }
                 console.log("Failed to get from Scraping Bee.")
             }
         }

package/src/types/Config.ts

@@ -0,0 +1,9 @@
+export interface Config {
+    region : string,
+    // fec_api_key? : string,
+    google_app_password? : string,
+    scraping_bee_api_key? : string,
+    youtube_api_key? : string,
+    citizenportal_api_key? : string,
+    [key : string]: any
+}

package/test/Utils.test.ts

@@ -0,0 +1,85 @@
+import { Config } from "../src/types/Config.ts"
+import { Utils } from "../src/Utils.ts"
+import { SecretsManager } from "@aws-sdk/client-secrets-manager"
+
+const region = "us-east-1"
+
+let config : Config = { region : region }
+
+beforeAll(async () => {
+    // const secret_name = "api_keys"
+
+    // const secrets_manager = new SecretsManager({ region: "us-east-1" })
+
+    // const api_keys = await secrets_manager.getSecretValue({
+    //     SecretId: secret_name
+    // }).then(response => JSON.parse(response.SecretString))
+
+    // config = {
+    //     google_email : "louishou@triangleanalytics.com",
+    //     alerts_email : "alerts@triangleanalytics.com",
+    //     region : "us-east-1",
+    //     ...api_keys
+    // }
+
+})
+
+describe("Testing Utils", () => {
+    describe("Testing Misc", () => {
+        test("email", async () => {
+            const utils = new Utils(config)
+            await utils.admin_alert("Testing Admin Alert.")
+        })
+        test("sha256", async () => {
+            const utils = new Utils(config)
+            const hash = await utils.sha256("Triangle Analytics")
+            expect(hash).toEqual("7f6f79a3fdee7c38bb3b0381211575a35ffcfcf5867d6045b07b4181e6f24153")
+        })
+    })
+    describe("Testing DynamoDB", () => {
+        test("scan", async () => {
+            const utils = new Utils(config)
+            const candidates = await utils.dynamodb.scan("candidates")
+            expect(candidates).toBeDefined()
+            expect(candidates.length).toBeGreaterThan(3000)
+        })
+        test("get; single key", async () => {
+            const utils = new Utils(config)
+            const candidate = await utils.dynamodb.get("candidates", { candidate_id : "S6MI00392" })
+            expect(candidate).toBeDefined()
+            if (candidate === undefined) {
+                return
+            }
+            expect(candidate.fec_name).toBeDefined()
+        })
+        test("get; composite key", async () => {
+            const utils = new Utils(config)
+            const candidate_article = await utils.dynamodb.get("candidate_articles", { candidate_id : "S6MI00392", article_id : "2025-08-1674855ce5e59c2b6a19a165037dffa5627b1cee071def07a614acb1c28d0e2eff" })
+            expect(candidate_article).toBeDefined()
+            if (candidate_article === undefined) {
+                return
+            }
+            expect(candidate_article.url).toBeDefined()
+        })
+        test("get_max", async () => {
+            const utils = new Utils(config)
+            const candidate_article = await utils.dynamodb.get_max("candidate_articles", { candidate_id : "S6MI00392" })
+            expect(candidate_article).toBeDefined()
+            if (candidate_article === undefined) {
+                return
+            }
+            expect(candidate_article.url).toBeDefined()
+            expect(candidate_article.article_id).toBeDefined()
+            expect(candidate_article.article_id.localeCompare("2025-08-1674855ce5e59c2b6a19a165037dffa5627b1cee071def07a614acb1c28d0e2eff") > 0).toBeTruthy()
+        })
+        test("query", async () => {
+            const utils = new Utils(config)
+            const candidate_articles = await utils.dynamodb.query("candidate_articles", { candidate_id : "S6MI00392" })
+            expect(candidate_articles).toBeDefined()
+            if (candidate_articles === undefined) {
+                return
+            }
+            expect(candidate_articles.length).toBeGreaterThan(400)
+        })
+    })
+})

package/tsconfig.json

@@ -0,0 +1,31 @@
+{
+	"compilerOptions": {
+        "outDir": "./dist",
+
+		// Treat files as modules even if it doesn't use import/export
+		"moduleDetection": "force",
+
+		// Ignore module structure
+		"module":"esnext",
+        "target" : "es2022",
+        "noImplicitAny" : true,
+        "noImplicitReturns" : true,
+        "moduleResolution": "node",
+
+		// Allow JS files to be imported from TS and vice versa
+		"allowJs": true,
+
+		// Use correct ESM import behavior
+		"esModuleInterop": true,
+        "resolveJsonModule": true,
+		// Disallow features that require cross-file awareness
+		"isolatedModules": true,
+    	"lib": ["es2015"],
+        "rootDir": "./",
+        "skipLibCheck": true,
+        "maxNodeModuleJsDepth": 0
+	},
+    
+    "include": ["./src/**/*.*"],
+    "exclude":["node_modules"]
+}

package/Utils.js

@@ -1,33 +0,0 @@
-import states from "./states.js"
-
-import Utils_Misc from "./utils/Utils_Misc.js"
-import Utils_DynamoDB from "./utils/Utils_DynamoDB.js"
-import Utils_S3 from "./utils/Utils_S3.js"
-import Utils_Bedrock from "./utils/Utils_Bedrock.js"
-import Utils_Bee from "./utils/Utils_Bee.js"
-import Utils_S3Vectors from "./utils/Utils_S3Vectors.js"
-import Utils_Youtube from "./utils/Utils_Youtube.js"
-import Utils_CitizenPortal from "./utils/Utils_CitizenPortal.js"
-
-export default class Utils extends Utils_Misc {
-    constructor(config) {
-        super(config)
-        this.config = config
-        this.states = states
-        if (config.region !== undefined) {
-            this.dynamodb = new Utils_DynamoDB(config)
-            this.s3 = new Utils_S3(config)
-            this.s3vectors = new Utils_S3Vectors(config)
-            this.bedrock = new Utils_Bedrock(config)
-        }
-        if (config.scraping_bee_api_key !== undefined) {
-            this.bee = new Utils_Bee(config)
-        }
-        if (config.youtube_api_key !== undefined) {
-            this.youtube = new Utils_Youtube(config)
-        }
-        if (config.citizenportal_api_key !== undefined) {
-            this.citizenportal = new Utils_CitizenPortal(config)
-        }
-    }
-}

package/test.js

@@ -1,62 +0,0 @@
-import Utils from "./Utils.js";
-import { SecretsManager } from "@aws-sdk/client-secrets-manager"
-
-const secret_name = "api_keys"
-
-const secrets_manager = new SecretsManager({ region: "us-east-2" })
-
-const api_keys = await secrets_manager.getSecretValue({
-    SecretId: secret_name
-}).then(response => JSON.parse(response.SecretString))
-
-const config = {
-    google_email : "louishou@triangleanalytics.com",
-    alerts_email : "alerts@triangleanalytics.com",
-    region : "us-east-1",
-    ...api_keys
-}
-
-const utils = new Utils(config)
-
-// const videos = await utils.youtube.batch_get_videos(["N4dyOzjjYYk"])
-
-// console.log(videos)
-
-const states = await utils.citizenportal.get("taxonomies", {
-    parent_id : "2145",
-    limit : 100
-})
-
-console.log(states)
-
-// const documents = await utils.citizenportal.get_jurisdiction_documents(28222)
-
-// console.log(documents.length)
-
-// // let response = await fetch("https://api.citizenportal.ai/api/v1/meetings", {
-// //     headers : {
-// //         "X-API-Key" : config.citizenportal_api_key
-// //     }
-// // })
-// //     .then(response => response.json())
-
-// // for (const doc of response.data) {
-
-// //     response = await fetch("https://api.citizenportal.ai/api/v1/meetings/" + doc.packageId + "/transcript", {
-// //         headers : {
-// //             "X-API-Key" : config.citizenportal_api_key
-// //         }
-// //     })
-// //     .then(response => response.json())
-// //     console.log(response)
-// // }
-
-// // console.log(response)
-// let response = await fetch("https://api.citizenportal.ai/api/v1/meetings/606000/transcript", {
-//     headers : {
-//         "X-API-Key" : config.citizenportal_api_key
-//     }
-// })
-//     .then(response => response.json())
-
-// console.log(response)

package/utils/Utils_CitizenPortal.js

@@ -1,26 +0,0 @@
-export default class Utils_CitizenPortal {
-
-    constructor(config) {
-        this.config = config
-    }
-
-    async get(path, options) {
-        for (let i = 0; i < 3; i++) {
-            try {
-                const citizenportal_api = "https://api.citizenportal.ai/api/v1/" + path + "?" + Object.entries(options).filter(([key, value]) => value !== undefined).map(([key, value]) => key + "=" + value).join("&")
-                console.log(citizenportal_api)
-                const response = await fetch(citizenportal_api, {
-                    headers : {
-                        "X-API-Key" : this.config.citizenportal_api_key
-                    }
-                }).then(response => response.json())
-                return response
-            } catch (e) {
-                console.log(e.stack)
-                console.log("Failed to get from CitizenPortal.")
-            }
-        }
-        console.log("Failed to get from CitizenPortal, quitting.")
-        return undefined
-    }
-}