triangle-utils 1.2.1 → 1.2.3

package/Utils.js

@@ -7,6 +7,7 @@ import Utils_Bedrock from "./utils/Utils_Bedrock.js"
 import Utils_Bee from "./utils/Utils_Bee.js"
 import Utils_S3Vectors from "./utils/Utils_S3Vectors.js"
 import Utils_Youtube from "./utils/Utils_Youtube.js"
+import Utils_CitizenPortal from "./utils/Utils_CitizenPortal.js"
 
 export default class Utils extends Utils_Misc {
     constructor(config) {
@@ -25,5 +26,8 @@ export default class Utils extends Utils_Misc {
         if (config.youtube_api_key !== undefined) {
             this.youtube = new Utils_Youtube(config)
         }
+        if (config.citizenportal_api_key !== undefined) {
+            this.citizenportal = new Utils_CitizenPortal(config)
+        }
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "triangle-utils",
-    "version": "1.2.1",
+    "version": "1.2.3",
     "main": "Utils.js",
     "author": "",
     "license": "ISC",

package/test.js

@@ -3,14 +3,12 @@ import { SecretsManager } from "@aws-sdk/client-secrets-manager"
 
 const secret_name = "api_keys"
 
-const secrets_manager = new SecretsManager({ region: "us-east-1" })
+const secrets_manager = new SecretsManager({ region: "us-east-2" })
 
 const api_keys = await secrets_manager.getSecretValue({
     SecretId: secret_name
 }).then(response => JSON.parse(response.SecretString))
 
-console.log(api_keys)
-
 const config = {
     google_email : "louishou@triangleanalytics.com",
     alerts_email : "alerts@triangleanalytics.com",
@@ -18,10 +16,44 @@ const config = {
     ...api_keys
 }
 
-config.fec_api_keys = config.fec_api_keys.split(",")
-
 const utils = new Utils(config)
 
-const videos = await utils.youtube.batch_get_videos(["N4dyOzjjYYk"])
+// const videos = await utils.youtube.batch_get_videos(["N4dyOzjjYYk"])
+
+// console.log(videos)
+
+// const states = await utils.citizenportal.get_sub_jurisdictions(2145)
+
+// console.log(states)
+
+const documents = await utils.citizenportal.get_jurisdiction_documents(28222)
+
+console.log(documents.length)
+
+// let response = await fetch("https://api.citizenportal.ai/api/v1/meetings", {
+//     headers : {
+//         "X-API-Key" : config.citizenportal_api_key
+//     }
+// })
+//     .then(response => response.json())
+
+// for (const doc of response.data) {
+
+//     response = await fetch("https://api.citizenportal.ai/api/v1/meetings/" + doc.packageId + "/transcript", {
+//         headers : {
+//             "X-API-Key" : config.citizenportal_api_key
+//         }
+//     })
+//     .then(response => response.json())
+//     console.log(response)
+// }
+
+// console.log(response)
+let response = await fetch("https://api.citizenportal.ai/api/v1/meetings/606000/transcript", {
+    headers : {
+        "X-API-Key" : config.citizenportal_api_key
+    }
+})
+    .then(response => response.json())
 
-console.log(videos)
+console.log(response)

package/utils/Utils_Bee.js

@@ -69,7 +69,7 @@ export default class Utils_Bee {
                 const data = await response.json()
                 const results = data.results
 
-                if (results === undefined) {
+                if (results === undefined || !Array.isArray(results)) {
                     console.log("Failed to YouTube Search.")
                     continue
                 }

package/utils/Utils_CitizenPortal.js

@@ -0,0 +1,62 @@
+export default class Utils_CitizenPortal {
+
+    constructor(config) {
+        this.config = config
+    }
+
+    async get_sub_jurisdictions(jurisdiction_id) {
+        for (let i = 0; i < 3; i++) {
+            try {
+                const citizenportal_api = "https://api.citizenportal.ai/api/v1/taxonomies?parent_id=" + jurisdiction_id.toString() + "&limit=1000"
+                console.log(citizenportal_api)
+                const response = await fetch(citizenportal_api, {
+                    headers : {
+                        "X-API-Key" : this.config.citizenportal_api_key
+                    }
+                }).then(response => response.json())
+                return response.taxonomies
+            } catch (e) {
+                console.log(e.stack)
+                console.log("Failed to get from CitizenPortal.")
+            }
+        }
+        console.log("Failed to get from CitizenPortal, quitting.")
+        return undefined
+    }
+
+    async get_jurisdiction_documents(jurisdiction_id) {
+        const documents = []
+        for (let page = 0; ; page++) {
+            const citizenportal_api = "https://api.citizenportal.ai/api/v1/meetings?taxonomy_id=" + jurisdiction_id.toString() + "&include_descendants=false&limit=100&offset=" + (100 * page).toString()
+            console.log(citizenportal_api)
+            let response = undefined
+            for (let i = 0; i < 3; i++) {
+                try {
+                    response = await fetch(citizenportal_api, {
+                        headers : {
+                            "X-API-Key" : this.config.citizenportal_api_key
+                        }
+                    })
+                        .then(response => response.json())
+                    console.log(response.data.length)
+                    if (response.data === undefined) {
+                        console.log("Failed to fetch, retrying.")
+                        continue
+                    }
+                    break
+                }
+                catch (error) {
+                    console.log("Failed to fetch, retrying.")
+                }
+            }
+            if (response === undefined || response.data === undefined) {
+                console.log("Failed thriced, exiting.")
+                return undefined
+            }
+            if (response.data.length === 0) {
+                return documents
+            }
+            documents.push(...response.data)
+        }
+    }
+}

package/utils/Utils_S3Vectors.js

@@ -48,23 +48,26 @@ export default class Utils_S3Vectors {
     }
 
     async batch_get(vector_bucket, vector_index, vector_keys) {
-        const vectors_by_key = await this.s3vectors.getVectors({
-            vectorBucketName : vector_bucket,
-            indexName : vector_index,
-            keys : vector_keys,
-            returnData : true,
-            returnMetadata : true
-        })
-            .then(response => response.vectors)
-            .then(vectors => Object.fromEntries(vectors.map(vector => [
-                vector.key,
-                {
-                    key : vector.key,
-                    data : vector.data.float32,
-                    metadata : vector.metadata
-                }
-            ])))
-        const vectors = vector_keys.map(vector_key => vectors_by_key[vector_key])
+        const vectors = []
+        for (let i = 0; i < vector_keys.length; i += 100) {
+            const vectors_by_key = await this.s3vectors.getVectors({
+                vectorBucketName : vector_bucket,
+                indexName : vector_index,
+                keys : vector_keys.slice(i, i + 100),
+                returnData : true,
+                returnMetadata : true
+            })
+                .then(response => response.vectors)
+                .then(vectors => Object.fromEntries(vectors.map(vector => [
+                    vector.key,
+                    {
+                        key : vector.key,
+                        data : vector.data.float32,
+                        metadata : vector.metadata
+                    }
+                ])))
+            vectors.push(...vector_keys.map(vector_key => vectors_by_key[vector_key]))
+        }
         return vectors
     }
 
@@ -83,17 +86,20 @@ export default class Utils_S3Vectors {
     }
 
     async batch_create(vector_bucket, vector_index, vectors) {
-        await this.s3vectors.putVectors({
-            vectorBucketName : vector_bucket,
-            indexName : vector_index,
-            vectors : vectors.map(vector => ({
-                key : vector.key,
-                data : {
-                    float32 : vector.data
-                },
-                metadata : vector.metadata
-            }))
-        })
+        for (let i = 0; i < vector_keys.length; i += 100) {
+            await this.s3vectors.putVectors({
+                vectorBucketName : vector_bucket,
+                indexName : vector_index,
+                vectors : vectors.slice(i, i + 100).map(vector => ({
+                    key : vector.key,
+                    data : {
+                        float32 : vector.data
+                    },
+                    metadata : vector.metadata
+                }))
+            })
+            
+        }
     }
 
     async delete(vector_bucket, vector_index, vector_key) {
@@ -105,11 +111,13 @@ export default class Utils_S3Vectors {
     }
 
     async batch_delete(vector_bucket, vector_index, vector_keys) {
-        await this.s3vectors.deleteVectors({
-            vectorBucketName : vector_bucket,
-            indexName : vector_index,
-            keys : vector_keys
-        })
+        for (let i = 0; i < vector_keys.length; i += 100) {
+            await this.s3vectors.deleteVectors({
+                vectorBucketName : vector_bucket,
+                indexName : vector_index.slice(i, i + 100),
+                keys : vector_keys
+            })
+        }
     }
 
     async query(vector_bucket, vector_index, data, num_vectors, filters) {