triangle-utils 1.0.36 → 1.1.0

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "triangle-utils",
-    "version": "1.0.36",
+    "version": "1.1.0",
     "main": "Utils.js",
     "author": "",
     "license": "ISC",
@@ -11,6 +11,7 @@
         "@aws-sdk/client-dynamodb": "^3.953.0",
         "@aws-sdk/client-s3": "^3.953.0",
         "@aws-sdk/client-s3vectors": "^3.953.0",
+        "@aws-sdk/client-secrets-manager": "^3.964.0",
         "@aws-sdk/s3-request-presigner": "^3.953.0",
         "nodemailer": "^7.0.11",
         "scrapingbee": "^1.7.6",

package/test.js

@@ -1,11 +1,24 @@
 import Utils from "./Utils.js";
+import { SecretsManager } from "@aws-sdk/client-secrets-manager"
 
+const secret_name = "api_keys"
 
-const utils = new Utils({ region : "us-east-2"})
+const secrets_manager = new SecretsManager({ region: "us-east-1" })
 
-const jurisdiction_documents = await utils.dynamodb.scan("jurisdiction_documents", {
-    defined_attribute_names : ["s3_filename"],
-    undefined_attribute_names : ["vector_keys"]
-})
+const api_keys = await secrets_manager.getSecretValue({
+    SecretId: secret_name
+}).then(response => JSON.parse(response.SecretString))
 
-console.log(jurisdiction_documents.length)
+const config = {
+    google_email : "louishou@triangleanalytics.com",
+    alerts_email : "alerts@triangleanalytics.com",
+    region : "us-east-1",
+    ...api_keys
+}
+
+config.fec_api_keys = config.fec_api_keys.split(",")
+
+const utils = new Utils(config)
+
+const videos = await utils.bee.youtube_search("Marie Gluesenkamp Perez")
+console.log(videos)

package/utils/Utils_Bee.js

@@ -55,4 +55,30 @@ export default class Utils_Bee {
         console.log("Failed to Google thrice, quitting.")
         return undefined
     }
+
+    async youtube_search(query) {
+        for (let i = 0; i < 3; i++) {
+            try {
+                const bee_url = "https://app.scrapingbee.com/api/v1/youtube/search?api_key=" + this.config.scraping_bee_api_key + "&search=" + encodeURI(query) + "&sort_by=upload_date"
+                console.log(bee_url)
+                const response = await fetch(bee_url)
+                if (!response.ok) {
+                    console.log("Failed to YouTube Search:", response.status)
+                    continue
+                }
+                const data = await response.json()
+                const results = data.results
+
+                if (results === undefined) {
+                    console.log("Failed to YouTube Search.")
+                    continue
+                }
+                return results
+            } catch {
+                console.log("Failed to YouTube Search.")
+            }
+        }
+        console.log("Failed to YouTube Search thrice, quitting.")
+        return undefined
+    }
 }

package/utils/Utils_DynamoDB.js

@@ -38,7 +38,7 @@ function convert_input(input) {
     return undefined
 }
 
-async function paginate(request, f) {
+async function compile_pages(request, f, compile = true) {
     const items = []
     let last_eval_key = undefined
     while (true) {
@@ -49,7 +49,7 @@ async function paginate(request, f) {
         )
         const new_items = response.Items.map(item => convert_output({ M : item }))
         items.push(...new_items)
-        if (response.LastEvaluatedKey === undefined) {
+        if (response.LastEvaluatedKey === undefined || !compile) {
             return items
         }
         last_eval_key = response.LastEvaluatedKey
@@ -86,7 +86,7 @@ export default class Utils_DynamoDB {
                 Segment : i,
                 TotalSegments : concurrency
             }).filter(([field, value]) => value !== undefined && (typeof value === typeof 3 || Object.keys(value).length !== 0)))
-            iterators.push(paginate(request, (request) => this.dynamodb.scan(request)))
+            iterators.push(compile_pages(request, (request) => this.dynamodb.scan(request)))
         }
         const segments = await Promise.all(iterators)
         return segments.flat()
@@ -131,7 +131,9 @@ export default class Utils_DynamoDB {
 
 
 
-    async query(table, primary_key, reverse=false) {
+    async query(table, primary_key, options = {}) {
+        const reverse = options.reverse !== undefined ? options.reverse : false
+        const compile = options.reverse !== undefined ? options.compile : true
         if (Object.keys(primary_key).length !== 1) {
             return undefined
         }
@@ -146,7 +148,7 @@ export default class Utils_DynamoDB {
             KeyConditionExpression: "#a = :a",
             ScanIndexForward : !reverse
         }
-        return await paginate(request, (request) => this.dynamodb.query(request))
+        return await compile_pages(request, (request) => this.dynamodb.query(request), compile)
     }
 
     async query_prefix(table, primary_key, secondary_key_prefix, reverse=false) {
@@ -166,7 +168,7 @@ export default class Utils_DynamoDB {
             KeyConditionExpression: "#a = :a AND begins_with(#b, :b)",
             ScanIndexForward : !reverse
         }
-        return await paginate(request, (request) => this.dynamodb.query(request))
+        return await compile_pages(request, (request) => this.dynamodb.query(request))
     }
 
     async query_range(table, primary_key, secondary_key_range, reverse=false) {
@@ -187,7 +189,7 @@ export default class Utils_DynamoDB {
             KeyConditionExpression: "#a = :a AND (#b BETWEEN :b1 AND :b2)",
             ScanIndexForward : !reverse
         }
-        return await paginate(request, (request) => this.dynamodb.query(request))
+        return await compile_pages(request, (request) => this.dynamodb.query(request))
     }
 
     async set(table, key, attributes) {