triangle-utils 1.1.3 → 1.2.0

package/Utils.js

@@ -6,6 +6,7 @@ import Utils_S3 from "./utils/Utils_S3.js"
 import Utils_Bedrock from "./utils/Utils_Bedrock.js"
 import Utils_Bee from "./utils/Utils_Bee.js"
 import Utils_S3Vectors from "./utils/Utils_S3Vectors.js"
+import Utils_Youtube from "./utils/Utils_Youtube.js"
 
 export default class Utils extends Utils_Misc {
     constructor(config) {
@@ -21,5 +22,8 @@ export default class Utils extends Utils_Misc {
         if (config.scraping_bee_api_key !== undefined) {
             this.bee = new Utils_Bee(config)
         }
+        if (config.youtube_api_key !== undefined) {
+            this.youtube = new Utils_Youtube(config)
+        }
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "triangle-utils",
-    "version": "1.1.3",
+    "version": "1.2.0",
     "main": "Utils.js",
     "author": "",
     "license": "ISC",
@@ -11,7 +11,9 @@
         "@aws-sdk/client-dynamodb": "^3.953.0",
         "@aws-sdk/client-s3": "^3.953.0",
         "@aws-sdk/client-s3vectors": "^3.953.0",
+        "@aws-sdk/client-secrets-manager": "^3.965.0",
         "@aws-sdk/s3-request-presigner": "^3.953.0",
+        "googleapis": "^170.0.0",
         "nodemailer": "^7.0.11",
         "scrapingbee": "^1.7.6",
         "triangle-utils": "^1.0.34"

package/test.js

@@ -9,6 +9,8 @@ const api_keys = await secrets_manager.getSecretValue({
     SecretId: secret_name
 }).then(response => JSON.parse(response.SecretString))
 
+console.log(api_keys)
+
 const config = {
     google_email : "louishou@triangleanalytics.com",
     alerts_email : "alerts@triangleanalytics.com",
@@ -20,5 +22,6 @@ config.fec_api_keys = config.fec_api_keys.split(",")
 
 const utils = new Utils(config)
 
-const videos = await utils.bee.youtube_search("Marie Gluesenkamp Perez")
+const videos = await utils.youtube.batch_get_videos(["N4dyOzjjYYk"])
+
 console.log(videos)

package/utils/Utils_DynamoDB.js

@@ -151,7 +151,9 @@ export default class Utils_DynamoDB {
         return await compile_pages(request, (request) => this.dynamodb.query(request), compile)
     }
 
-    async query_prefix(table, primary_key, secondary_key_prefix, reverse=false) {
+    async query_prefix(table, primary_key, secondary_key_prefix, options = {}) {
+        const reverse = options.reverse !== undefined ? options.reverse : false
+        const compile = options.reverse !== undefined ? options.compile : true
         if (Object.keys(primary_key).length !== 1 || Object.keys(secondary_key_prefix).length !== 1) {
             return undefined
         }
@@ -168,10 +170,12 @@ export default class Utils_DynamoDB {
             KeyConditionExpression: "#a = :a AND begins_with(#b, :b)",
             ScanIndexForward : !reverse
         }
-        return await compile_pages(request, (request) => this.dynamodb.query(request))
+        return await compile_pages(request, (request) => this.dynamodb.query(request), compile)
     }
 
-    async query_range(table, primary_key, secondary_key_range, reverse=false) {
+    async query_range(table, primary_key, secondary_key_range, options = {}) {
+        const reverse = options.reverse !== undefined ? options.reverse : false
+        const compile = options.reverse !== undefined ? options.compile : true
         if (Object.keys(primary_key).length !== 1 || Object.keys(secondary_key_range).length !== 1 || Object.values(secondary_key_range)[0].length !== 2) {
             return undefined
         }
@@ -189,7 +193,7 @@ export default class Utils_DynamoDB {
             KeyConditionExpression: "#a = :a AND (#b BETWEEN :b1 AND :b2)",
             ScanIndexForward : !reverse
         }
-        return await compile_pages(request, (request) => this.dynamodb.query(request))
+        return await compile_pages(request, (request) => this.dynamodb.query(request), compile)
     }
 
     async set(table, key, attributes) {

package/utils/Utils_Youtube.js

@@ -0,0 +1,32 @@
+import { youtube_v3 } from "googleapis"
+
+export default class Utils_Youtube {
+
+    constructor(config) {
+        this.config = config
+        this.youtube = new youtube_v3.Youtube({
+            auth: config.youtube_api_key
+        })
+    }
+
+    async batch_get_videos(video_ids, attributes = ["snippet", "contentDetails"]) {
+        for (let i = 0; i < 3; i++) {
+            try {
+                const response = await this.youtube.videos.list({
+                    part : attributes,
+                    id : video_ids
+                })
+                if (response.status !== 200) {
+                    console.log("Status", response.status)
+                    return undefined
+                }
+                return response.data.items
+            } catch (e) {
+                console.log(e.stack)
+                console.log("Failed to get from Scraping Bee.")
+            }
+        }
+        console.log("Failed to get from Scraping Bee, quitting.")
+        return undefined
+    }
+}