triangle-utils 1.3.4 → 1.4.0

package/src/{Utils_Youtube.ts → UtilsYoutube.ts}

@@ -1,19 +1,19 @@
 import { youtube_v3 } from "googleapis"
-import { Config } from "./types/Config"
 
-export class Utils_Youtube {
+export class UtilsYoutube {
 
-    readonly config : Config
-    readonly youtube : youtube_v3.Youtube
+    readonly youtube : youtube_v3.Youtube | undefined
 
-    constructor(config : Config) {
-        this.config = config
-        this.youtube = new youtube_v3.Youtube({
-            auth: config.youtube_api_key
-        })
+    constructor(youtube_api_key : string | undefined) {
+        this.youtube = youtube_api_key !== undefined ? new youtube_v3.Youtube({
+            auth: youtube_api_key
+        }) : undefined
     }
 
     async batch_get_videos(video_ids : string[], attributes : string[] = ["snippet", "contentDetails"]) {
+        if (this.youtube === undefined) {
+            return undefined
+        }
         for (let i = 0; i < 3; i++) {
             try {
                 const response = await this.youtube.videos.list({

package/src/index.ts

@@ -0,0 +1,43 @@
+import { TriangleUtilsConfig } from "./types/TriangleUtilsConfig"
+import { UtilsDynamoDB } from "./UtilsDynamoDB"
+import { UtilsS3 } from "./UtilsS3"
+import { UtilsBedrock } from "./UtilsBedrock"
+import { UtilsBee } from "./UtilsBee"
+import { UtilsS3Vectors } from "./UtilsS3Vectors"
+import { UtilsCognito } from "./UtilsCognito"
+import { UtilsMisc } from "./UtilsMisc"
+import { UtilsYoutube } from "./UtilsYoutube"
+
+
+
+export class TriangleUtils extends UtilsMisc {
+
+    readonly dynamodb : UtilsDynamoDB
+    readonly s3 : UtilsS3
+    readonly s3vectors : UtilsS3Vectors
+    readonly bedrock : UtilsBedrock
+    readonly cognito : UtilsCognito
+    readonly bee : UtilsBee
+    readonly youtube : UtilsYoutube
+
+    constructor(config : TriangleUtilsConfig) {
+        super(config)
+        this.dynamodb = new UtilsDynamoDB(config.region)
+        this.s3 = new UtilsS3(config.region)
+        this.s3vectors = new UtilsS3Vectors(config.region)
+        this.bedrock = new UtilsBedrock(config.region)
+        this.cognito = new UtilsCognito(config.region)
+        this.bee = new UtilsBee(config.scraping_bee_api_key)
+        this.youtube = new UtilsYoutube(config.youtube_api_key)
+    }
+}
+
+export * from "./types/TriangleUtilsConfig"
+export * from "./UtilsMisc"
+export * from "./UtilsDynamoDB"
+export * from "./UtilsS3"
+export * from "./UtilsBedrock"
+export * from "./UtilsBee"
+export * from "./UtilsS3Vectors"
+export * from "./UtilsCognito"
+export * from "./UtilsYoutube"

package/src/types/TriangleUtilsConfig.ts

@@ -0,0 +1,6 @@
+export interface TriangleUtilsConfig {
+    region : string,
+    scraping_bee_api_key? : string,
+    youtube_api_key? : string,
+    [key : string]: any
+}

package/test/Utils.test.ts

@@ -1,50 +1,57 @@
-import { Config } from "../src/types/Config.ts"
-import { Utils } from "../src/Utils.ts"
 import { SecretsManager } from "@aws-sdk/client-secrets-manager"
 
+import { beforeAll, test, describe, expect } from "vitest"
+
+import { TriangleUtils, TriangleUtilsConfig, UtilsMisc } from "../src"
+
 const region = "us-east-1"
 
-let config : Config = { region : region }
+let config : TriangleUtilsConfig = { region : region }
 
 beforeAll(async () => {
-    // const secret_name = "api_keys"
+    const secret_name = "api_keys"
 
-    // const secrets_manager = new SecretsManager({ region: "us-east-1" })
+    const secrets_manager = new SecretsManager({ region: "us-east-1" })
+
+    const response = await secrets_manager.getSecretValue({
+        SecretId: secret_name
+    })
 
-    // const api_keys = await secrets_manager.getSecretValue({
-    //     SecretId: secret_name
-    // }).then(response => JSON.parse(response.SecretString))
+    if (response === undefined || response.SecretString === undefined) {
+        return
+    }
+    const api_keys = JSON.parse(response.SecretString)
 
-    // config = {
-    //     google_email : "louishou@triangleanalytics.com",
-    //     alerts_email : "alerts@triangleanalytics.com",
-    //     region : "us-east-1",
-    //     ...api_keys
-    // }
+    config = {
+        google_email : "louishou@triangleanalytics.com",
+        alerts_email : "alerts@triangleanalytics.com",
+        region : "us-east-1",
+        ...api_keys
+    }
 
 })
 
 describe("Testing Utils", () => {
     describe("Testing Misc", () => {
         test("email", async () => {
-            const utils = new Utils(config)
-            await utils.admin_alert("Testing Admin Alert.")
+            const utils = new TriangleUtils(config)
+            const success = await utils.admin_alert("Testing Admin Alert.")
+            expect(success).toBeTruthy()
         })
         test("sha256", async () => {
-            const utils = new Utils(config)
-            const hash = await utils.sha256("Triangle Analytics")
+            const hash = await UtilsMisc.sha256("Triangle Analytics")
             expect(hash).toEqual("7f6f79a3fdee7c38bb3b0381211575a35ffcfcf5867d6045b07b4181e6f24153")
         })
     })
     describe("Testing DynamoDB", () => {
         test("scan", async () => {
-            const utils = new Utils(config)
+            const utils = new TriangleUtils(config)
             const candidates = await utils.dynamodb.scan("candidates")
             expect(candidates).toBeDefined()
             expect(candidates.length).toBeGreaterThan(3000)
-        })
+        }, 20000)
         test("get; single key", async () => {
-            const utils = new Utils(config)
+            const utils = new TriangleUtils(config)
             const candidate = await utils.dynamodb.get("candidates", { candidate_id : "S6MI00392" })
             expect(candidate).toBeDefined()
             if (candidate === undefined) {
@@ -53,7 +60,7 @@ describe("Testing Utils", () => {
             expect(candidate.fec_name).toBeDefined()
         })
         test("get; composite key", async () => {
-            const utils = new Utils(config)
+            const utils = new TriangleUtils(config)
             const candidate_article = await utils.dynamodb.get("candidate_articles", { candidate_id : "S6MI00392", article_id : "2025-08-1674855ce5e59c2b6a19a165037dffa5627b1cee071def07a614acb1c28d0e2eff" })
             expect(candidate_article).toBeDefined()
             if (candidate_article === undefined) {
@@ -62,7 +69,7 @@ describe("Testing Utils", () => {
             expect(candidate_article.url).toBeDefined()
         })
         test("get_max", async () => {
-            const utils = new Utils(config)
+            const utils = new TriangleUtils(config)
             const candidate_article = await utils.dynamodb.get_max("candidate_articles", { candidate_id : "S6MI00392" })
             expect(candidate_article).toBeDefined()
             if (candidate_article === undefined) {
@@ -73,7 +80,7 @@ describe("Testing Utils", () => {
             expect(candidate_article.article_id.localeCompare("2025-08-1674855ce5e59c2b6a19a165037dffa5627b1cee071def07a614acb1c28d0e2eff") > 0).toBeTruthy()
         })
         test("query", async () => {
-            const utils = new Utils(config)
+            const utils = new TriangleUtils(config)
             const candidate_articles = await utils.dynamodb.query("candidate_articles", { candidate_id : "S6MI00392" })
             expect(candidate_articles).toBeDefined()
             if (candidate_articles === undefined) {
@@ -81,5 +88,109 @@ describe("Testing Utils", () => {
             }
             expect(candidate_articles.length).toBeGreaterThan(400)
         })
+        test("query_prefix", async () => {
+            const utils = new TriangleUtils(config)
+            const candidate_articles = await utils.dynamodb.query_prefix("candidate_articles", { candidate_id : "S6MI00392" }, { article_id : "2025-08-16"})
+            expect(candidate_articles).toBeDefined()
+            if (candidate_articles === undefined) {
+                return
+            }
+            expect(candidate_articles.length).toBeGreaterThan(2)
+            expect(!candidate_articles.map(candidate_article => candidate_article.article_id.substring(0, 10) === "2025-08-16").includes(false)).toBeTruthy()
+        })
+        test("query_range", async () => {
+            const utils = new TriangleUtils(config)
+            const candidate_articles = await utils.dynamodb.query_range("candidate_articles", { candidate_id : "S6MI00392" }, { article_id : ["2025-08-01", "2025-08-31"] })
+            expect(candidate_articles).toBeDefined()
+            if (candidate_articles === undefined) {
+                return
+            }
+            expect(candidate_articles.length).toBeGreaterThan(10)
+            expect(!candidate_articles.map(candidate_article => candidate_article.article_id.substring(0, 7) === "2025-08").includes(false)).toBeTruthy()
+        })
+    })
+    describe("Testing S3", () => {
+        test("exists, get, create, delete, download_url, upload_url", async () => {
+            const utils = new TriangleUtils(config)
+            const s3_filename = "s3://triangleanalytics-raw-scrapes/test_file.txt"
+            await utils.s3.delete_file(s3_filename)
+            const file_exists_0 = await utils.s3.file_exists(s3_filename)
+            expect(file_exists_0).toBeFalsy()
+            await utils.s3.create_file(s3_filename, "test file contents.")
+            const file_exists_1 = await utils.s3.file_exists(s3_filename)
+            expect(file_exists_1).toBeTruthy()
+            const text = await utils.s3.get_file(s3_filename)
+            expect(text).toEqual("test file contents.")
+            const download_url = await utils.s3.generate_download_url(s3_filename)
+            expect(download_url).toBeDefined()
+            if (download_url === undefined) {
+                return
+            }
+            expect(download_url.includes("https://triangleanalytics-raw-scrapes.s3.us-east-1.amazonaws.com/test_file.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&")).toBeDefined()
+            const upload_url = await utils.s3.generate_upload_url(s3_filename)
+            expect(upload_url).toBeDefined()
+            await utils.s3.delete_file(s3_filename)
+            if (upload_url === undefined) {
+                return
+            }
+            expect(upload_url.includes("https://triangleanalytics-raw-scrapes.s3.us-east-1.amazonaws.com/test_file.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&")).toBeDefined()
+            const file_exists_2 = await utils.s3.file_exists(s3_filename)
+            expect(file_exists_2).toBeFalsy()
+
+        })
+    })
+    // describe("Testing Bee", () => {
+    //     test("bee get page text", async () => {
+    //         const utils = new Utils(config)
+    //         const text = await utils.bee.get("https://marieforcongress.com/", { render_js : false, return_page_text : true })
+    //         expect(text).toBeDefined()
+    //         if (text === undefined) {
+    //             return
+    //         }
+    //         expect(text.includes("Marie Gluesenkamp Perez’s working class roots run deep in Washington State.")).toBeTruthy()
+    //     })
+    //     test("bee google search", async () => {
+    //         const utils = new Utils(config)
+    //         const results = await utils.bee.google_search("Marie Gluesenkamp Perez", true)
+    //         expect(results).toBeDefined()
+    //         if (results === undefined) {
+    //             return
+    //         }
+    //         expect(results.length).toBeGreaterThan(1)
+    //         for (const result of results) {
+    //             expect(result.link).toBeDefined()
+    //             expect(result.date).toBeDefined()
+    //             expect(result.title).toBeDefined()
+    //             expect(result.source).toBeDefined()
+    //         }
+    //     })
+    //     test("bee youtube search", async () => {
+    //         const utils = new Utils(config)
+    //         const results = await utils.bee.youtube_search("Marie Gluesenkamp Perez", true)
+    //         expect(results).toBeDefined()
+    //         if (results === undefined) {
+    //             return
+    //         }
+    //         expect(results.length).toBeGreaterThan(1)
+    //         for (const result of results) {
+    //             expect(result.videoId).toBeDefined()
+    //         }
+    //     })
+    // })
+    describe("Testing Youtube", () => {
+        test("batch_get_videos", async () => {
+            const utils = new TriangleUtils(config)
+            const videos = await utils.youtube.batch_get_videos(["N4dyOzjjYYk", "rUyg3P1R5NQ"])
+            expect(videos).toBeDefined()
+            if (videos === undefined) {
+                return
+            }
+            expect(videos.length).toEqual(2)
+            for (const video of videos) {
+                expect(video.snippet !== undefined)
+                expect(video.contentDetails !== undefined)
+            }
+
+        })
     })
 })

package/vite.config.ts

@@ -0,0 +1,10 @@
+import { configDefaults, defineConfig } from "vitest/config"
+
+export default defineConfig({
+    test: {
+        exclude: [
+            ...configDefaults.exclude,
+            "dist/**"
+        ],
+    },
+})

package/dist/src/Utils.d.ts

@@ -1,17 +0,0 @@
-import { Utils_Misc } from "./Utils_Misc.js";
-import { Utils_DynamoDB } from "./Utils_DynamoDB.js";
-import { Utils_S3 } from "./Utils_S3.js";
-import { Utils_Bedrock } from "./Utils_Bedrock.js";
-import { Utils_Bee } from "./Utils_Bee.js";
-import { Utils_S3Vectors } from "./Utils_S3Vectors.js";
-import { Utils_Youtube } from "./Utils_Youtube.js";
-import { Config } from "./types/Config.js";
-export declare class Utils extends Utils_Misc {
-    readonly dynamodb: Utils_DynamoDB;
-    readonly s3: Utils_S3;
-    readonly s3vectors: Utils_S3Vectors;
-    readonly bedrock: Utils_Bedrock;
-    readonly bee: Utils_Bee;
-    readonly youtube: Utils_Youtube;
-    constructor(config: Config);
-}

package/dist/src/Utils.js

@@ -1,24 +0,0 @@
-import { Utils_Misc } from "./Utils_Misc.js";
-import { Utils_DynamoDB } from "./Utils_DynamoDB.js";
-import { Utils_S3 } from "./Utils_S3.js";
-import { Utils_Bedrock } from "./Utils_Bedrock.js";
-import { Utils_Bee } from "./Utils_Bee.js";
-import { Utils_S3Vectors } from "./Utils_S3Vectors.js";
-import { Utils_Youtube } from "./Utils_Youtube.js";
-export class Utils extends Utils_Misc {
-    dynamodb;
-    s3;
-    s3vectors;
-    bedrock;
-    bee;
-    youtube;
-    constructor(config) {
-        super(config);
-        this.dynamodb = new Utils_DynamoDB(config);
-        this.s3 = new Utils_S3(config);
-        this.s3vectors = new Utils_S3Vectors(config);
-        this.bedrock = new Utils_Bedrock(config);
-        this.bee = new Utils_Bee(config);
-        this.youtube = new Utils_Youtube(config);
-    }
-}

package/dist/src/Utils_Bedrock.d.ts

@@ -1,10 +0,0 @@
-import { BedrockRuntime } from "@aws-sdk/client-bedrock-runtime";
-import { Config } from "./types/Config.js";
-export declare class Utils_Bedrock {
-    readonly bedrock: BedrockRuntime;
-    readonly text_decoder: TextDecoder;
-    constructor(config: Config);
-    llama_invoke(model_id: string, prompt: string, temperature?: number, max_gen_len?: number, top_p?: number): Promise<string | undefined>;
-    gpt_converse(model_id: string, prompt: string, temperature?: number, max_gen_len?: number, top_p?: number): Promise<string | undefined>;
-    titan_invoke(text: string): Promise<string | undefined>;
-}

package/dist/src/Utils_Bee.d.ts

@@ -1,11 +0,0 @@
-import { ScrapingBeeClient } from "scrapingbee";
-import { Config } from "./types/Config.js";
-export declare class Utils_Bee {
-    readonly config: Config;
-    readonly bee: ScrapingBeeClient | undefined;
-    readonly text_decoder: TextDecoder;
-    constructor(config: Config);
-    get(url: string, params?: Record<string, string>): Promise<string | undefined>;
-    google_search(query: string, news?: boolean): Promise<any>;
-    youtube_search(query: string, options?: {}): Promise<any>;
-}

package/dist/src/Utils_Misc.d.ts

@@ -1,30 +0,0 @@
-import * as nodemailer from "nodemailer";
-import { Config } from "./types/Config.js";
-interface Encryption {
-    iv: string;
-    ciphertext: string;
-}
-export declare class Utils_Misc {
-    readonly config: Config;
-    readonly text_encoder: TextEncoder;
-    readonly transporter: nodemailer.Transporter<import("nodemailer/lib/smtp-transport").SentMessageInfo, import("nodemailer/lib/smtp-transport").Options> | undefined;
-    constructor(config: Config);
-    wait(duration: number): Promise<unknown>;
-    get_current_time(): string;
-    get_current_milliseconds(): number;
-    send_email(recipient: string, subject: string, text: string): Promise<void>;
-    admin_alert(text: string): Promise<void>;
-    safe_run(f: () => Promise<any>): Promise<void>;
-    iterate<T>(inputs: T[], f: (input: T) => Promise<any>, concurrency?: number, print_indices?: boolean): Promise<void>;
-    sha256(input: string): Promise<string>;
-    encrypt(text: string): Encryption;
-    decrypt(encryption: Encryption): string;
-    get_secret_hash(username: string): string;
-    get_election_id(year: string, office: string, state: string, district: string): string | undefined;
-    get_chunk_indices(text_length: number, max_length?: number, overlap?: number): [number, number][];
-    chunkify(text: string, max_length?: number, overlap?: number): {
-        chunk_index: [number, number];
-        chunk_text: string;
-    }[];
-}
-export {};