triangle-utils 1.3.4 → 1.4.1

package/dist/src/index.js

@@ -0,0 +1,36 @@
+import { UtilsDynamoDB } from "./UtilsDynamoDB.js";
+import { UtilsS3 } from "./UtilsS3.js";
+import { UtilsBedrock } from "./UtilsBedrock.js";
+import { UtilsBee } from "./UtilsBee.js";
+import { UtilsS3Vectors } from "./UtilsS3Vectors.js";
+import { UtilsCognito } from "./UtilsCognito.js";
+import { UtilsMisc } from "./UtilsMisc.js";
+import { UtilsYoutube } from "./UtilsYoutube.js";
+export class TriangleUtils extends UtilsMisc {
+    dynamodb;
+    s3;
+    s3vectors;
+    bedrock;
+    cognito;
+    bee;
+    youtube;
+    constructor(config) {
+        super(config);
+        this.dynamodb = new UtilsDynamoDB(config.region);
+        this.s3 = new UtilsS3(config.region);
+        this.s3vectors = new UtilsS3Vectors(config.region);
+        this.bedrock = new UtilsBedrock(config.region);
+        this.cognito = new UtilsCognito(config.region);
+        this.bee = new UtilsBee(config.scraping_bee_api_key);
+        this.youtube = new UtilsYoutube(config.youtube_api_key);
+    }
+}
+export * from "./types/TriangleUtilsConfig.js";
+export * from "./UtilsMisc.js";
+export * from "./UtilsDynamoDB.js";
+export * from "./UtilsS3.js";
+export * from "./UtilsBedrock.js";
+export * from "./UtilsBee.js";
+export * from "./UtilsS3Vectors.js";
+export * from "./UtilsCognito.js";
+export * from "./UtilsYoutube.js";

package/dist/src/types/{Config.d.ts → TriangleUtilsConfig.d.ts}

@@ -1,8 +1,6 @@
-export interface Config {
+export interface TriangleUtilsConfig {
     region: string;
-    google_app_password?: string;
     scraping_bee_api_key?: string;
     youtube_api_key?: string;
-    citizenportal_api_key?: string;
     [key: string]: any;
 }

package/package.json

@@ -1,12 +1,12 @@
 {
     "name": "triangle-utils",
-    "version": "1.3.4",
-    "main": "dist/src/Utils.js",
+    "version": "1.4.1",
+    "main": "dist/src/index.js",
     "directories": {
-        "test": "test"
+        "test": "vitest"
     },
     "scripts": {
-        "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js",
+        "test": "vitest",
         "build": "tsc && tsc-alias"
     },
     "author": "",
@@ -15,6 +15,7 @@
     "type": "module",
     "dependencies": {
         "@aws-sdk/client-bedrock-runtime": "^3.953.0",
+        "@aws-sdk/client-cognito-identity-provider": "^3.1004.0",
         "@aws-sdk/client-dynamodb": "^3.953.0",
         "@aws-sdk/client-s3": "^3.953.0",
         "@aws-sdk/client-s3vectors": "^3.953.0",
@@ -24,20 +25,19 @@
         "@types/nodemailer": "^7.0.9",
         "googleapis": "^170.0.0",
         "nodemailer": "^7.0.11",
-        "scrapingbee": "^1.8.2",
-        "triangle-utils": "^1.0.34",
-        "tsc-alias": "^1.8.16"
+        "scrapingbee": "^1.8.2"
     },
     "devDependencies": {
         "@eslint/js": "^9.39.2",
         "@types/jest": "^30.0.0",
         "eslint": "^9.39.2",
         "globals": "^17.3.0",
-        "jest": "^30.2.0",
         "jiti": "^2.6.1",
         "ts-jest": "^29.4.6",
+        "tsc-alias": "^1.8.16",
         "tsx": "^4.21.0",
         "typescript": "^5.9.3",
-        "typescript-eslint": "^8.55.0"
+        "typescript-eslint": "^8.55.0",
+        "vitest": "^4.0.18"
     }
 }

package/src/{Utils_Bedrock.ts → UtilsBedrock.ts}

@@ -1,13 +1,12 @@
 import { BedrockRuntime } from "@aws-sdk/client-bedrock-runtime"
-import { Config } from "./types/Config"
 
-export class Utils_Bedrock {
+export class UtilsBedrock {
 
-    readonly bedrock : BedrockRuntime
-    readonly text_decoder : TextDecoder
+    private readonly bedrock : BedrockRuntime
+    private readonly text_decoder : TextDecoder
 
-    constructor(config : Config) {
-        this.bedrock = new BedrockRuntime({ region: config.region })
+    constructor(region : string) {
+        this.bedrock = new BedrockRuntime({ region: region })
         this.text_decoder = new TextDecoder()
     }
 
@@ -68,7 +67,7 @@ export class Utils_Bedrock {
         return undefined
     }
 
-    async titan_invoke(text : string) : Promise<string | undefined> {
+    async titan_invoke(text : string) : Promise<number[] | undefined> {
         for (let i = 0; i < 3; i++) {
             try {
                 const output = await this.bedrock.invokeModel({
@@ -79,7 +78,10 @@ export class Utils_Bedrock {
                     contentType : "application/json"
                 })
                 const response = JSON.parse(this.text_decoder.decode(output.body))
-                return response.embedding
+                if (response.embedding !== undefined && Array.isArray(response.embedding) && response.embedding.every((i : any) => typeof i === "number")) {
+                    const embedding : number[] = response.embedding
+                    return embedding
+                }
             } catch (error) {
                 if (error instanceof Error) {
                     console.log(error.stack)

package/src/{Utils_Bee.ts → UtilsBee.ts}

@@ -1,25 +1,22 @@
 import { ScrapingBeeClient } from "scrapingbee"
-import { Config } from "./types/Config"
 
-export class Utils_Bee {
+export class UtilsBee {
 
-    readonly config : Config
-    readonly bee : ScrapingBeeClient | undefined
-    readonly text_decoder : TextDecoder
+    private readonly bee : ScrapingBeeClient | undefined
+    private readonly text_decoder : TextDecoder
 
-    constructor(config : Config) {
-        this.config = config
-        this.bee = config.scraping_bee_api_key !== undefined ? new ScrapingBeeClient(config.scraping_bee_api_key) : undefined
+    constructor(scraping_bee_api_key : string | undefined) {
+        this.bee = scraping_bee_api_key !== undefined ? new ScrapingBeeClient(scraping_bee_api_key) : undefined
         this.text_decoder = new TextDecoder()
     }
 
-    async get(url : string, params : Record<string, string> = {}) {
+    async get(url : string, params : Record<string, string | number | boolean> = {}) {
         if (this.bee === undefined) {
             return undefined
         }
         for (let i = 0; i < 3; i++) {
             try {
-                const response = await this.bee.get({
+                const response = await this.bee.htmlApi({
                     url : url,
                     params : params
                 })
@@ -40,7 +37,7 @@ export class Utils_Bee {
         return undefined
     }
 
-    async google_search(query : string, news : boolean = false) {
+    async google_search(query : string, news : boolean = false) : Promise<Record<string, any>[] | undefined> {
         if (this.bee === undefined) {
             return undefined
         }
@@ -56,13 +53,22 @@ export class Utils_Bee {
                     console.log("Failed to Google:", response.status)
                     continue
                 }
-                const data = response.data
-                const results = news ? data.news_results : data.organic_results
+                const data : Record<string, any> = JSON.parse(response.data.toString("utf-8"))
+                
+                if (data.news_results === undefined || data.organic_results === undefined) {
+                    continue
+                }
+                const results : Record<string, any>[] | undefined = news ? data.news_results : data.organic_results
+
 
                 if (results === undefined) {
                     console.log("Failed to Google.")
                     continue
                 }
+                if (!Array.isArray(results)) {
+                    console.log("Results are not an array.")
+                    return
+                }
                 return results
             } catch {
                 console.log("Failed to Google.")
@@ -72,7 +78,7 @@ export class Utils_Bee {
         return undefined
     }
 
-    async youtube_search(query : string, options = {}) {
+    async youtube_search(query : string, options = {}) : Promise<Record<string, any>[] | undefined> {
         if (this.bee === undefined) {
             return undefined
         }
@@ -86,7 +92,7 @@ export class Utils_Bee {
                     console.log("Failed to YouTube Search:", response.status)
                     continue
                 }
-                const data = response.data
+                const data : Record<string, any> = JSON.parse(response.data.toString("utf-8"))
                 const results = data.results
 
                 if (results === undefined) {

package/src/UtilsCognito.ts

@@ -0,0 +1,116 @@
+import { CognitoIdentityProvider, NotAuthorizedException } from "@aws-sdk/client-cognito-identity-provider"
+
+export class UtilsCognito {
+
+    private readonly cognito : CognitoIdentityProvider
+
+    constructor(region : string) {
+        this.cognito = new CognitoIdentityProvider({ region : region })
+    }
+
+    async admin_get_user(user_pool_id : string, username : string) {
+        try {
+            return await this.cognito.adminGetUser({
+                UserPoolId : user_pool_id,
+                Username : username
+            })
+        } catch (error) {
+            return undefined
+        }
+    }
+
+    async admin_create_user(user_pool_id : string, username : string, user_id : string) {
+        try {
+            return await this.cognito.adminCreateUser({
+                UserPoolId : user_pool_id,
+                Username : username,
+                UserAttributes : [
+                    { Name : "preferred_username", Value : user_id },
+                    { Name : "email_verified", Value : "true" }
+                ]
+            })
+        } catch (error) {
+            if (error instanceof Error) {
+                return new Error(error.toString().replace("Exception: ", ""))
+            }
+            return new Error()
+        }
+    }
+
+    async admin_reset_password(user_pool_id : string, username : string) {
+        const user = await this.admin_get_user(user_pool_id, username)
+        if (user === undefined) {
+            return new Error("User does not exist.")
+        }
+        try {
+            return await this.cognito.adminCreateUser({
+                UserPoolId : user_pool_id,
+                Username : username,
+                MessageAction : "RESEND"
+            })
+        } catch (error) {
+            if (error instanceof Error) {
+                return new Error(error.toString().replace("Exception: ", ""))
+            }
+            return new Error()
+        }
+    }
+
+    async get_user_id(token : string) : Promise<string | undefined> {
+        try {
+            const user_info = await this.cognito.getUser({
+                AccessToken : token
+            })
+            const user_attributes = user_info.UserAttributes
+            if (user_attributes === undefined) {
+                return undefined
+            }
+            const user_attribute = user_attributes.filter(attribute => attribute.Name === "preferred_username")[0]
+            const user_id = user_attribute.Value
+            return user_id
+        } catch (error) {
+            if (error instanceof NotAuthorizedException) {
+                return undefined
+            }
+            throw error
+        }
+    }
+
+    async get_new_token(client_id : string, client_secret : string, refresh_token : string) : Promise<string | undefined> {
+        try {
+            const response = await this.cognito.getTokensFromRefreshToken({
+                ClientId : client_id,
+                ClientSecret : client_secret,
+                RefreshToken : refresh_token
+            })
+            const new_auth_result = response.AuthenticationResult
+            if (new_auth_result === undefined) {
+                return undefined
+            }
+            const new_token = new_auth_result.AccessToken
+            return new_token
+        } catch (error) {
+            return undefined
+        }
+    }
+
+    async sign_out(token : string) {
+        await this.cognito.globalSignOut({
+            AccessToken : token
+        })
+    }
+
+    async admin_disable_user(user_pool_id : string, username : string) : Promise<void> {
+        await this.cognito.adminDisableUser({
+            UserPoolId : user_pool_id,
+            Username : username,
+        })
+    }
+
+    async admin_delete_user(user_pool_id : string, username : string) : Promise<void> {
+        await this.cognito.adminDeleteUser({
+            UserPoolId : user_pool_id,
+            Username : username,
+        })
+    }
+}

package/src/{Utils_DynamoDB.ts → UtilsDynamoDB.ts}

@@ -1,19 +1,18 @@
 import { AttributeValue, DynamoDB, QueryCommandInput, QueryCommandOutput, ScanCommandInput, ScanCommandOutput, UpdateItemCommandInput } from "@aws-sdk/client-dynamodb"
-import { Config } from "./types/Config"
 
-function convert_output(dynamoobject : AttributeValue) : string | boolean | number | any[] | Set<any> | Record<string, any> | undefined {
-    if (dynamoobject.S !== undefined) {
-        return dynamoobject.S
-    } else if (dynamoobject.N !== undefined) {
-        return Number(dynamoobject.N)
-    } else if (dynamoobject.L !== undefined) {
-        return dynamoobject.L.map((a : AttributeValue) => convert_output(a))
-    } else if (dynamoobject.SS !== undefined) {
-        return new Set(dynamoobject.SS)
-    } else if (dynamoobject.M !== undefined) {
-        return Object.fromEntries(Object.entries(dynamoobject.M).map(([key, value]) => [key, convert_output(value)]))
-    } else if (dynamoobject.BOOL !== undefined) {
-        return dynamoobject.BOOL
+function convert_output(dynamodb_output : AttributeValue) : string | boolean | number | any[] | Set<any> | Record<string, any> | undefined {
+    if (dynamodb_output.S !== undefined) {
+        return dynamodb_output.S
+    } else if (dynamodb_output.N !== undefined) {
+        return Number(dynamodb_output.N)
+    } else if (dynamodb_output.L !== undefined) {
+        return dynamodb_output.L.map((a : AttributeValue) => convert_output(a))
+    } else if (dynamodb_output.SS !== undefined) {
+        return new Set(dynamodb_output.SS)
+    } else if (dynamodb_output.M !== undefined) {
+        return Object.fromEntries(Object.entries(dynamodb_output.M).map(([key, value]) => [key, convert_output(value)]))
+    } else if (dynamodb_output.BOOL !== undefined) {
+        return dynamodb_output.BOOL
     }
     return undefined
 }
@@ -84,12 +83,12 @@ async function compile_pages(
     }
 }
 
-export class Utils_DynamoDB {
+export class UtilsDynamoDB {
 
-    readonly dynamodb : DynamoDB
+    private readonly dynamodb : DynamoDB
 
-    constructor(config : Config) {
-        this.dynamodb = new DynamoDB({ region : config.region })
+    constructor(region : string) {
+        this.dynamodb = new DynamoDB({ region : region })
     }
 
     async scan(
@@ -241,16 +240,16 @@ export class Utils_DynamoDB {
             reverse? : boolean,
             compile? : boolean
         } = {}
-    ) : Promise<Record<string, any>[] | undefined>{
+    ) : Promise<Record<string, any>[]>{
         const reverse = options.reverse !== undefined ? options.reverse : false
         const compile = options.compile !== undefined ? options.compile : true
         if (Object.keys(primary_key).length !== 1 || Object.keys(secondary_key_prefix).length !== 1) {
-            return undefined
+            return []
         }
         const converted_primary_value = convert_input(Object.values(primary_key)[0])
         const converted_secondary_prefix_value = convert_input(Object.values(secondary_key_prefix)[0])
         if (converted_primary_value === undefined || converted_secondary_prefix_value === undefined) {
-            return undefined
+            return []
         }
         const request : QueryCommandInput = {
             TableName : table,
@@ -271,7 +270,7 @@ export class Utils_DynamoDB {
     async query_range(
         table : string,
         primary_key : Record<string, any>,
-        secondary_key_range : Record<string, string|number[]>,
+        secondary_key_range : Record<string, (string|number)[]>,
         options : {
             reverse? : boolean,
             compile? : boolean
@@ -388,7 +387,7 @@ export class Utils_DynamoDB {
         await this.dynamodb.updateItem(request)
     }
 
-    async create(table : string, key : Record<string, any>, attributes : Record<string, any[]> = {}) : Promise<void> {
+    async create(table : string, key : Record<string, any>, attributes : Record<string, any> = {}) : Promise<void> {
         const item = await this.get(table, key, true)
         if (item !== undefined) {
             return

package/src/UtilsMisc.ts

@@ -0,0 +1,188 @@
+import * as nodemailer from "nodemailer"
+import * as crypto from "crypto"
+
+import { TriangleUtilsConfig } from "./types/TriangleUtilsConfig"
+
+interface Encryption {
+    iv : string,
+    ciphertext : string
+}
+
+export class UtilsMisc {
+
+    readonly config : TriangleUtilsConfig
+    readonly transporter
+
+    constructor(config : TriangleUtilsConfig) {
+        this.config = config
+        if (config.google_email !== undefined && config.google_app_password !== undefined) {
+            this.transporter = nodemailer.createTransport({
+                service: "Gmail",
+                host: "smtp.gmail.com",
+                port: 465,
+                secure: true,
+                auth: {
+                    user: config.google_email,
+                    pass: config.google_app_password,
+                }
+            })
+        }
+    }
+
+    async send_email(recipient : string, subject : string, text : string) : Promise<boolean> {
+        if (this.transporter === undefined) {
+            return false
+        }
+        for (let i = 0; i < 3; i++) {
+            try {
+                await this.transporter.sendMail({
+                    from: this.config.alerts_email,
+                    to: recipient,
+                    subject: subject,
+                    text: text
+                })
+                return true
+            } catch (error) {
+                console.log("EMAIL ERROR", error)
+            }
+        }
+        return false
+    }
+
+    async admin_alert(text : string) : Promise<boolean> {
+        console.log("ADMIN ALERT:", text)
+        return await this.send_email(this.config.google_email, "ADMIN ALERT", text)
+    }
+
+    async safe_run(f : () => Promise<any>) : Promise<void> {
+        try {
+            await f()
+        } catch (error) {
+            if (!(error instanceof Error)) {
+                return
+            }
+            if (error.stack !== undefined) {
+                await this.admin_alert(error.stack.toString())
+            } else {
+                await this.admin_alert(error.toString())
+            }
+        }
+    }
+
+    async iterate<T>(inputs : T[], f : (input : T) => Promise<any>, concurrency : number = 1, print_indices : boolean = false) {
+        let index = 0
+        let iterators = []
+        for (let i = 0; i < concurrency; i++) {
+            iterators.push((async () => {
+                while (index < inputs.length) {
+                    index += 1
+                    if (print_indices) {
+                        console.log(i + ":" + (index - 1) + "/" + inputs.length)
+                    }
+                    await this.safe_run(() => f(inputs[index - 1]))
+                }
+            })())
+        }
+        await Promise.all(iterators)
+    }
+
+
+    static async wait(duration : number) {
+        return new Promise(resolve => setTimeout(resolve, duration))
+    }
+
+    static get_current_time() {
+        return (new Date()).toISOString()
+    }
+
+    static get_current_milliseconds() {
+        return (new Date()).getTime()
+    }
+
+    static async sha256(input : string) : Promise<string> {
+        const text_encoder = new TextEncoder()
+        const input_buffer = text_encoder.encode(input);                   
+
+        const hash_buffer = await crypto.subtle.digest("SHA-256", input_buffer)
+
+        const hash_array = Array.from(new Uint8Array(hash_buffer))
+                
+        const hash = hash_array.map(b => b.toString(16).padStart(2, "0")).join("")
+        return hash
+    }
+
+    static encrypt(secret_key : string, text : string) : Encryption {
+        const iv = crypto.randomBytes(16)
+        const cipher = crypto.createCipheriv("aes-256-cbc", Buffer.from(secret_key, "hex"), iv)
+        let ciphertext = cipher.update(text, "utf8", "hex")
+        ciphertext += cipher.final("hex")
+        return {
+            iv: iv.toString("hex"),
+            ciphertext: ciphertext
+        }
+    }
+
+    static decrypt(secret_key : string, encryption : Encryption) : string {
+        const iv = encryption.iv
+        const ciphertext = encryption.ciphertext
+        const decipher = crypto.createDecipheriv("aes-256-cbc", Buffer.from(secret_key, "hex"), Buffer.from(iv, "hex"))
+        let text = decipher.update(ciphertext, "hex", "utf8")
+        text += decipher.final("utf8")
+        return text
+    }
+
+    static get_secret_hash(username : string, cognito_client_id : string, cognito_client_secret : string) : string {
+        const secret_hash = crypto.createHmac("sha256", cognito_client_secret)
+            .update(username + cognito_client_id)
+            .digest("base64")
+        return secret_hash
+    }
+
+    static get_chunk_indices(text_length : number, max_length = 2500, overlap = 50) : [number, number][] {
+        const num_chunks = Math.ceil((text_length + overlap) / max_length)
+        const chunk_length = Math.ceil((text_length - overlap) / num_chunks + overlap)
+        const chunk_indices : [number, number][] = []
+        for (let i = 0; i < num_chunks; i++) {
+            chunk_indices.push([
+                (chunk_length - overlap) * i,
+                (chunk_length - overlap) * i + chunk_length
+            ])
+        }
+        return chunk_indices
+    }
+
+    static chunkify(text : string, max_length = 2500, overlap = 50) : {
+        chunk_index : [number, number],
+        chunk_text : string
+    }[] {
+        const chunk_indices = this.get_chunk_indices(text.length, max_length, overlap)
+        const chunks =  chunk_indices.map(chunk_index => ({
+            chunk_index : chunk_index,
+            chunk_text : text.substring(...chunk_index)
+        }))
+        return chunks
+    }
+    
+    static chunkify_lined(text : string, max_length = 2500, overlap = 50) : {
+        chunk_index : [number, number],
+        chunk_text : string
+    }[] {
+        const chunk_indices : [number, number][] = this.get_chunk_indices(text.length, max_length, overlap)
+        const lined_chunk_indices : [number, number][] = []
+        for (const chunk_index of chunk_indices) {
+            const lined_chunk_index : [number, number] = [
+                text.lastIndexOf("\n", chunk_index[0]) + 1,
+                text.indexOf("\n", chunk_index[1]) + 1
+            ]
+            if (lined_chunk_index[1] < lined_chunk_index[0]) {
+                lined_chunk_index[1] += text.length
+            }
+            lined_chunk_indices.push(lined_chunk_index)
+        }
+        return lined_chunk_indices.map(chunk_index => ({
+            chunk_index : chunk_index,
+            chunk_text : text.substring(...chunk_index)
+        }))
+    }
+
+}

package/src/{Utils_S3.ts → UtilsS3.ts}

@@ -1,6 +1,5 @@
 import { S3, NoSuchKey, GetObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3"
 import { getSignedUrl } from "@aws-sdk/s3-request-presigner"
-import { Config } from "./types/Config"
 
 interface S3Key {
     Bucket : string,
@@ -17,12 +16,12 @@ function parse_s3_filename(s3_filename : string) : S3Key | undefined {
     return { Bucket : bucket, Key : filename }
 }
 
-export class Utils_S3 {
+export class UtilsS3 {
 
-    readonly s3 : S3
+    private readonly s3 : S3
 
-    constructor(config : Config) {
-        this.s3 = new S3({ region : config.region })
+    constructor(region : string) {
+        this.s3 = new S3({ region : region })
     }
 
     async file_exists(s3_filename : string) {

package/src/{Utils_S3Vectors.ts → UtilsS3Vectors.ts}

@@ -1,18 +1,17 @@
 import { ListVectorsCommandInput, S3Vectors } from "@aws-sdk/client-s3vectors"
-import { Config } from "./types/Config"
 
-interface Vector {
+export interface Vector {
     key : string,
     data : number[],
     metadata : any
 }
 
-export class Utils_S3Vectors {
+export class UtilsS3Vectors {
 
-    readonly s3vectors : S3Vectors
+    private readonly s3vectors : S3Vectors
 
-    constructor(config : Config) {
-        this.s3vectors = new S3Vectors({ region : config.region })
+    constructor(region : string) {
+        this.s3vectors = new S3Vectors({ region : region })
     }
 
     async scan(vector_bucket : string, vector_index : string) : Promise<string[]> {
@@ -134,7 +133,7 @@ export class Utils_S3Vectors {
         }
     }
 
-    async query(vector_bucket : string, vector_index : string, data : number[], num_vectors : number, filters : Record<string, any> = {}) {
+    async query(vector_bucket : string, vector_index : string, data : number[], num_vectors : number, filters : Record<string, any> | undefined = undefined) {
         const response = await this.s3vectors.queryVectors({
             vectorBucketName : vector_bucket,
             indexName : vector_index,