triangle-utils 1.2.12 → 1.3.1

package/dist/src/Utils.js

@@ -0,0 +1,24 @@
+import { Utils_Misc } from "./Utils_Misc";
+import { Utils_DynamoDB } from "./Utils_DynamoDB";
+import { Utils_S3 } from "./Utils_S3";
+import { Utils_Bedrock } from "./Utils_Bedrock";
+import { Utils_Bee } from "./Utils_Bee";
+import { Utils_S3Vectors } from "./Utils_S3Vectors";
+import { Utils_Youtube } from "./Utils_Youtube";
+export class Utils extends Utils_Misc {
+    dynamodb;
+    s3;
+    s3vectors;
+    bedrock;
+    bee;
+    youtube;
+    constructor(config) {
+        super(config);
+        this.dynamodb = new Utils_DynamoDB(config);
+        this.s3 = new Utils_S3(config);
+        this.s3vectors = new Utils_S3Vectors(config);
+        this.bedrock = new Utils_Bedrock(config);
+        this.bee = new Utils_Bee(config);
+        this.youtube = new Utils_Youtube(config);
+    }
+}

package/dist/src/Utils_Bedrock.js

@@ -0,0 +1,89 @@
+import { BedrockRuntime } from "@aws-sdk/client-bedrock-runtime";
+export class Utils_Bedrock {
+    bedrock;
+    text_decoder;
+    constructor(config) {
+        this.bedrock = new BedrockRuntime({ region: config.region });
+        this.text_decoder = new TextDecoder();
+    }
+    async llama_invoke(model_id, prompt, temperature = 0.5, max_gen_len = 512, top_p = 0.9) {
+        for (let i = 0; i < 3; i++) {
+            try {
+                const output = await this.bedrock.invokeModel({
+                    modelId: model_id,
+                    body: JSON.stringify({
+                        prompt: prompt,
+                        temperature: temperature,
+                        top_p: top_p,
+                        max_gen_len: max_gen_len
+                    }),
+                    contentType: "application/json"
+                });
+                const response = JSON.parse(this.text_decoder.decode(output.body));
+                return response.generation;
+            }
+            catch (e) {
+                console.log("Failed to get from Bedrock.");
+            }
+        }
+        console.log("Failed to get from Bedrock, quitting.");
+        return undefined;
+    }
+    async gpt_converse(model_id, prompt, temperature = 0.5, max_gen_len = 512, top_p = 0.9) {
+        for (let i = 0; i < 3; i++) {
+            try {
+                const response = await this.bedrock.converse({
+                    modelId: model_id,
+                    messages: [
+                        { role: "user", content: [{ text: prompt }] }
+                    ],
+                    inferenceConfig: {
+                        temperature: temperature,
+                        topP: top_p,
+                        maxTokens: max_gen_len
+                    }
+                });
+                const output = response.output;
+                if (output === undefined || output.message === undefined || output.message.content === undefined) {
+                    continue;
+                }
+                for (const c of output.message.content) {
+                    if (c.text !== undefined) {
+                        return c.text;
+                    }
+                }
+            }
+            catch (error) {
+                if (error instanceof Error) {
+                    console.log(error.stack);
+                }
+                console.log("Failed to get from Bedrock.");
+            }
+        }
+        console.log("Failed to get from Bedrock, quitting.");
+        return undefined;
+    }
+    async titan_invoke(text) {
+        for (let i = 0; i < 3; i++) {
+            try {
+                const output = await this.bedrock.invokeModel({
+                    modelId: "amazon.titan-embed-text-v2:0",
+                    body: JSON.stringify({
+                        inputText: text
+                    }),
+                    contentType: "application/json"
+                });
+                const response = JSON.parse(this.text_decoder.decode(output.body));
+                return response.embedding;
+            }
+            catch (error) {
+                if (error instanceof Error) {
+                    console.log(error.stack);
+                }
+                console.log("Failed to get from Bedrock.");
+            }
+        }
+        console.log("Failed to get from Bedrock, quitting.");
+        return undefined;
+    }
+}

package/dist/src/Utils_Bee.js

@@ -0,0 +1,101 @@
+import { ScrapingBeeClient } from "scrapingbee";
+export class Utils_Bee {
+    config;
+    bee;
+    text_decoder;
+    constructor(config) {
+        this.config = config;
+        this.bee = config.scraping_bee_api_key !== undefined ? new ScrapingBeeClient(config.scraping_bee_api_key) : undefined;
+        this.text_decoder = new TextDecoder();
+    }
+    async get(url, params = {}) {
+        if (this.bee === undefined) {
+            return undefined;
+        }
+        for (let i = 0; i < 3; i++) {
+            try {
+                const response = await this.bee.get({
+                    url: url,
+                    params: params
+                });
+                if (response.status !== 200) {
+                    console.log("Status", response.status);
+                    return undefined;
+                }
+                const text = this.text_decoder.decode(response.data);
+                return text;
+            }
+            catch (error) {
+                if (error instanceof Error) {
+                    console.log(error.stack);
+                }
+                console.log("Failed to get from Scraping Bee.");
+            }
+        }
+        console.log("Failed to get from Scraping Bee, quitting.");
+        return undefined;
+    }
+    async google_search(query, news = false) {
+        if (this.bee === undefined) {
+            return undefined;
+        }
+        for (let i = 0; i < 3; i++) {
+            try {
+                const response = await this.bee.googleSearch({
+                    search: query,
+                    params: {
+                        search_type: news ? "news" : undefined
+                    }
+                });
+                if (response.status !== 200) {
+                    console.log("Failed to Google:", response.status);
+                    continue;
+                }
+                const data = response.data;
+                const results = news ? data.news_results : data.organic_results;
+                if (results === undefined) {
+                    console.log("Failed to Google.");
+                    continue;
+                }
+                return results;
+            }
+            catch {
+                console.log("Failed to Google.");
+            }
+        }
+        console.log("Failed to Google thrice, quitting.");
+        return undefined;
+    }
+    async youtube_search(query, options = {}) {
+        if (this.bee === undefined) {
+            return undefined;
+        }
+        for (let i = 0; i < 3; i++) {
+            try {
+                const response = await this.bee.youtubeSearch({
+                    search: query,
+                    params: options
+                });
+                if (response.status !== 200) {
+                    console.log("Failed to YouTube Search:", response.status);
+                    continue;
+                }
+                const data = response.data;
+                const results = data.results;
+                if (results === undefined) {
+                    console.log("Failed to YouTube Search.");
+                    continue;
+                }
+                if (Array.isArray(results)) {
+                    return results;
+                }
+                return JSON.parse(results);
+            }
+            catch {
+                console.log("Failed to YouTube Search.");
+            }
+        }
+        console.log("Failed to YouTube Search thrice, quitting.");
+        return undefined;
+    }
+}

package/dist/src/Utils_DynamoDB.js

@@ -0,0 +1,381 @@
+import { DynamoDB } from "@aws-sdk/client-dynamodb";
+function convert_output(dynamoobject) {
+    if (dynamoobject.S !== undefined) {
+        return dynamoobject.S;
+    }
+    else if (dynamoobject.N !== undefined) {
+        return Number(dynamoobject.N);
+    }
+    else if (dynamoobject.L !== undefined) {
+        return dynamoobject.L.map((a) => convert_output(a));
+    }
+    else if (dynamoobject.SS !== undefined) {
+        return new Set(dynamoobject.SS);
+    }
+    else if (dynamoobject.M !== undefined) {
+        return Object.fromEntries(Object.entries(dynamoobject.M).map(([key, value]) => [key, convert_output(value)]));
+    }
+    else if (dynamoobject.BOOL !== undefined) {
+        return dynamoobject.BOOL;
+    }
+    return undefined;
+}
+function not_undefined(x) {
+    return x !== undefined;
+}
+function is_item(x) {
+    return typeof x === "object" && Object.keys(x).filter(key => typeof key !== "string").length === 0 && !Array.isArray(x);
+}
+function convert_input(input) {
+    if (typeof input === "string") {
+        return { S: input };
+    }
+    else if (typeof input === "boolean") {
+        return { BOOL: input };
+    }
+    else if (typeof input === "number") {
+        return { N: input.toString() };
+    }
+    else if (Array.isArray(input)) {
+        const converted_list = input.map((a) => convert_input(a))
+            .filter(converted_input => converted_input !== undefined);
+        if (converted_list.length !== input.length) {
+            return undefined;
+        }
+        return { L: converted_list };
+    }
+    else if (input instanceof Set) {
+        const converted_list = Array.from(input)
+            .filter(converted_input => typeof converted_input === "string");
+        if (converted_list.length !== input.size) {
+            return undefined;
+        }
+        return { SS: converted_list };
+    }
+    else {
+        const converted_inputs = Object.fromEntries(Object.entries(input)
+            .filter(([key, value]) => value !== undefined && value !== null && key !== "")
+            .map(([key, value]) => [key, convert_input(value)])
+            .filter(([key, value]) => not_undefined(value)));
+        return {
+            M: converted_inputs
+        };
+    }
+}
+async function compile_pages(request, f, compile = true) {
+    const items = [];
+    let last_eval_key = undefined;
+    while (true) {
+        const request_page = {
+            ...request,
+            ExclusiveStartKey: last_eval_key
+        };
+        const response = await f(request_page);
+        if (response.Items === undefined) {
+            return [];
+        }
+        const new_items = response.Items.map(item => convert_output({ M: item }));
+        items.push(...new_items);
+        if (response.LastEvaluatedKey === undefined || !compile) {
+            return items;
+        }
+        last_eval_key = response.LastEvaluatedKey;
+    }
+}
+export class Utils_DynamoDB {
+    dynamodb;
+    constructor(config) {
+        this.dynamodb = new DynamoDB({ region: config.region });
+    }
+    async scan(table, options = {}) {
+        const filters = options.filters !== undefined ? options.filters : {};
+        const undefined_attribute_names = options.undefined_attribute_names !== undefined ? options.undefined_attribute_names : [];
+        const defined_attribute_names = options.defined_attribute_names !== undefined ? options.defined_attribute_names : [];
+        const concurrency = options.concurrency !== undefined ? options.concurrency : 1;
+        const attribute_names = options.attribute_names !== undefined ? options.attribute_names : [];
+        const iterators = [];
+        for (let i = 0; i < concurrency; i++) {
+            const expression_attribute_names = Object.fromEntries([...Object.keys(filters), ...attribute_names, ...undefined_attribute_names, ...defined_attribute_names]
+                .map(attribute_name => ["#" + attribute_name, attribute_name]));
+            const expression_attribute_values = Object.fromEntries(Object.entries(filters)
+                .map(([attribute_name, attribute_value]) => [":" + attribute_name, convert_input(attribute_value)])
+                .filter(([attribute_name, attribute_value]) => not_undefined(attribute_value)));
+            const filter_expression = [
+                ...Object.keys(filters).map(attribute_name => "#" + attribute_name + " = :" + attribute_name),
+                ...undefined_attribute_names.map(attribute_name => "attribute_not_exists(#" + attribute_name + ")"),
+                ...defined_attribute_names.map(attribute_name => "attribute_exists(#" + attribute_name + ")")
+            ].join(" AND ");
+            const projection_expression = attribute_names.map(attribute_name => "#" + attribute_name).join(", ");
+            const request = {
+                TableName: table,
+                ExpressionAttributeNames: Object.keys(expression_attribute_names).length > 0 ? expression_attribute_names : undefined,
+                ExpressionAttributeValues: Object.keys(expression_attribute_values).length > 0 ? expression_attribute_values : undefined,
+                FilterExpression: filter_expression.length > 0 ? filter_expression : undefined,
+                ProjectionExpression: projection_expression.length > 0 ? projection_expression : undefined,
+                Segment: i,
+                TotalSegments: concurrency
+            };
+            iterators.push(compile_pages(request, (request) => this.dynamodb.scan(request)));
+        }
+        const segments = await Promise.all(iterators);
+        const items = segments.flat().filter(is_item);
+        return items;
+    }
+    async get(table, key, consistent = false) {
+        const converted_key = convert_input(key)?.M;
+        if (converted_key === undefined) {
+            return undefined;
+        }
+        const item = await this.dynamodb.getItem({
+            ConsistentRead: consistent,
+            TableName: table,
+            Key: converted_key
+        })
+            .then(response => response.Item);
+        if (item === undefined) {
+            return undefined;
+        }
+        const converted_output = convert_output({ M: item });
+        if (!is_item(converted_output)) {
+            return undefined;
+        }
+        return converted_output;
+    }
+    async get_max(table, primary_key) {
+        if (Object.keys(primary_key).length !== 1) {
+            return undefined;
+        }
+        const key = Object.keys(primary_key)[0];
+        const value = convert_input(Object.values(primary_key)[0]);
+        if (value === undefined) {
+            return undefined;
+        }
+        const request = {
+            TableName: table,
+            ExpressionAttributeNames: {
+                "#a": key
+            },
+            ExpressionAttributeValues: {
+                ":a": value
+            },
+            KeyConditionExpression: "#a = :a",
+            Limit: 1,
+            ScanIndexForward: false
+        };
+        const items = await this.dynamodb.query(request)
+            .then(response => response.Items);
+        if (items === undefined || items[0] === undefined) {
+            return undefined;
+        }
+        const converted_output = convert_output({ M: items[0] });
+        if (!is_item(converted_output)) {
+            return undefined;
+        }
+        return converted_output;
+    }
+    async query(table, primary_key, options = {}) {
+        const reverse = options.reverse !== undefined ? options.reverse : false;
+        const compile = options.compile !== undefined ? options.compile : true;
+        if (Object.keys(primary_key).length !== 1) {
+            return undefined;
+        }
+        const key = Object.keys(primary_key)[0];
+        const value = convert_input(Object.values(primary_key)[0]);
+        if (value === undefined) {
+            return undefined;
+        }
+        const request = {
+            TableName: table,
+            ExpressionAttributeNames: {
+                "#a": key
+            },
+            ExpressionAttributeValues: {
+                ":a": value
+            },
+            KeyConditionExpression: "#a = :a",
+            ScanIndexForward: !reverse
+        };
+        return await compile_pages(request, (request) => this.dynamodb.query(request), compile);
+    }
+    async query_prefix(table, primary_key, secondary_key_prefix, options = {}) {
+        const reverse = options.reverse !== undefined ? options.reverse : false;
+        const compile = options.compile !== undefined ? options.compile : true;
+        if (Object.keys(primary_key).length !== 1 || Object.keys(secondary_key_prefix).length !== 1) {
+            return undefined;
+        }
+        const converted_primary_value = convert_input(Object.values(primary_key)[0]);
+        const converted_secondary_prefix_value = convert_input(Object.values(secondary_key_prefix)[0]);
+        if (converted_primary_value === undefined || converted_secondary_prefix_value === undefined) {
+            return undefined;
+        }
+        const request = {
+            TableName: table,
+            ExpressionAttributeNames: {
+                "#a": Object.keys(primary_key)[0],
+                "#b": Object.keys(secondary_key_prefix)[0]
+            },
+            ExpressionAttributeValues: {
+                ":a": converted_primary_value,
+                ":b": converted_secondary_prefix_value
+            },
+            KeyConditionExpression: "#a = :a AND begins_with(#b, :b)",
+            ScanIndexForward: !reverse
+        };
+        return await compile_pages(request, (request) => this.dynamodb.query(request), compile);
+    }
+    async query_range(table, primary_key, secondary_key_range, options = {}) {
+        const reverse = options.reverse !== undefined ? options.reverse : false;
+        const compile = options.compile !== undefined ? options.compile : true;
+        if (Object.keys(primary_key).length !== 1 || Object.keys(secondary_key_range).length !== 1 || Object.values(secondary_key_range)[0].length !== 2) {
+            return undefined;
+        }
+        const converted_primary_value = convert_input(Object.values(primary_key)[0]);
+        const converted_secondary_range_start_value = convert_input(Object.values(secondary_key_range)[0][0]);
+        const converted_secondary_range_end_value = convert_input(Object.values(secondary_key_range)[0][1]);
+        if (converted_primary_value === undefined || converted_secondary_range_start_value === undefined || converted_secondary_range_end_value === undefined) {
+            return undefined;
+        }
+        const request = {
+            TableName: table,
+            ExpressionAttributeNames: {
+                "#a": Object.keys(primary_key)[0],
+                "#b": Object.keys(secondary_key_range)[0]
+            },
+            ExpressionAttributeValues: {
+                ":a": converted_primary_value,
+                ":b1": converted_secondary_range_start_value,
+                ":b2": converted_secondary_range_end_value
+            },
+            KeyConditionExpression: "#a = :a AND (#b BETWEEN :b1 AND :b2)",
+            ScanIndexForward: !reverse
+        };
+        return await compile_pages(request, (request) => this.dynamodb.query(request), compile);
+    }
+    async set(table, key, attributes) {
+        const converted_key = convert_input(key)?.M;
+        const request = {
+            TableName: table,
+            Key: converted_key,
+            UpdateExpression: "set " + Object.keys(attributes)
+                .filter(attribute_name => !Object.keys(key).includes(attribute_name))
+                .filter(attribute_name => attributes[attribute_name] !== undefined)
+                .map(attribute_name => "#" + attribute_name + " = :" + attribute_name).join(", "),
+            ExpressionAttributeNames: Object.fromEntries(Object.keys(attributes)
+                .filter(attribute_name => !Object.keys(key).includes(attribute_name))
+                .filter(attribute_name => attributes[attribute_name] !== undefined)
+                .map(attribute_name => ["#" + attribute_name, attribute_name])),
+            ExpressionAttributeValues: Object.fromEntries(Object.entries(attributes)
+                .filter(([attribute_name, attribute_value]) => !Object.keys(key).includes(attribute_name) && not_undefined(attribute_value))
+                .map(([attribute_name, attribute_value]) => [":" + attribute_name, convert_input(attribute_value)])
+                .filter(([attribute_name, attribute_value]) => not_undefined(attribute_value)))
+        };
+        await this.dynamodb.updateItem(request);
+    }
+    async append(table, key, attributes) {
+        const converted_key = convert_input(key)?.M;
+        const request = {
+            TableName: table,
+            Key: converted_key,
+            UpdateExpression: "set " + Object.keys(attributes)
+                .filter(attribute_name => attributes[attribute_name] !== undefined)
+                .map(attribute_name => "#" + attribute_name + " = list_append(#" + attribute_name + ", :" + attribute_name + ")").join(", "),
+            ExpressionAttributeNames: Object.fromEntries(Object.keys(attributes)
+                .filter(attribute_name => attributes[attribute_name] !== undefined)
+                .map(attribute_name => ["#" + attribute_name, attribute_name])),
+            ExpressionAttributeValues: Object.fromEntries(Object.entries(attributes)
+                .filter(([attribute_name, attribute_value]) => not_undefined(attribute_value))
+                .map(([attribute_name, attribute_value]) => [":" + attribute_name, convert_input(attribute_value)])
+                .filter(([attribute_name, attribute_value]) => not_undefined(attribute_value)))
+        };
+        this.dynamodb.updateItem(request);
+    }
+    async add(table, key, attributes) {
+        const item = await this.get(table, key, true);
+        if (item === undefined) {
+            return;
+        }
+        const new_attributes = {};
+        for (const [attribute, values] of Object.entries(attributes)) {
+            if (item[attribute] === undefined) {
+                continue;
+            }
+            const new_values = values.filter(value => !item[attribute].includes(value));
+            if (new_values.length > 0) {
+                new_attributes[attribute] = new_values;
+            }
+        }
+        if (Object.values(new_attributes).flat().length === 0) {
+            return undefined;
+        }
+        return await this.append(table, key, attributes);
+    }
+    async remove(table, key, attributes) {
+        const converted_key = convert_input(key)?.M;
+        const request = {
+            TableName: table,
+            Key: converted_key,
+            UpdateExpression: "remove " + attributes
+                .map(attribute_name => "#" + attribute_name).join(", "),
+            ExpressionAttributeNames: Object.fromEntries(attributes
+                .map(attribute_name => ["#" + attribute_name, attribute_name]))
+        };
+        await this.dynamodb.updateItem(request);
+    }
+    async create(table, key, attributes = {}) {
+        const item = await this.get(table, key, true);
+        if (item !== undefined) {
+            return;
+        }
+        const converted_key = convert_input(key)?.M;
+        const converted_attributes = convert_input(attributes)?.M;
+        await this.dynamodb.putItem({
+            TableName: table,
+            Item: { ...converted_key, ...converted_attributes }
+        });
+    }
+    async delete(table, key) {
+        const converted_key = convert_input(key)?.M;
+        await this.dynamodb.deleteItem({
+            TableName: table,
+            Key: converted_key
+        });
+    }
+    async duplicate_attribute(table, attribute_name, new_attribute_name) {
+        const table_metadata = await this.dynamodb.describeTable({
+            TableName: table
+        });
+        if (table_metadata.Table === undefined || table_metadata.Table.KeySchema === undefined) {
+            return;
+        }
+        const table_key_names = table_metadata.Table.KeySchema.map(key => key.AttributeName)
+            .filter(table_key_name => table_key_name !== undefined);
+        const items = await this.scan(table, { attribute_names: table_key_names.concat([attribute_name, new_attribute_name]) });
+        if (items.filter(item => item[new_attribute_name] !== undefined).length > 0) {
+            console.log("Cannot rename.", new_attribute_name, "is an existing item.");
+            return;
+        }
+        for (const item of items) {
+            if (item[attribute_name] === undefined) {
+                continue;
+            }
+            const key = Object.fromEntries(table_key_names.map(key_name => [key_name, item[key_name]]));
+            await this.set(table, key, { [new_attribute_name]: item[attribute_name] });
+        }
+    }
+    async remove_attribute(table, attribute_name) {
+        const table_metadata = await this.dynamodb.describeTable({
+            TableName: table
+        });
+        if (table_metadata.Table === undefined || table_metadata.Table.KeySchema === undefined) {
+            return;
+        }
+        const table_key_names = table_metadata.Table.KeySchema.map(key => key.AttributeName)
+            .filter(table_key_name => table_key_name !== undefined);
+        const items = await this.scan(table)
+            .then(items => items.filter(item => item[attribute_name] !== undefined));
+        for (const item of items) {
+            const key = Object.fromEntries(table_key_names.map(key_name => [key_name, item[key_name]]));
+            await this.remove(table, key, [attribute_name]);
+        }
+    }
+}