triangle-utils 1.3.4 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/UtilsBedrock.d.ts +8 -0
- package/dist/src/{Utils_Bedrock.js → UtilsBedrock.js} +7 -4
- package/dist/src/UtilsBee.d.ts +8 -0
- package/dist/src/{Utils_Bee.js → UtilsBee.js} +13 -8
- package/dist/src/UtilsCognito.d.ts +12 -0
- package/dist/src/UtilsCognito.js +110 -0
- package/dist/src/{Utils_DynamoDB.d.ts → UtilsDynamoDB.d.ts} +6 -8
- package/dist/src/{Utils_DynamoDB.js → UtilsDynamoDB.js} +18 -18
- package/dist/src/UtilsMisc.d.ts +32 -0
- package/dist/src/UtilsMisc.js +156 -0
- package/dist/src/{Utils_S3.d.ts → UtilsS3.d.ts} +3 -5
- package/dist/src/{Utils_S3.js → UtilsS3.js} +3 -3
- package/dist/src/{Utils_S3Vectors.d.ts → UtilsS3Vectors.d.ts} +5 -8
- package/dist/src/{Utils_S3Vectors.js → UtilsS3Vectors.js} +4 -4
- package/dist/src/UtilsYoutube.d.ts +6 -0
- package/dist/src/{Utils_Youtube.js → UtilsYoutube.js} +8 -7
- package/dist/src/index.d.ts +28 -0
- package/dist/src/index.js +36 -0
- package/dist/src/types/{Config.d.ts → TriangleUtilsConfig.d.ts} +1 -3
- package/package.json +8 -8
- package/src/{Utils_Bedrock.ts → UtilsBedrock.ts} +10 -8
- package/src/{Utils_Bee.ts → UtilsBee.ts} +21 -15
- package/src/UtilsCognito.ts +116 -0
- package/src/{Utils_DynamoDB.ts → UtilsDynamoDB.ts} +22 -23
- package/src/UtilsMisc.ts +188 -0
- package/src/{Utils_S3.ts → UtilsS3.ts} +4 -5
- package/src/{Utils_S3Vectors.ts → UtilsS3Vectors.ts} +6 -7
- package/src/{Utils_Youtube.ts → UtilsYoutube.ts} +9 -9
- package/src/index.ts +43 -0
- package/src/types/TriangleUtilsConfig.ts +6 -0
- package/test/Utils.test.ts +135 -24
- package/vite.config.ts +10 -0
- package/dist/src/Utils.d.ts +0 -17
- package/dist/src/Utils.js +0 -24
- package/dist/src/Utils_Bedrock.d.ts +0 -10
- package/dist/src/Utils_Bee.d.ts +0 -11
- package/dist/src/Utils_Misc.d.ts +0 -30
- package/dist/src/Utils_Misc.js +0 -627
- package/dist/src/Utils_Youtube.d.ts +0 -8
- package/src/Utils.ts +0 -28
- package/src/Utils_Misc.ts +0 -678
- package/src/types/Config.ts +0 -9
- /package/dist/src/types/{Config.js → TriangleUtilsConfig.js} +0 -0
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export declare class UtilsBedrock {
|
|
2
|
+
private readonly bedrock;
|
|
3
|
+
private readonly text_decoder;
|
|
4
|
+
constructor(region: string);
|
|
5
|
+
llama_invoke(model_id: string, prompt: string, temperature?: number, max_gen_len?: number, top_p?: number): Promise<string | undefined>;
|
|
6
|
+
gpt_converse(model_id: string, prompt: string, temperature?: number, max_gen_len?: number, top_p?: number): Promise<string | undefined>;
|
|
7
|
+
titan_invoke(text: string): Promise<number[] | undefined>;
|
|
8
|
+
}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { BedrockRuntime } from "@aws-sdk/client-bedrock-runtime";
|
|
2
|
-
export class
|
|
2
|
+
export class UtilsBedrock {
|
|
3
3
|
bedrock;
|
|
4
4
|
text_decoder;
|
|
5
|
-
constructor(
|
|
6
|
-
this.bedrock = new BedrockRuntime({ region:
|
|
5
|
+
constructor(region) {
|
|
6
|
+
this.bedrock = new BedrockRuntime({ region: region });
|
|
7
7
|
this.text_decoder = new TextDecoder();
|
|
8
8
|
}
|
|
9
9
|
async llama_invoke(model_id, prompt, temperature = 0.5, max_gen_len = 512, top_p = 0.9) {
|
|
@@ -74,7 +74,10 @@ export class Utils_Bedrock {
|
|
|
74
74
|
contentType: "application/json"
|
|
75
75
|
});
|
|
76
76
|
const response = JSON.parse(this.text_decoder.decode(output.body));
|
|
77
|
-
|
|
77
|
+
if (response.embedding !== undefined && Array.isArray(response.embedding) && response.embedding.every((i) => typeof i === "number")) {
|
|
78
|
+
const embedding = response.embedding;
|
|
79
|
+
return embedding;
|
|
80
|
+
}
|
|
78
81
|
}
|
|
79
82
|
catch (error) {
|
|
80
83
|
if (error instanceof Error) {
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export declare class UtilsBee {
|
|
2
|
+
private readonly bee;
|
|
3
|
+
private readonly text_decoder;
|
|
4
|
+
constructor(scraping_bee_api_key: string | undefined);
|
|
5
|
+
get(url: string, params?: Record<string, string | number | boolean>): Promise<string | undefined>;
|
|
6
|
+
google_search(query: string, news?: boolean): Promise<Record<string, any>[] | undefined>;
|
|
7
|
+
youtube_search(query: string, options?: {}): Promise<Record<string, any>[] | undefined>;
|
|
8
|
+
}
|
|
@@ -1,11 +1,9 @@
|
|
|
1
1
|
import { ScrapingBeeClient } from "scrapingbee";
|
|
2
|
-
export class
|
|
3
|
-
config;
|
|
2
|
+
export class UtilsBee {
|
|
4
3
|
bee;
|
|
5
4
|
text_decoder;
|
|
6
|
-
constructor(
|
|
7
|
-
this.
|
|
8
|
-
this.bee = config.scraping_bee_api_key !== undefined ? new ScrapingBeeClient(config.scraping_bee_api_key) : undefined;
|
|
5
|
+
constructor(scraping_bee_api_key) {
|
|
6
|
+
this.bee = scraping_bee_api_key !== undefined ? new ScrapingBeeClient(scraping_bee_api_key) : undefined;
|
|
9
7
|
this.text_decoder = new TextDecoder();
|
|
10
8
|
}
|
|
11
9
|
async get(url, params = {}) {
|
|
@@ -14,7 +12,7 @@ export class Utils_Bee {
|
|
|
14
12
|
}
|
|
15
13
|
for (let i = 0; i < 3; i++) {
|
|
16
14
|
try {
|
|
17
|
-
const response = await this.bee.
|
|
15
|
+
const response = await this.bee.htmlApi({
|
|
18
16
|
url: url,
|
|
19
17
|
params: params
|
|
20
18
|
});
|
|
@@ -51,12 +49,19 @@ export class Utils_Bee {
|
|
|
51
49
|
console.log("Failed to Google:", response.status);
|
|
52
50
|
continue;
|
|
53
51
|
}
|
|
54
|
-
const data = response.data;
|
|
52
|
+
const data = JSON.parse(response.data.toString("utf-8"));
|
|
53
|
+
if (data.news_results === undefined || data.organic_results === undefined) {
|
|
54
|
+
continue;
|
|
55
|
+
}
|
|
55
56
|
const results = news ? data.news_results : data.organic_results;
|
|
56
57
|
if (results === undefined) {
|
|
57
58
|
console.log("Failed to Google.");
|
|
58
59
|
continue;
|
|
59
60
|
}
|
|
61
|
+
if (!Array.isArray(results)) {
|
|
62
|
+
console.log("Results are not an array.");
|
|
63
|
+
return;
|
|
64
|
+
}
|
|
60
65
|
return results;
|
|
61
66
|
}
|
|
62
67
|
catch {
|
|
@@ -80,7 +85,7 @@ export class Utils_Bee {
|
|
|
80
85
|
console.log("Failed to YouTube Search:", response.status);
|
|
81
86
|
continue;
|
|
82
87
|
}
|
|
83
|
-
const data = response.data;
|
|
88
|
+
const data = JSON.parse(response.data.toString("utf-8"));
|
|
84
89
|
const results = data.results;
|
|
85
90
|
if (results === undefined) {
|
|
86
91
|
console.log("Failed to YouTube Search.");
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export declare class UtilsCognito {
|
|
2
|
+
private readonly cognito;
|
|
3
|
+
constructor(region: string);
|
|
4
|
+
admin_get_user(user_pool_id: string, username: string): Promise<import("@aws-sdk/client-cognito-identity-provider").AdminGetUserCommandOutput | undefined>;
|
|
5
|
+
admin_create_user(user_pool_id: string, username: string, user_id: string): Promise<Error | import("@aws-sdk/client-cognito-identity-provider").AdminCreateUserCommandOutput>;
|
|
6
|
+
admin_reset_password(user_pool_id: string, username: string): Promise<Error | import("@aws-sdk/client-cognito-identity-provider").AdminCreateUserCommandOutput>;
|
|
7
|
+
get_user_id(token: string): Promise<string | undefined>;
|
|
8
|
+
get_new_token(client_id: string, client_secret: string, refresh_token: string): Promise<string | undefined>;
|
|
9
|
+
sign_out(token: string): Promise<void>;
|
|
10
|
+
admin_disable_user(user_pool_id: string, username: string): Promise<void>;
|
|
11
|
+
admin_delete_user(user_pool_id: string, username: string): Promise<void>;
|
|
12
|
+
}
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import { CognitoIdentityProvider, NotAuthorizedException } from "@aws-sdk/client-cognito-identity-provider";
|
|
2
|
+
export class UtilsCognito {
|
|
3
|
+
cognito;
|
|
4
|
+
constructor(region) {
|
|
5
|
+
this.cognito = new CognitoIdentityProvider({ region: region });
|
|
6
|
+
}
|
|
7
|
+
async admin_get_user(user_pool_id, username) {
|
|
8
|
+
try {
|
|
9
|
+
return await this.cognito.adminGetUser({
|
|
10
|
+
UserPoolId: user_pool_id,
|
|
11
|
+
Username: username
|
|
12
|
+
});
|
|
13
|
+
}
|
|
14
|
+
catch (error) {
|
|
15
|
+
return undefined;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
async admin_create_user(user_pool_id, username, user_id) {
|
|
19
|
+
try {
|
|
20
|
+
return await this.cognito.adminCreateUser({
|
|
21
|
+
UserPoolId: user_pool_id,
|
|
22
|
+
Username: username,
|
|
23
|
+
UserAttributes: [
|
|
24
|
+
{ Name: "preferred_username", Value: user_id },
|
|
25
|
+
{ Name: "email_verified", Value: "true" }
|
|
26
|
+
]
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
catch (error) {
|
|
30
|
+
if (error instanceof Error) {
|
|
31
|
+
return new Error(error.toString().replace("Exception: ", ""));
|
|
32
|
+
}
|
|
33
|
+
return new Error();
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
async admin_reset_password(user_pool_id, username) {
|
|
37
|
+
const user = await this.admin_get_user(user_pool_id, username);
|
|
38
|
+
if (user === undefined) {
|
|
39
|
+
return new Error("User does not exist.");
|
|
40
|
+
}
|
|
41
|
+
try {
|
|
42
|
+
return await this.cognito.adminCreateUser({
|
|
43
|
+
UserPoolId: user_pool_id,
|
|
44
|
+
Username: username,
|
|
45
|
+
MessageAction: "RESEND"
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
catch (error) {
|
|
49
|
+
if (error instanceof Error) {
|
|
50
|
+
return new Error(error.toString().replace("Exception: ", ""));
|
|
51
|
+
}
|
|
52
|
+
return new Error();
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
async get_user_id(token) {
|
|
56
|
+
try {
|
|
57
|
+
const user_info = await this.cognito.getUser({
|
|
58
|
+
AccessToken: token
|
|
59
|
+
});
|
|
60
|
+
const user_attributes = user_info.UserAttributes;
|
|
61
|
+
if (user_attributes === undefined) {
|
|
62
|
+
return undefined;
|
|
63
|
+
}
|
|
64
|
+
const user_attribute = user_attributes.filter(attribute => attribute.Name === "preferred_username")[0];
|
|
65
|
+
const user_id = user_attribute.Value;
|
|
66
|
+
return user_id;
|
|
67
|
+
}
|
|
68
|
+
catch (error) {
|
|
69
|
+
if (error instanceof NotAuthorizedException) {
|
|
70
|
+
return undefined;
|
|
71
|
+
}
|
|
72
|
+
throw error;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
async get_new_token(client_id, client_secret, refresh_token) {
|
|
76
|
+
try {
|
|
77
|
+
const response = await this.cognito.getTokensFromRefreshToken({
|
|
78
|
+
ClientId: client_id,
|
|
79
|
+
ClientSecret: client_secret,
|
|
80
|
+
RefreshToken: refresh_token
|
|
81
|
+
});
|
|
82
|
+
const new_auth_result = response.AuthenticationResult;
|
|
83
|
+
if (new_auth_result === undefined) {
|
|
84
|
+
return undefined;
|
|
85
|
+
}
|
|
86
|
+
const new_token = new_auth_result.AccessToken;
|
|
87
|
+
return new_token;
|
|
88
|
+
}
|
|
89
|
+
catch (error) {
|
|
90
|
+
return undefined;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
async sign_out(token) {
|
|
94
|
+
await this.cognito.globalSignOut({
|
|
95
|
+
AccessToken: token
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
async admin_disable_user(user_pool_id, username) {
|
|
99
|
+
await this.cognito.adminDisableUser({
|
|
100
|
+
UserPoolId: user_pool_id,
|
|
101
|
+
Username: username,
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
async admin_delete_user(user_pool_id, username) {
|
|
105
|
+
await this.cognito.adminDeleteUser({
|
|
106
|
+
UserPoolId: user_pool_id,
|
|
107
|
+
Username: username,
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
}
|
|
@@ -1,8 +1,6 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
readonly dynamodb: DynamoDB;
|
|
5
|
-
constructor(config: Config);
|
|
1
|
+
export declare class UtilsDynamoDB {
|
|
2
|
+
private readonly dynamodb;
|
|
3
|
+
constructor(region: string);
|
|
6
4
|
scan(table: string, options?: {
|
|
7
5
|
filters?: Record<string, any>;
|
|
8
6
|
undefined_attribute_names?: string[];
|
|
@@ -19,8 +17,8 @@ export declare class Utils_DynamoDB {
|
|
|
19
17
|
query_prefix(table: string, primary_key: Record<string, any>, secondary_key_prefix: Record<string, string>, options?: {
|
|
20
18
|
reverse?: boolean;
|
|
21
19
|
compile?: boolean;
|
|
22
|
-
}): Promise<Record<string, any>[]
|
|
23
|
-
query_range(table: string, primary_key: Record<string, any>, secondary_key_range: Record<string, string | number[]>, options?: {
|
|
20
|
+
}): Promise<Record<string, any>[]>;
|
|
21
|
+
query_range(table: string, primary_key: Record<string, any>, secondary_key_range: Record<string, (string | number)[]>, options?: {
|
|
24
22
|
reverse?: boolean;
|
|
25
23
|
compile?: boolean;
|
|
26
24
|
}): Promise<Record<string, any>[]>;
|
|
@@ -28,7 +26,7 @@ export declare class Utils_DynamoDB {
|
|
|
28
26
|
append(table: string, key: Record<string, any>, attributes: Record<string, any[]>): Promise<void>;
|
|
29
27
|
add(table: string, key: Record<string, any>, attributes: Record<string, any[]>): Promise<void>;
|
|
30
28
|
remove(table: string, key: Record<string, any>, attributes: string[]): Promise<void>;
|
|
31
|
-
create(table: string, key: Record<string, any>, attributes?: Record<string, any
|
|
29
|
+
create(table: string, key: Record<string, any>, attributes?: Record<string, any>): Promise<void>;
|
|
32
30
|
delete(table: string, key: Record<string, any>): Promise<void>;
|
|
33
31
|
duplicate_attribute(table: string, attribute_name: string, new_attribute_name: string): Promise<void>;
|
|
34
32
|
remove_attribute(table: string, attribute_name: string): Promise<void>;
|
|
@@ -1,22 +1,22 @@
|
|
|
1
1
|
import { DynamoDB } from "@aws-sdk/client-dynamodb";
|
|
2
|
-
function convert_output(
|
|
3
|
-
if (
|
|
4
|
-
return
|
|
2
|
+
function convert_output(dynamodb_output) {
|
|
3
|
+
if (dynamodb_output.S !== undefined) {
|
|
4
|
+
return dynamodb_output.S;
|
|
5
5
|
}
|
|
6
|
-
else if (
|
|
7
|
-
return Number(
|
|
6
|
+
else if (dynamodb_output.N !== undefined) {
|
|
7
|
+
return Number(dynamodb_output.N);
|
|
8
8
|
}
|
|
9
|
-
else if (
|
|
10
|
-
return
|
|
9
|
+
else if (dynamodb_output.L !== undefined) {
|
|
10
|
+
return dynamodb_output.L.map((a) => convert_output(a));
|
|
11
11
|
}
|
|
12
|
-
else if (
|
|
13
|
-
return new Set(
|
|
12
|
+
else if (dynamodb_output.SS !== undefined) {
|
|
13
|
+
return new Set(dynamodb_output.SS);
|
|
14
14
|
}
|
|
15
|
-
else if (
|
|
16
|
-
return Object.fromEntries(Object.entries(
|
|
15
|
+
else if (dynamodb_output.M !== undefined) {
|
|
16
|
+
return Object.fromEntries(Object.entries(dynamodb_output.M).map(([key, value]) => [key, convert_output(value)]));
|
|
17
17
|
}
|
|
18
|
-
else if (
|
|
19
|
-
return
|
|
18
|
+
else if (dynamodb_output.BOOL !== undefined) {
|
|
19
|
+
return dynamodb_output.BOOL;
|
|
20
20
|
}
|
|
21
21
|
return undefined;
|
|
22
22
|
}
|
|
@@ -82,10 +82,10 @@ async function compile_pages(request, f, compile = true) {
|
|
|
82
82
|
last_eval_key = response.LastEvaluatedKey;
|
|
83
83
|
}
|
|
84
84
|
}
|
|
85
|
-
export class
|
|
85
|
+
export class UtilsDynamoDB {
|
|
86
86
|
dynamodb;
|
|
87
|
-
constructor(
|
|
88
|
-
this.dynamodb = new DynamoDB({ region:
|
|
87
|
+
constructor(region) {
|
|
88
|
+
this.dynamodb = new DynamoDB({ region: region });
|
|
89
89
|
}
|
|
90
90
|
async scan(table, options = {}) {
|
|
91
91
|
const filters = options.filters !== undefined ? options.filters : {};
|
|
@@ -201,12 +201,12 @@ export class Utils_DynamoDB {
|
|
|
201
201
|
const reverse = options.reverse !== undefined ? options.reverse : false;
|
|
202
202
|
const compile = options.compile !== undefined ? options.compile : true;
|
|
203
203
|
if (Object.keys(primary_key).length !== 1 || Object.keys(secondary_key_prefix).length !== 1) {
|
|
204
|
-
return
|
|
204
|
+
return [];
|
|
205
205
|
}
|
|
206
206
|
const converted_primary_value = convert_input(Object.values(primary_key)[0]);
|
|
207
207
|
const converted_secondary_prefix_value = convert_input(Object.values(secondary_key_prefix)[0]);
|
|
208
208
|
if (converted_primary_value === undefined || converted_secondary_prefix_value === undefined) {
|
|
209
|
-
return
|
|
209
|
+
return [];
|
|
210
210
|
}
|
|
211
211
|
const request = {
|
|
212
212
|
TableName: table,
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import * as nodemailer from "nodemailer";
|
|
2
|
+
import { TriangleUtilsConfig } from "./types/TriangleUtilsConfig.js";
|
|
3
|
+
interface Encryption {
|
|
4
|
+
iv: string;
|
|
5
|
+
ciphertext: string;
|
|
6
|
+
}
|
|
7
|
+
export declare class UtilsMisc {
|
|
8
|
+
readonly config: TriangleUtilsConfig;
|
|
9
|
+
readonly transporter: nodemailer.Transporter<import("nodemailer/lib/smtp-transport").SentMessageInfo, import("nodemailer/lib/smtp-transport").Options> | undefined;
|
|
10
|
+
constructor(config: TriangleUtilsConfig);
|
|
11
|
+
send_email(recipient: string, subject: string, text: string): Promise<boolean>;
|
|
12
|
+
admin_alert(text: string): Promise<boolean>;
|
|
13
|
+
safe_run(f: () => Promise<any>): Promise<void>;
|
|
14
|
+
iterate<T>(inputs: T[], f: (input: T) => Promise<any>, concurrency?: number, print_indices?: boolean): Promise<void>;
|
|
15
|
+
static wait(duration: number): Promise<unknown>;
|
|
16
|
+
static get_current_time(): string;
|
|
17
|
+
static get_current_milliseconds(): number;
|
|
18
|
+
static sha256(input: string): Promise<string>;
|
|
19
|
+
static encrypt(secret_key: string, text: string): Encryption;
|
|
20
|
+
static decrypt(secret_key: string, encryption: Encryption): string;
|
|
21
|
+
static get_secret_hash(username: string, cognito_client_id: string, cognito_client_secret: string): string;
|
|
22
|
+
static get_chunk_indices(text_length: number, max_length?: number, overlap?: number): [number, number][];
|
|
23
|
+
static chunkify(text: string, max_length?: number, overlap?: number): {
|
|
24
|
+
chunk_index: [number, number];
|
|
25
|
+
chunk_text: string;
|
|
26
|
+
}[];
|
|
27
|
+
static chunkify_lined(text: string, max_length?: number, overlap?: number): {
|
|
28
|
+
chunk_index: [number, number];
|
|
29
|
+
chunk_text: string;
|
|
30
|
+
}[];
|
|
31
|
+
}
|
|
32
|
+
export {};
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
import * as nodemailer from "nodemailer";
|
|
2
|
+
import * as crypto from "crypto";
|
|
3
|
+
export class UtilsMisc {
|
|
4
|
+
config;
|
|
5
|
+
transporter;
|
|
6
|
+
constructor(config) {
|
|
7
|
+
this.config = config;
|
|
8
|
+
if (config.google_email !== undefined && config.google_app_password !== undefined) {
|
|
9
|
+
this.transporter = nodemailer.createTransport({
|
|
10
|
+
service: "Gmail",
|
|
11
|
+
host: "smtp.gmail.com",
|
|
12
|
+
port: 465,
|
|
13
|
+
secure: true,
|
|
14
|
+
auth: {
|
|
15
|
+
user: config.google_email,
|
|
16
|
+
pass: config.google_app_password,
|
|
17
|
+
}
|
|
18
|
+
});
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
async send_email(recipient, subject, text) {
|
|
22
|
+
if (this.transporter === undefined) {
|
|
23
|
+
return false;
|
|
24
|
+
}
|
|
25
|
+
for (let i = 0; i < 3; i++) {
|
|
26
|
+
try {
|
|
27
|
+
await this.transporter.sendMail({
|
|
28
|
+
from: this.config.alerts_email,
|
|
29
|
+
to: recipient,
|
|
30
|
+
subject: subject,
|
|
31
|
+
text: text
|
|
32
|
+
});
|
|
33
|
+
return true;
|
|
34
|
+
}
|
|
35
|
+
catch (error) {
|
|
36
|
+
console.log("EMAIL ERROR", error);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
return false;
|
|
40
|
+
}
|
|
41
|
+
async admin_alert(text) {
|
|
42
|
+
console.log("ADMIN ALERT:", text);
|
|
43
|
+
return await this.send_email(this.config.google_email, "ADMIN ALERT", text);
|
|
44
|
+
}
|
|
45
|
+
async safe_run(f) {
|
|
46
|
+
try {
|
|
47
|
+
await f();
|
|
48
|
+
}
|
|
49
|
+
catch (error) {
|
|
50
|
+
if (!(error instanceof Error)) {
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
if (error.stack !== undefined) {
|
|
54
|
+
await this.admin_alert(error.stack.toString());
|
|
55
|
+
}
|
|
56
|
+
else {
|
|
57
|
+
await this.admin_alert(error.toString());
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
async iterate(inputs, f, concurrency = 1, print_indices = false) {
|
|
62
|
+
let index = 0;
|
|
63
|
+
let iterators = [];
|
|
64
|
+
for (let i = 0; i < concurrency; i++) {
|
|
65
|
+
iterators.push((async () => {
|
|
66
|
+
while (index < inputs.length) {
|
|
67
|
+
index += 1;
|
|
68
|
+
if (print_indices) {
|
|
69
|
+
console.log(i + ":" + (index - 1) + "/" + inputs.length);
|
|
70
|
+
}
|
|
71
|
+
await this.safe_run(() => f(inputs[index - 1]));
|
|
72
|
+
}
|
|
73
|
+
})());
|
|
74
|
+
}
|
|
75
|
+
await Promise.all(iterators);
|
|
76
|
+
}
|
|
77
|
+
static async wait(duration) {
|
|
78
|
+
return new Promise(resolve => setTimeout(resolve, duration));
|
|
79
|
+
}
|
|
80
|
+
static get_current_time() {
|
|
81
|
+
return (new Date()).toISOString();
|
|
82
|
+
}
|
|
83
|
+
static get_current_milliseconds() {
|
|
84
|
+
return (new Date()).getTime();
|
|
85
|
+
}
|
|
86
|
+
static async sha256(input) {
|
|
87
|
+
const text_encoder = new TextEncoder();
|
|
88
|
+
const input_buffer = text_encoder.encode(input);
|
|
89
|
+
const hash_buffer = await crypto.subtle.digest("SHA-256", input_buffer);
|
|
90
|
+
const hash_array = Array.from(new Uint8Array(hash_buffer));
|
|
91
|
+
const hash = hash_array.map(b => b.toString(16).padStart(2, "0")).join("");
|
|
92
|
+
return hash;
|
|
93
|
+
}
|
|
94
|
+
static encrypt(secret_key, text) {
|
|
95
|
+
const iv = crypto.randomBytes(16);
|
|
96
|
+
const cipher = crypto.createCipheriv("aes-256-cbc", Buffer.from(secret_key, "hex"), iv);
|
|
97
|
+
let ciphertext = cipher.update(text, "utf8", "hex");
|
|
98
|
+
ciphertext += cipher.final("hex");
|
|
99
|
+
return {
|
|
100
|
+
iv: iv.toString("hex"),
|
|
101
|
+
ciphertext: ciphertext
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
static decrypt(secret_key, encryption) {
|
|
105
|
+
const iv = encryption.iv;
|
|
106
|
+
const ciphertext = encryption.ciphertext;
|
|
107
|
+
const decipher = crypto.createDecipheriv("aes-256-cbc", Buffer.from(secret_key, "hex"), Buffer.from(iv, "hex"));
|
|
108
|
+
let text = decipher.update(ciphertext, "hex", "utf8");
|
|
109
|
+
text += decipher.final("utf8");
|
|
110
|
+
return text;
|
|
111
|
+
}
|
|
112
|
+
static get_secret_hash(username, cognito_client_id, cognito_client_secret) {
|
|
113
|
+
const secret_hash = crypto.createHmac("sha256", cognito_client_secret)
|
|
114
|
+
.update(username + cognito_client_id)
|
|
115
|
+
.digest("base64");
|
|
116
|
+
return secret_hash;
|
|
117
|
+
}
|
|
118
|
+
static get_chunk_indices(text_length, max_length = 2500, overlap = 50) {
|
|
119
|
+
const num_chunks = Math.ceil((text_length + overlap) / max_length);
|
|
120
|
+
const chunk_length = Math.ceil((text_length - overlap) / num_chunks + overlap);
|
|
121
|
+
const chunk_indices = [];
|
|
122
|
+
for (let i = 0; i < num_chunks; i++) {
|
|
123
|
+
chunk_indices.push([
|
|
124
|
+
(chunk_length - overlap) * i,
|
|
125
|
+
(chunk_length - overlap) * i + chunk_length
|
|
126
|
+
]);
|
|
127
|
+
}
|
|
128
|
+
return chunk_indices;
|
|
129
|
+
}
|
|
130
|
+
static chunkify(text, max_length = 2500, overlap = 50) {
|
|
131
|
+
const chunk_indices = this.get_chunk_indices(text.length, max_length, overlap);
|
|
132
|
+
const chunks = chunk_indices.map(chunk_index => ({
|
|
133
|
+
chunk_index: chunk_index,
|
|
134
|
+
chunk_text: text.substring(...chunk_index)
|
|
135
|
+
}));
|
|
136
|
+
return chunks;
|
|
137
|
+
}
|
|
138
|
+
static chunkify_lined(text, max_length = 2500, overlap = 50) {
|
|
139
|
+
const chunk_indices = this.get_chunk_indices(text.length, max_length, overlap);
|
|
140
|
+
const lined_chunk_indices = [];
|
|
141
|
+
for (const chunk_index of chunk_indices) {
|
|
142
|
+
const lined_chunk_index = [
|
|
143
|
+
text.lastIndexOf("\n", chunk_index[0]) + 1,
|
|
144
|
+
text.indexOf("\n", chunk_index[1]) + 1
|
|
145
|
+
];
|
|
146
|
+
if (lined_chunk_index[1] < lined_chunk_index[0]) {
|
|
147
|
+
lined_chunk_index[1] += text.length;
|
|
148
|
+
}
|
|
149
|
+
lined_chunk_indices.push(lined_chunk_index);
|
|
150
|
+
}
|
|
151
|
+
return lined_chunk_indices.map(chunk_index => ({
|
|
152
|
+
chunk_index: chunk_index,
|
|
153
|
+
chunk_text: text.substring(...chunk_index)
|
|
154
|
+
}));
|
|
155
|
+
}
|
|
156
|
+
}
|
|
@@ -1,8 +1,6 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
readonly s3: S3;
|
|
5
|
-
constructor(config: Config);
|
|
1
|
+
export declare class UtilsS3 {
|
|
2
|
+
private readonly s3;
|
|
3
|
+
constructor(region: string);
|
|
6
4
|
file_exists(s3_filename: string): Promise<boolean | undefined>;
|
|
7
5
|
get_file(s3_filename: string): Promise<string | undefined>;
|
|
8
6
|
create_file(s3_filename: string, content: string | Buffer, options?: {
|
|
@@ -9,10 +9,10 @@ function parse_s3_filename(s3_filename) {
|
|
|
9
9
|
const filename = bucket_filename.split("/").slice(1).join("/");
|
|
10
10
|
return { Bucket: bucket, Key: filename };
|
|
11
11
|
}
|
|
12
|
-
export class
|
|
12
|
+
export class UtilsS3 {
|
|
13
13
|
s3;
|
|
14
|
-
constructor(
|
|
15
|
-
this.s3 = new S3({ region:
|
|
14
|
+
constructor(region) {
|
|
15
|
+
this.s3 = new S3({ region: region });
|
|
16
16
|
}
|
|
17
17
|
async file_exists(s3_filename) {
|
|
18
18
|
const s3_key = parse_s3_filename(s3_filename);
|
|
@@ -1,13 +1,11 @@
|
|
|
1
|
-
|
|
2
|
-
import { Config } from "./types/Config.js";
|
|
3
|
-
interface Vector {
|
|
1
|
+
export interface Vector {
|
|
4
2
|
key: string;
|
|
5
3
|
data: number[];
|
|
6
4
|
metadata: any;
|
|
7
5
|
}
|
|
8
|
-
export declare class
|
|
9
|
-
readonly s3vectors
|
|
10
|
-
constructor(
|
|
6
|
+
export declare class UtilsS3Vectors {
|
|
7
|
+
private readonly s3vectors;
|
|
8
|
+
constructor(region: string);
|
|
11
9
|
scan(vector_bucket: string, vector_index: string): Promise<string[]>;
|
|
12
10
|
get(vector_bucket: string, vector_index: string, vector_key: string): Promise<Vector | undefined>;
|
|
13
11
|
batch_get(vector_bucket: string, vector_index: string, vector_keys: string[]): Promise<(Vector | undefined)[]>;
|
|
@@ -15,6 +13,5 @@ export declare class Utils_S3Vectors {
|
|
|
15
13
|
batch_create(vector_bucket: string, vector_index: string, vectors: Vector[]): Promise<void>;
|
|
16
14
|
delete(vector_bucket: string, vector_index: string, vector_key: string): Promise<void>;
|
|
17
15
|
batch_delete(vector_bucket: string, vector_index: string, vector_keys: string[]): Promise<void>;
|
|
18
|
-
query(vector_bucket: string, vector_index: string, data: number[], num_vectors: number, filters?: Record<string, any>): Promise<import("@aws-sdk/client-s3vectors").QueryOutputVector[] | undefined>;
|
|
16
|
+
query(vector_bucket: string, vector_index: string, data: number[], num_vectors: number, filters?: Record<string, any> | undefined): Promise<import("@aws-sdk/client-s3vectors").QueryOutputVector[] | undefined>;
|
|
19
17
|
}
|
|
20
|
-
export {};
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { S3Vectors } from "@aws-sdk/client-s3vectors";
|
|
2
|
-
export class
|
|
2
|
+
export class UtilsS3Vectors {
|
|
3
3
|
s3vectors;
|
|
4
|
-
constructor(
|
|
5
|
-
this.s3vectors = new S3Vectors({ region:
|
|
4
|
+
constructor(region) {
|
|
5
|
+
this.s3vectors = new S3Vectors({ region: region });
|
|
6
6
|
}
|
|
7
7
|
async scan(vector_bucket, vector_index) {
|
|
8
8
|
const vector_keys = [];
|
|
@@ -114,7 +114,7 @@ export class Utils_S3Vectors {
|
|
|
114
114
|
});
|
|
115
115
|
}
|
|
116
116
|
}
|
|
117
|
-
async query(vector_bucket, vector_index, data, num_vectors, filters =
|
|
117
|
+
async query(vector_bucket, vector_index, data, num_vectors, filters = undefined) {
|
|
118
118
|
const response = await this.s3vectors.queryVectors({
|
|
119
119
|
vectorBucketName: vector_bucket,
|
|
120
120
|
indexName: vector_index,
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { youtube_v3 } from "googleapis";
|
|
2
|
+
export declare class UtilsYoutube {
|
|
3
|
+
readonly youtube: youtube_v3.Youtube | undefined;
|
|
4
|
+
constructor(youtube_api_key: string | undefined);
|
|
5
|
+
batch_get_videos(video_ids: string[], attributes?: string[]): Promise<youtube_v3.Schema$Video[] | undefined>;
|
|
6
|
+
}
|
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
import { youtube_v3 } from "googleapis";
|
|
2
|
-
export class
|
|
3
|
-
config;
|
|
2
|
+
export class UtilsYoutube {
|
|
4
3
|
youtube;
|
|
5
|
-
constructor(
|
|
6
|
-
this.
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
});
|
|
4
|
+
constructor(youtube_api_key) {
|
|
5
|
+
this.youtube = youtube_api_key !== undefined ? new youtube_v3.Youtube({
|
|
6
|
+
auth: youtube_api_key
|
|
7
|
+
}) : undefined;
|
|
10
8
|
}
|
|
11
9
|
async batch_get_videos(video_ids, attributes = ["snippet", "contentDetails"]) {
|
|
10
|
+
if (this.youtube === undefined) {
|
|
11
|
+
return undefined;
|
|
12
|
+
}
|
|
12
13
|
for (let i = 0; i < 3; i++) {
|
|
13
14
|
try {
|
|
14
15
|
const response = await this.youtube.videos.list({
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { TriangleUtilsConfig } from "./types/TriangleUtilsConfig.js";
|
|
2
|
+
import { UtilsDynamoDB } from "./UtilsDynamoDB.js";
|
|
3
|
+
import { UtilsS3 } from "./UtilsS3.js";
|
|
4
|
+
import { UtilsBedrock } from "./UtilsBedrock.js";
|
|
5
|
+
import { UtilsBee } from "./UtilsBee.js";
|
|
6
|
+
import { UtilsS3Vectors } from "./UtilsS3Vectors.js";
|
|
7
|
+
import { UtilsCognito } from "./UtilsCognito.js";
|
|
8
|
+
import { UtilsMisc } from "./UtilsMisc.js";
|
|
9
|
+
import { UtilsYoutube } from "./UtilsYoutube.js";
|
|
10
|
+
export declare class TriangleUtils extends UtilsMisc {
|
|
11
|
+
readonly dynamodb: UtilsDynamoDB;
|
|
12
|
+
readonly s3: UtilsS3;
|
|
13
|
+
readonly s3vectors: UtilsS3Vectors;
|
|
14
|
+
readonly bedrock: UtilsBedrock;
|
|
15
|
+
readonly cognito: UtilsCognito;
|
|
16
|
+
readonly bee: UtilsBee;
|
|
17
|
+
readonly youtube: UtilsYoutube;
|
|
18
|
+
constructor(config: TriangleUtilsConfig);
|
|
19
|
+
}
|
|
20
|
+
export * from "./types/TriangleUtilsConfig.js";
|
|
21
|
+
export * from "./UtilsMisc.js";
|
|
22
|
+
export * from "./UtilsDynamoDB.js";
|
|
23
|
+
export * from "./UtilsS3.js";
|
|
24
|
+
export * from "./UtilsBedrock.js";
|
|
25
|
+
export * from "./UtilsBee.js";
|
|
26
|
+
export * from "./UtilsS3Vectors.js";
|
|
27
|
+
export * from "./UtilsCognito.js";
|
|
28
|
+
export * from "./UtilsYoutube.js";
|