triangle-utils 1.3.4 → 1.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/dist/src/UtilsBedrock.d.ts +8 -0
  2. package/dist/src/{Utils_Bedrock.js → UtilsBedrock.js} +7 -4
  3. package/dist/src/UtilsBee.d.ts +8 -0
  4. package/dist/src/{Utils_Bee.js → UtilsBee.js} +13 -8
  5. package/dist/src/UtilsCognito.d.ts +12 -0
  6. package/dist/src/UtilsCognito.js +110 -0
  7. package/dist/src/{Utils_DynamoDB.d.ts → UtilsDynamoDB.d.ts} +6 -8
  8. package/dist/src/{Utils_DynamoDB.js → UtilsDynamoDB.js} +18 -18
  9. package/dist/src/UtilsMisc.d.ts +32 -0
  10. package/dist/src/UtilsMisc.js +156 -0
  11. package/dist/src/{Utils_S3.d.ts → UtilsS3.d.ts} +3 -5
  12. package/dist/src/{Utils_S3.js → UtilsS3.js} +3 -3
  13. package/dist/src/{Utils_S3Vectors.d.ts → UtilsS3Vectors.d.ts} +5 -8
  14. package/dist/src/{Utils_S3Vectors.js → UtilsS3Vectors.js} +4 -4
  15. package/dist/src/UtilsYoutube.d.ts +6 -0
  16. package/dist/src/{Utils_Youtube.js → UtilsYoutube.js} +8 -7
  17. package/dist/src/index.d.ts +28 -0
  18. package/dist/src/index.js +36 -0
  19. package/dist/src/types/{Config.d.ts → TriangleUtilsConfig.d.ts} +1 -3
  20. package/package.json +9 -9
  21. package/src/{Utils_Bedrock.ts → UtilsBedrock.ts} +10 -8
  22. package/src/{Utils_Bee.ts → UtilsBee.ts} +21 -15
  23. package/src/UtilsCognito.ts +116 -0
  24. package/src/{Utils_DynamoDB.ts → UtilsDynamoDB.ts} +22 -23
  25. package/src/UtilsMisc.ts +188 -0
  26. package/src/{Utils_S3.ts → UtilsS3.ts} +4 -5
  27. package/src/{Utils_S3Vectors.ts → UtilsS3Vectors.ts} +6 -7
  28. package/src/{Utils_Youtube.ts → UtilsYoutube.ts} +9 -9
  29. package/src/index.ts +43 -0
  30. package/src/types/TriangleUtilsConfig.ts +6 -0
  31. package/test/Utils.test.ts +135 -24
  32. package/vite.config.ts +10 -0
  33. package/dist/src/Utils.d.ts +0 -17
  34. package/dist/src/Utils.js +0 -24
  35. package/dist/src/Utils_Bedrock.d.ts +0 -10
  36. package/dist/src/Utils_Bee.d.ts +0 -11
  37. package/dist/src/Utils_Misc.d.ts +0 -30
  38. package/dist/src/Utils_Misc.js +0 -627
  39. package/dist/src/Utils_Youtube.d.ts +0 -8
  40. package/src/Utils.ts +0 -28
  41. package/src/Utils_Misc.ts +0 -678
  42. package/src/types/Config.ts +0 -9
  43. /package/dist/src/types/{Config.js → TriangleUtilsConfig.js} +0 -0
@@ -0,0 +1,8 @@
1
+ export declare class UtilsBedrock {
2
+ private readonly bedrock;
3
+ private readonly text_decoder;
4
+ constructor(region: string);
5
+ llama_invoke(model_id: string, prompt: string, temperature?: number, max_gen_len?: number, top_p?: number): Promise<string | undefined>;
6
+ gpt_converse(model_id: string, prompt: string, temperature?: number, max_gen_len?: number, top_p?: number): Promise<string | undefined>;
7
+ titan_invoke(text: string): Promise<number[] | undefined>;
8
+ }
@@ -1,9 +1,9 @@
1
1
  import { BedrockRuntime } from "@aws-sdk/client-bedrock-runtime";
2
- export class Utils_Bedrock {
2
+ export class UtilsBedrock {
3
3
  bedrock;
4
4
  text_decoder;
5
- constructor(config) {
6
- this.bedrock = new BedrockRuntime({ region: config.region });
5
+ constructor(region) {
6
+ this.bedrock = new BedrockRuntime({ region: region });
7
7
  this.text_decoder = new TextDecoder();
8
8
  }
9
9
  async llama_invoke(model_id, prompt, temperature = 0.5, max_gen_len = 512, top_p = 0.9) {
@@ -74,7 +74,10 @@ export class Utils_Bedrock {
74
74
  contentType: "application/json"
75
75
  });
76
76
  const response = JSON.parse(this.text_decoder.decode(output.body));
77
- return response.embedding;
77
+ if (response.embedding !== undefined && Array.isArray(response.embedding) && response.embedding.every((i) => typeof i === "number")) {
78
+ const embedding = response.embedding;
79
+ return embedding;
80
+ }
78
81
  }
79
82
  catch (error) {
80
83
  if (error instanceof Error) {
@@ -0,0 +1,8 @@
1
+ export declare class UtilsBee {
2
+ private readonly bee;
3
+ private readonly text_decoder;
4
+ constructor(scraping_bee_api_key: string | undefined);
5
+ get(url: string, params?: Record<string, string | number | boolean>): Promise<string | undefined>;
6
+ google_search(query: string, news?: boolean): Promise<Record<string, any>[] | undefined>;
7
+ youtube_search(query: string, options?: {}): Promise<Record<string, any>[] | undefined>;
8
+ }
@@ -1,11 +1,9 @@
1
1
  import { ScrapingBeeClient } from "scrapingbee";
2
- export class Utils_Bee {
3
- config;
2
+ export class UtilsBee {
4
3
  bee;
5
4
  text_decoder;
6
- constructor(config) {
7
- this.config = config;
8
- this.bee = config.scraping_bee_api_key !== undefined ? new ScrapingBeeClient(config.scraping_bee_api_key) : undefined;
5
+ constructor(scraping_bee_api_key) {
6
+ this.bee = scraping_bee_api_key !== undefined ? new ScrapingBeeClient(scraping_bee_api_key) : undefined;
9
7
  this.text_decoder = new TextDecoder();
10
8
  }
11
9
  async get(url, params = {}) {
@@ -14,7 +12,7 @@ export class Utils_Bee {
14
12
  }
15
13
  for (let i = 0; i < 3; i++) {
16
14
  try {
17
- const response = await this.bee.get({
15
+ const response = await this.bee.htmlApi({
18
16
  url: url,
19
17
  params: params
20
18
  });
@@ -51,12 +49,19 @@ export class Utils_Bee {
51
49
  console.log("Failed to Google:", response.status);
52
50
  continue;
53
51
  }
54
- const data = response.data;
52
+ const data = JSON.parse(response.data.toString("utf-8"));
53
+ if (data.news_results === undefined || data.organic_results === undefined) {
54
+ continue;
55
+ }
55
56
  const results = news ? data.news_results : data.organic_results;
56
57
  if (results === undefined) {
57
58
  console.log("Failed to Google.");
58
59
  continue;
59
60
  }
61
+ if (!Array.isArray(results)) {
62
+ console.log("Results are not an array.");
63
+ return;
64
+ }
60
65
  return results;
61
66
  }
62
67
  catch {
@@ -80,7 +85,7 @@ export class Utils_Bee {
80
85
  console.log("Failed to YouTube Search:", response.status);
81
86
  continue;
82
87
  }
83
- const data = response.data;
88
+ const data = JSON.parse(response.data.toString("utf-8"));
84
89
  const results = data.results;
85
90
  if (results === undefined) {
86
91
  console.log("Failed to YouTube Search.");
@@ -0,0 +1,12 @@
1
+ export declare class UtilsCognito {
2
+ private readonly cognito;
3
+ constructor(region: string);
4
+ admin_get_user(user_pool_id: string, username: string): Promise<import("@aws-sdk/client-cognito-identity-provider").AdminGetUserCommandOutput | undefined>;
5
+ admin_create_user(user_pool_id: string, username: string, user_id: string): Promise<Error | import("@aws-sdk/client-cognito-identity-provider").AdminCreateUserCommandOutput>;
6
+ admin_reset_password(user_pool_id: string, username: string): Promise<Error | import("@aws-sdk/client-cognito-identity-provider").AdminCreateUserCommandOutput>;
7
+ get_user_id(token: string): Promise<string | undefined>;
8
+ get_new_token(client_id: string, client_secret: string, refresh_token: string): Promise<string | undefined>;
9
+ sign_out(token: string): Promise<void>;
10
+ admin_disable_user(user_pool_id: string, username: string): Promise<void>;
11
+ admin_delete_user(user_pool_id: string, username: string): Promise<void>;
12
+ }
@@ -0,0 +1,110 @@
1
+ import { CognitoIdentityProvider, NotAuthorizedException } from "@aws-sdk/client-cognito-identity-provider";
2
+ export class UtilsCognito {
3
+ cognito;
4
+ constructor(region) {
5
+ this.cognito = new CognitoIdentityProvider({ region: region });
6
+ }
7
+ async admin_get_user(user_pool_id, username) {
8
+ try {
9
+ return await this.cognito.adminGetUser({
10
+ UserPoolId: user_pool_id,
11
+ Username: username
12
+ });
13
+ }
14
+ catch (error) {
15
+ return undefined;
16
+ }
17
+ }
18
+ async admin_create_user(user_pool_id, username, user_id) {
19
+ try {
20
+ return await this.cognito.adminCreateUser({
21
+ UserPoolId: user_pool_id,
22
+ Username: username,
23
+ UserAttributes: [
24
+ { Name: "preferred_username", Value: user_id },
25
+ { Name: "email_verified", Value: "true" }
26
+ ]
27
+ });
28
+ }
29
+ catch (error) {
30
+ if (error instanceof Error) {
31
+ return new Error(error.toString().replace("Exception: ", ""));
32
+ }
33
+ return new Error();
34
+ }
35
+ }
36
+ async admin_reset_password(user_pool_id, username) {
37
+ const user = await this.admin_get_user(user_pool_id, username);
38
+ if (user === undefined) {
39
+ return new Error("User does not exist.");
40
+ }
41
+ try {
42
+ return await this.cognito.adminCreateUser({
43
+ UserPoolId: user_pool_id,
44
+ Username: username,
45
+ MessageAction: "RESEND"
46
+ });
47
+ }
48
+ catch (error) {
49
+ if (error instanceof Error) {
50
+ return new Error(error.toString().replace("Exception: ", ""));
51
+ }
52
+ return new Error();
53
+ }
54
+ }
55
+ async get_user_id(token) {
56
+ try {
57
+ const user_info = await this.cognito.getUser({
58
+ AccessToken: token
59
+ });
60
+ const user_attributes = user_info.UserAttributes;
61
+ if (user_attributes === undefined) {
62
+ return undefined;
63
+ }
64
+ const user_attribute = user_attributes.filter(attribute => attribute.Name === "preferred_username")[0];
65
+ const user_id = user_attribute.Value;
66
+ return user_id;
67
+ }
68
+ catch (error) {
69
+ if (error instanceof NotAuthorizedException) {
70
+ return undefined;
71
+ }
72
+ throw error;
73
+ }
74
+ }
75
+ async get_new_token(client_id, client_secret, refresh_token) {
76
+ try {
77
+ const response = await this.cognito.getTokensFromRefreshToken({
78
+ ClientId: client_id,
79
+ ClientSecret: client_secret,
80
+ RefreshToken: refresh_token
81
+ });
82
+ const new_auth_result = response.AuthenticationResult;
83
+ if (new_auth_result === undefined) {
84
+ return undefined;
85
+ }
86
+ const new_token = new_auth_result.AccessToken;
87
+ return new_token;
88
+ }
89
+ catch (error) {
90
+ return undefined;
91
+ }
92
+ }
93
+ async sign_out(token) {
94
+ await this.cognito.globalSignOut({
95
+ AccessToken: token
96
+ });
97
+ }
98
+ async admin_disable_user(user_pool_id, username) {
99
+ await this.cognito.adminDisableUser({
100
+ UserPoolId: user_pool_id,
101
+ Username: username,
102
+ });
103
+ }
104
+ async admin_delete_user(user_pool_id, username) {
105
+ await this.cognito.adminDeleteUser({
106
+ UserPoolId: user_pool_id,
107
+ Username: username,
108
+ });
109
+ }
110
+ }
@@ -1,8 +1,6 @@
1
- import { DynamoDB } from "@aws-sdk/client-dynamodb";
2
- import { Config } from "./types/Config.js";
3
- export declare class Utils_DynamoDB {
4
- readonly dynamodb: DynamoDB;
5
- constructor(config: Config);
1
+ export declare class UtilsDynamoDB {
2
+ private readonly dynamodb;
3
+ constructor(region: string);
6
4
  scan(table: string, options?: {
7
5
  filters?: Record<string, any>;
8
6
  undefined_attribute_names?: string[];
@@ -19,8 +17,8 @@ export declare class Utils_DynamoDB {
19
17
  query_prefix(table: string, primary_key: Record<string, any>, secondary_key_prefix: Record<string, string>, options?: {
20
18
  reverse?: boolean;
21
19
  compile?: boolean;
22
- }): Promise<Record<string, any>[] | undefined>;
23
- query_range(table: string, primary_key: Record<string, any>, secondary_key_range: Record<string, string | number[]>, options?: {
20
+ }): Promise<Record<string, any>[]>;
21
+ query_range(table: string, primary_key: Record<string, any>, secondary_key_range: Record<string, (string | number)[]>, options?: {
24
22
  reverse?: boolean;
25
23
  compile?: boolean;
26
24
  }): Promise<Record<string, any>[]>;
@@ -28,7 +26,7 @@ export declare class Utils_DynamoDB {
28
26
  append(table: string, key: Record<string, any>, attributes: Record<string, any[]>): Promise<void>;
29
27
  add(table: string, key: Record<string, any>, attributes: Record<string, any[]>): Promise<void>;
30
28
  remove(table: string, key: Record<string, any>, attributes: string[]): Promise<void>;
31
- create(table: string, key: Record<string, any>, attributes?: Record<string, any[]>): Promise<void>;
29
+ create(table: string, key: Record<string, any>, attributes?: Record<string, any>): Promise<void>;
32
30
  delete(table: string, key: Record<string, any>): Promise<void>;
33
31
  duplicate_attribute(table: string, attribute_name: string, new_attribute_name: string): Promise<void>;
34
32
  remove_attribute(table: string, attribute_name: string): Promise<void>;
@@ -1,22 +1,22 @@
1
1
  import { DynamoDB } from "@aws-sdk/client-dynamodb";
2
- function convert_output(dynamoobject) {
3
- if (dynamoobject.S !== undefined) {
4
- return dynamoobject.S;
2
+ function convert_output(dynamodb_output) {
3
+ if (dynamodb_output.S !== undefined) {
4
+ return dynamodb_output.S;
5
5
  }
6
- else if (dynamoobject.N !== undefined) {
7
- return Number(dynamoobject.N);
6
+ else if (dynamodb_output.N !== undefined) {
7
+ return Number(dynamodb_output.N);
8
8
  }
9
- else if (dynamoobject.L !== undefined) {
10
- return dynamoobject.L.map((a) => convert_output(a));
9
+ else if (dynamodb_output.L !== undefined) {
10
+ return dynamodb_output.L.map((a) => convert_output(a));
11
11
  }
12
- else if (dynamoobject.SS !== undefined) {
13
- return new Set(dynamoobject.SS);
12
+ else if (dynamodb_output.SS !== undefined) {
13
+ return new Set(dynamodb_output.SS);
14
14
  }
15
- else if (dynamoobject.M !== undefined) {
16
- return Object.fromEntries(Object.entries(dynamoobject.M).map(([key, value]) => [key, convert_output(value)]));
15
+ else if (dynamodb_output.M !== undefined) {
16
+ return Object.fromEntries(Object.entries(dynamodb_output.M).map(([key, value]) => [key, convert_output(value)]));
17
17
  }
18
- else if (dynamoobject.BOOL !== undefined) {
19
- return dynamoobject.BOOL;
18
+ else if (dynamodb_output.BOOL !== undefined) {
19
+ return dynamodb_output.BOOL;
20
20
  }
21
21
  return undefined;
22
22
  }
@@ -82,10 +82,10 @@ async function compile_pages(request, f, compile = true) {
82
82
  last_eval_key = response.LastEvaluatedKey;
83
83
  }
84
84
  }
85
- export class Utils_DynamoDB {
85
+ export class UtilsDynamoDB {
86
86
  dynamodb;
87
- constructor(config) {
88
- this.dynamodb = new DynamoDB({ region: config.region });
87
+ constructor(region) {
88
+ this.dynamodb = new DynamoDB({ region: region });
89
89
  }
90
90
  async scan(table, options = {}) {
91
91
  const filters = options.filters !== undefined ? options.filters : {};
@@ -201,12 +201,12 @@ export class Utils_DynamoDB {
201
201
  const reverse = options.reverse !== undefined ? options.reverse : false;
202
202
  const compile = options.compile !== undefined ? options.compile : true;
203
203
  if (Object.keys(primary_key).length !== 1 || Object.keys(secondary_key_prefix).length !== 1) {
204
- return undefined;
204
+ return [];
205
205
  }
206
206
  const converted_primary_value = convert_input(Object.values(primary_key)[0]);
207
207
  const converted_secondary_prefix_value = convert_input(Object.values(secondary_key_prefix)[0]);
208
208
  if (converted_primary_value === undefined || converted_secondary_prefix_value === undefined) {
209
- return undefined;
209
+ return [];
210
210
  }
211
211
  const request = {
212
212
  TableName: table,
@@ -0,0 +1,32 @@
1
+ import * as nodemailer from "nodemailer";
2
+ import { TriangleUtilsConfig } from "./types/TriangleUtilsConfig.js";
3
+ interface Encryption {
4
+ iv: string;
5
+ ciphertext: string;
6
+ }
7
+ export declare class UtilsMisc {
8
+ readonly config: TriangleUtilsConfig;
9
+ readonly transporter: nodemailer.Transporter<import("nodemailer/lib/smtp-transport").SentMessageInfo, import("nodemailer/lib/smtp-transport").Options> | undefined;
10
+ constructor(config: TriangleUtilsConfig);
11
+ send_email(recipient: string, subject: string, text: string): Promise<boolean>;
12
+ admin_alert(text: string): Promise<boolean>;
13
+ safe_run(f: () => Promise<any>): Promise<void>;
14
+ iterate<T>(inputs: T[], f: (input: T) => Promise<any>, concurrency?: number, print_indices?: boolean): Promise<void>;
15
+ static wait(duration: number): Promise<unknown>;
16
+ static get_current_time(): string;
17
+ static get_current_milliseconds(): number;
18
+ static sha256(input: string): Promise<string>;
19
+ static encrypt(secret_key: string, text: string): Encryption;
20
+ static decrypt(secret_key: string, encryption: Encryption): string;
21
+ static get_secret_hash(username: string, cognito_client_id: string, cognito_client_secret: string): string;
22
+ static get_chunk_indices(text_length: number, max_length?: number, overlap?: number): [number, number][];
23
+ static chunkify(text: string, max_length?: number, overlap?: number): {
24
+ chunk_index: [number, number];
25
+ chunk_text: string;
26
+ }[];
27
+ static chunkify_lined(text: string, max_length?: number, overlap?: number): {
28
+ chunk_index: [number, number];
29
+ chunk_text: string;
30
+ }[];
31
+ }
32
+ export {};
@@ -0,0 +1,156 @@
1
+ import * as nodemailer from "nodemailer";
2
+ import * as crypto from "crypto";
3
+ export class UtilsMisc {
4
+ config;
5
+ transporter;
6
+ constructor(config) {
7
+ this.config = config;
8
+ if (config.google_email !== undefined && config.google_app_password !== undefined) {
9
+ this.transporter = nodemailer.createTransport({
10
+ service: "Gmail",
11
+ host: "smtp.gmail.com",
12
+ port: 465,
13
+ secure: true,
14
+ auth: {
15
+ user: config.google_email,
16
+ pass: config.google_app_password,
17
+ }
18
+ });
19
+ }
20
+ }
21
+ async send_email(recipient, subject, text) {
22
+ if (this.transporter === undefined) {
23
+ return false;
24
+ }
25
+ for (let i = 0; i < 3; i++) {
26
+ try {
27
+ await this.transporter.sendMail({
28
+ from: this.config.alerts_email,
29
+ to: recipient,
30
+ subject: subject,
31
+ text: text
32
+ });
33
+ return true;
34
+ }
35
+ catch (error) {
36
+ console.log("EMAIL ERROR", error);
37
+ }
38
+ }
39
+ return false;
40
+ }
41
+ async admin_alert(text) {
42
+ console.log("ADMIN ALERT:", text);
43
+ return await this.send_email(this.config.google_email, "ADMIN ALERT", text);
44
+ }
45
+ async safe_run(f) {
46
+ try {
47
+ await f();
48
+ }
49
+ catch (error) {
50
+ if (!(error instanceof Error)) {
51
+ return;
52
+ }
53
+ if (error.stack !== undefined) {
54
+ await this.admin_alert(error.stack.toString());
55
+ }
56
+ else {
57
+ await this.admin_alert(error.toString());
58
+ }
59
+ }
60
+ }
61
+ async iterate(inputs, f, concurrency = 1, print_indices = false) {
62
+ let index = 0;
63
+ let iterators = [];
64
+ for (let i = 0; i < concurrency; i++) {
65
+ iterators.push((async () => {
66
+ while (index < inputs.length) {
67
+ index += 1;
68
+ if (print_indices) {
69
+ console.log(i + ":" + (index - 1) + "/" + inputs.length);
70
+ }
71
+ await this.safe_run(() => f(inputs[index - 1]));
72
+ }
73
+ })());
74
+ }
75
+ await Promise.all(iterators);
76
+ }
77
+ static async wait(duration) {
78
+ return new Promise(resolve => setTimeout(resolve, duration));
79
+ }
80
+ static get_current_time() {
81
+ return (new Date()).toISOString();
82
+ }
83
+ static get_current_milliseconds() {
84
+ return (new Date()).getTime();
85
+ }
86
+ static async sha256(input) {
87
+ const text_encoder = new TextEncoder();
88
+ const input_buffer = text_encoder.encode(input);
89
+ const hash_buffer = await crypto.subtle.digest("SHA-256", input_buffer);
90
+ const hash_array = Array.from(new Uint8Array(hash_buffer));
91
+ const hash = hash_array.map(b => b.toString(16).padStart(2, "0")).join("");
92
+ return hash;
93
+ }
94
+ static encrypt(secret_key, text) {
95
+ const iv = crypto.randomBytes(16);
96
+ const cipher = crypto.createCipheriv("aes-256-cbc", Buffer.from(secret_key, "hex"), iv);
97
+ let ciphertext = cipher.update(text, "utf8", "hex");
98
+ ciphertext += cipher.final("hex");
99
+ return {
100
+ iv: iv.toString("hex"),
101
+ ciphertext: ciphertext
102
+ };
103
+ }
104
+ static decrypt(secret_key, encryption) {
105
+ const iv = encryption.iv;
106
+ const ciphertext = encryption.ciphertext;
107
+ const decipher = crypto.createDecipheriv("aes-256-cbc", Buffer.from(secret_key, "hex"), Buffer.from(iv, "hex"));
108
+ let text = decipher.update(ciphertext, "hex", "utf8");
109
+ text += decipher.final("utf8");
110
+ return text;
111
+ }
112
+ static get_secret_hash(username, cognito_client_id, cognito_client_secret) {
113
+ const secret_hash = crypto.createHmac("sha256", cognito_client_secret)
114
+ .update(username + cognito_client_id)
115
+ .digest("base64");
116
+ return secret_hash;
117
+ }
118
+ static get_chunk_indices(text_length, max_length = 2500, overlap = 50) {
119
+ const num_chunks = Math.ceil((text_length + overlap) / max_length);
120
+ const chunk_length = Math.ceil((text_length - overlap) / num_chunks + overlap);
121
+ const chunk_indices = [];
122
+ for (let i = 0; i < num_chunks; i++) {
123
+ chunk_indices.push([
124
+ (chunk_length - overlap) * i,
125
+ (chunk_length - overlap) * i + chunk_length
126
+ ]);
127
+ }
128
+ return chunk_indices;
129
+ }
130
+ static chunkify(text, max_length = 2500, overlap = 50) {
131
+ const chunk_indices = this.get_chunk_indices(text.length, max_length, overlap);
132
+ const chunks = chunk_indices.map(chunk_index => ({
133
+ chunk_index: chunk_index,
134
+ chunk_text: text.substring(...chunk_index)
135
+ }));
136
+ return chunks;
137
+ }
138
+ static chunkify_lined(text, max_length = 2500, overlap = 50) {
139
+ const chunk_indices = this.get_chunk_indices(text.length, max_length, overlap);
140
+ const lined_chunk_indices = [];
141
+ for (const chunk_index of chunk_indices) {
142
+ const lined_chunk_index = [
143
+ text.lastIndexOf("\n", chunk_index[0]) + 1,
144
+ text.indexOf("\n", chunk_index[1]) + 1
145
+ ];
146
+ if (lined_chunk_index[1] < lined_chunk_index[0]) {
147
+ lined_chunk_index[1] += text.length;
148
+ }
149
+ lined_chunk_indices.push(lined_chunk_index);
150
+ }
151
+ return lined_chunk_indices.map(chunk_index => ({
152
+ chunk_index: chunk_index,
153
+ chunk_text: text.substring(...chunk_index)
154
+ }));
155
+ }
156
+ }
@@ -1,8 +1,6 @@
1
- import { S3 } from "@aws-sdk/client-s3";
2
- import { Config } from "./types/Config.js";
3
- export declare class Utils_S3 {
4
- readonly s3: S3;
5
- constructor(config: Config);
1
+ export declare class UtilsS3 {
2
+ private readonly s3;
3
+ constructor(region: string);
6
4
  file_exists(s3_filename: string): Promise<boolean | undefined>;
7
5
  get_file(s3_filename: string): Promise<string | undefined>;
8
6
  create_file(s3_filename: string, content: string | Buffer, options?: {
@@ -9,10 +9,10 @@ function parse_s3_filename(s3_filename) {
9
9
  const filename = bucket_filename.split("/").slice(1).join("/");
10
10
  return { Bucket: bucket, Key: filename };
11
11
  }
12
- export class Utils_S3 {
12
+ export class UtilsS3 {
13
13
  s3;
14
- constructor(config) {
15
- this.s3 = new S3({ region: config.region });
14
+ constructor(region) {
15
+ this.s3 = new S3({ region: region });
16
16
  }
17
17
  async file_exists(s3_filename) {
18
18
  const s3_key = parse_s3_filename(s3_filename);
@@ -1,13 +1,11 @@
1
- import { S3Vectors } from "@aws-sdk/client-s3vectors";
2
- import { Config } from "./types/Config.js";
3
- interface Vector {
1
+ export interface Vector {
4
2
  key: string;
5
3
  data: number[];
6
4
  metadata: any;
7
5
  }
8
- export declare class Utils_S3Vectors {
9
- readonly s3vectors: S3Vectors;
10
- constructor(config: Config);
6
+ export declare class UtilsS3Vectors {
7
+ private readonly s3vectors;
8
+ constructor(region: string);
11
9
  scan(vector_bucket: string, vector_index: string): Promise<string[]>;
12
10
  get(vector_bucket: string, vector_index: string, vector_key: string): Promise<Vector | undefined>;
13
11
  batch_get(vector_bucket: string, vector_index: string, vector_keys: string[]): Promise<(Vector | undefined)[]>;
@@ -15,6 +13,5 @@ export declare class Utils_S3Vectors {
15
13
  batch_create(vector_bucket: string, vector_index: string, vectors: Vector[]): Promise<void>;
16
14
  delete(vector_bucket: string, vector_index: string, vector_key: string): Promise<void>;
17
15
  batch_delete(vector_bucket: string, vector_index: string, vector_keys: string[]): Promise<void>;
18
- query(vector_bucket: string, vector_index: string, data: number[], num_vectors: number, filters?: Record<string, any>): Promise<import("@aws-sdk/client-s3vectors").QueryOutputVector[] | undefined>;
16
+ query(vector_bucket: string, vector_index: string, data: number[], num_vectors: number, filters?: Record<string, any> | undefined): Promise<import("@aws-sdk/client-s3vectors").QueryOutputVector[] | undefined>;
19
17
  }
20
- export {};
@@ -1,8 +1,8 @@
1
1
  import { S3Vectors } from "@aws-sdk/client-s3vectors";
2
- export class Utils_S3Vectors {
2
+ export class UtilsS3Vectors {
3
3
  s3vectors;
4
- constructor(config) {
5
- this.s3vectors = new S3Vectors({ region: config.region });
4
+ constructor(region) {
5
+ this.s3vectors = new S3Vectors({ region: region });
6
6
  }
7
7
  async scan(vector_bucket, vector_index) {
8
8
  const vector_keys = [];
@@ -114,7 +114,7 @@ export class Utils_S3Vectors {
114
114
  });
115
115
  }
116
116
  }
117
- async query(vector_bucket, vector_index, data, num_vectors, filters = {}) {
117
+ async query(vector_bucket, vector_index, data, num_vectors, filters = undefined) {
118
118
  const response = await this.s3vectors.queryVectors({
119
119
  vectorBucketName: vector_bucket,
120
120
  indexName: vector_index,
@@ -0,0 +1,6 @@
1
+ import { youtube_v3 } from "googleapis";
2
+ export declare class UtilsYoutube {
3
+ readonly youtube: youtube_v3.Youtube | undefined;
4
+ constructor(youtube_api_key: string | undefined);
5
+ batch_get_videos(video_ids: string[], attributes?: string[]): Promise<youtube_v3.Schema$Video[] | undefined>;
6
+ }
@@ -1,14 +1,15 @@
1
1
  import { youtube_v3 } from "googleapis";
2
- export class Utils_Youtube {
3
- config;
2
+ export class UtilsYoutube {
4
3
  youtube;
5
- constructor(config) {
6
- this.config = config;
7
- this.youtube = new youtube_v3.Youtube({
8
- auth: config.youtube_api_key
9
- });
4
+ constructor(youtube_api_key) {
5
+ this.youtube = youtube_api_key !== undefined ? new youtube_v3.Youtube({
6
+ auth: youtube_api_key
7
+ }) : undefined;
10
8
  }
11
9
  async batch_get_videos(video_ids, attributes = ["snippet", "contentDetails"]) {
10
+ if (this.youtube === undefined) {
11
+ return undefined;
12
+ }
12
13
  for (let i = 0; i < 3; i++) {
13
14
  try {
14
15
  const response = await this.youtube.videos.list({
@@ -0,0 +1,28 @@
1
+ import { TriangleUtilsConfig } from "./types/TriangleUtilsConfig.js";
2
+ import { UtilsDynamoDB } from "./UtilsDynamoDB.js";
3
+ import { UtilsS3 } from "./UtilsS3.js";
4
+ import { UtilsBedrock } from "./UtilsBedrock.js";
5
+ import { UtilsBee } from "./UtilsBee.js";
6
+ import { UtilsS3Vectors } from "./UtilsS3Vectors.js";
7
+ import { UtilsCognito } from "./UtilsCognito.js";
8
+ import { UtilsMisc } from "./UtilsMisc.js";
9
+ import { UtilsYoutube } from "./UtilsYoutube.js";
10
+ export declare class TriangleUtils extends UtilsMisc {
11
+ readonly dynamodb: UtilsDynamoDB;
12
+ readonly s3: UtilsS3;
13
+ readonly s3vectors: UtilsS3Vectors;
14
+ readonly bedrock: UtilsBedrock;
15
+ readonly cognito: UtilsCognito;
16
+ readonly bee: UtilsBee;
17
+ readonly youtube: UtilsYoutube;
18
+ constructor(config: TriangleUtilsConfig);
19
+ }
20
+ export * from "./types/TriangleUtilsConfig.js";
21
+ export * from "./UtilsMisc.js";
22
+ export * from "./UtilsDynamoDB.js";
23
+ export * from "./UtilsS3.js";
24
+ export * from "./UtilsBedrock.js";
25
+ export * from "./UtilsBee.js";
26
+ export * from "./UtilsS3Vectors.js";
27
+ export * from "./UtilsCognito.js";
28
+ export * from "./UtilsYoutube.js";