triangle-utils 1.4.148 → 1.4.150
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/UtilsRDS.d.ts +19 -0
- package/dist/src/UtilsRDS.js +20 -1
- package/dist/src/UtilsS3.d.ts +1 -1
- package/dist/src/UtilsS3.js +11 -11
- package/dist/src/UtilsTextract.d.ts +6 -0
- package/dist/src/UtilsTextract.js +45 -0
- package/dist/src/f.js +7 -1
- package/dist/src/index.d.ts +2 -0
- package/dist/src/index.js +3 -0
- package/f.txt +0 -0
- package/package.json +2 -1
- package/src/UtilsRDS.ts +52 -11
- package/src/UtilsS3.ts +11 -11
- package/src/UtilsTextract.ts +50 -0
- package/src/f.ts +8 -1
- package/src/index.ts +3 -0
package/dist/src/UtilsRDS.d.ts
CHANGED
|
@@ -11,34 +11,53 @@ export declare class UtilsRDS {
|
|
|
11
11
|
}): Promise<any[]>;
|
|
12
12
|
exec(sql: string, options?: {
|
|
13
13
|
is_verbose?: boolean;
|
|
14
|
+
is_forced?: boolean;
|
|
15
|
+
}): Promise<any[]>;
|
|
16
|
+
scan(table_id: string, options?: {
|
|
17
|
+
filters?: Record<string, any>;
|
|
18
|
+
undefined_attribute_names?: string[];
|
|
19
|
+
defined_attribute_names?: string[];
|
|
20
|
+
attribute_names?: string[];
|
|
21
|
+
max_num_items?: number;
|
|
22
|
+
is_verbose?: boolean;
|
|
23
|
+
is_forced?: boolean;
|
|
14
24
|
}): Promise<any[]>;
|
|
15
25
|
get(table_id: string, primary_key: Record<string, any>, options?: {
|
|
16
26
|
attribute_names?: string[];
|
|
17
27
|
is_verbose?: boolean;
|
|
28
|
+
is_forced?: boolean;
|
|
18
29
|
}): Promise<Record<string, any> | undefined>;
|
|
19
30
|
batch_get(table_id: string, primary_key: Record<string, any[]>, options?: {
|
|
20
31
|
attribute_names?: string[];
|
|
21
32
|
is_verbose?: boolean;
|
|
33
|
+
is_forced?: boolean;
|
|
22
34
|
}): Promise<any[]>;
|
|
23
35
|
create(table_id: string, item: Record<string, any>, options?: {
|
|
24
36
|
is_verbose?: boolean;
|
|
37
|
+
is_forced?: boolean;
|
|
25
38
|
}): Promise<any[] | undefined>;
|
|
26
39
|
batch_create(table_id: string, items: Record<string, any>[], options?: {
|
|
27
40
|
is_verbose?: boolean;
|
|
41
|
+
is_forced?: boolean;
|
|
28
42
|
}): Promise<any[] | undefined>;
|
|
29
43
|
put(table_id: string, item: Record<string, any>, options?: {
|
|
30
44
|
is_verbose?: boolean;
|
|
45
|
+
is_forced?: boolean;
|
|
31
46
|
}): Promise<any[] | undefined>;
|
|
32
47
|
batch_put(table_id: string, items: Record<string, any>[], options?: {
|
|
33
48
|
is_verbose?: boolean;
|
|
49
|
+
is_forced?: boolean;
|
|
34
50
|
}): Promise<any[] | undefined>;
|
|
35
51
|
delete(table_id: string, primary_key: Record<string, any>, options?: {
|
|
36
52
|
is_verbose?: boolean;
|
|
53
|
+
is_forced?: boolean;
|
|
37
54
|
}): Promise<any[]>;
|
|
38
55
|
batch_delete(table_id: string, primary_key: Record<string, any[]>, options?: {
|
|
39
56
|
is_verbose?: boolean;
|
|
57
|
+
is_forced?: boolean;
|
|
40
58
|
}): Promise<any[]>;
|
|
41
59
|
set(table_id: string, primary_key: Record<string, any>, attributes: Record<string, any>, options?: {
|
|
42
60
|
is_verbose?: boolean;
|
|
61
|
+
is_forced?: boolean;
|
|
43
62
|
}): Promise<any[]>;
|
|
44
63
|
}
|
package/dist/src/UtilsRDS.js
CHANGED
|
@@ -86,10 +86,11 @@ export class UtilsRDS {
|
|
|
86
86
|
}
|
|
87
87
|
async exec(sql, options = {}) {
|
|
88
88
|
const is_verbose = Boolean(options.is_verbose);
|
|
89
|
+
const is_forced = Boolean(options.is_forced);
|
|
89
90
|
if (is_verbose) {
|
|
90
91
|
console.log(sql);
|
|
91
92
|
}
|
|
92
|
-
const items = sql.length <= 60000 ?
|
|
93
|
+
const items = (sql.length <= 60000 && !is_forced) ?
|
|
93
94
|
await this.rds_data.executeStatement({
|
|
94
95
|
resourceArn: this.rds_cluster_arn,
|
|
95
96
|
secretArn: this.rds_secret_arn,
|
|
@@ -115,6 +116,24 @@ export class UtilsRDS {
|
|
|
115
116
|
}
|
|
116
117
|
return items;
|
|
117
118
|
}
|
|
119
|
+
async scan(table_id, options = {}) {
|
|
120
|
+
const undefined_attribute_names = options.undefined_attribute_names || [];
|
|
121
|
+
const defined_attribute_names = options.defined_attribute_names || [];
|
|
122
|
+
const filters = options.filters || {};
|
|
123
|
+
const clauses_where = [
|
|
124
|
+
...Object.entries(filters).map(([key_name, value]) => key_name + " = " + convert_rds_input(value)),
|
|
125
|
+
...undefined_attribute_names.map(undefined_attribute_name => undefined_attribute_name + " IS NULL"),
|
|
126
|
+
...defined_attribute_names.map(undefined_attribute_name => undefined_attribute_name + " IS NOT NULL")
|
|
127
|
+
];
|
|
128
|
+
const sql = [
|
|
129
|
+
"SELECT " + (options.attribute_names === undefined ? "*" : options.attribute_names.join(", ")),
|
|
130
|
+
"FROM " + table_id,
|
|
131
|
+
(clauses_where.length !== 0 ? ("WHERE " + clauses_where.join(" AND ")) : ""),
|
|
132
|
+
(options.max_num_items !== undefined ? ("LIMIT " + options.max_num_items) : "")
|
|
133
|
+
].join("\n");
|
|
134
|
+
const items = await this.exec(sql, options);
|
|
135
|
+
return items;
|
|
136
|
+
}
|
|
118
137
|
async get(table_id, primary_key, options = {}) {
|
|
119
138
|
const sql = "SELECT " + (options.attribute_names === undefined ? "*" : options.attribute_names.join(", ")) + " FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, value]) => key_name + " = " + convert_rds_input(value)).join(" AND ");
|
|
120
139
|
const items = await this.exec(sql, options);
|
package/dist/src/UtilsS3.d.ts
CHANGED
|
@@ -8,7 +8,7 @@ export declare class UtilsS3 {
|
|
|
8
8
|
private readonly s3;
|
|
9
9
|
constructor(region: string);
|
|
10
10
|
get_client(): S3;
|
|
11
|
-
get_s3_input(s3_id: string): S3Input | undefined;
|
|
11
|
+
static get_s3_input(s3_id: string): S3Input | undefined;
|
|
12
12
|
get_headers(s3_id: string): Promise<{
|
|
13
13
|
DeleteMarker?: boolean | undefined;
|
|
14
14
|
AcceptRanges?: string | undefined;
|
package/dist/src/UtilsS3.js
CHANGED
|
@@ -10,7 +10,7 @@ export class UtilsS3 {
|
|
|
10
10
|
get_client() {
|
|
11
11
|
return this.s3;
|
|
12
12
|
}
|
|
13
|
-
get_s3_input(s3_id) {
|
|
13
|
+
static get_s3_input(s3_id) {
|
|
14
14
|
const [bucket, path] = (s3_id.match(/^s3\:\/\/([^\/]+)\/([^$]+)$/) || []).slice(1, 3);
|
|
15
15
|
if (bucket === undefined || path === undefined) {
|
|
16
16
|
return undefined;
|
|
@@ -18,7 +18,7 @@ export class UtilsS3 {
|
|
|
18
18
|
return { Bucket: bucket, Key: path };
|
|
19
19
|
}
|
|
20
20
|
async get_headers(s3_id) {
|
|
21
|
-
const s3_input =
|
|
21
|
+
const s3_input = UtilsS3.get_s3_input(s3_id);
|
|
22
22
|
if (s3_input === undefined) {
|
|
23
23
|
return undefined;
|
|
24
24
|
}
|
|
@@ -34,7 +34,7 @@ export class UtilsS3 {
|
|
|
34
34
|
}
|
|
35
35
|
}
|
|
36
36
|
async get(s3_id, encoding = "utf-8") {
|
|
37
|
-
const s3_input =
|
|
37
|
+
const s3_input = UtilsS3.get_s3_input(s3_id);
|
|
38
38
|
if (s3_input === undefined) {
|
|
39
39
|
return undefined;
|
|
40
40
|
}
|
|
@@ -55,7 +55,7 @@ export class UtilsS3 {
|
|
|
55
55
|
}
|
|
56
56
|
}
|
|
57
57
|
async get_buffer(s3_id) {
|
|
58
|
-
const s3_input =
|
|
58
|
+
const s3_input = UtilsS3.get_s3_input(s3_id);
|
|
59
59
|
if (s3_input === undefined) {
|
|
60
60
|
return undefined;
|
|
61
61
|
}
|
|
@@ -76,7 +76,7 @@ export class UtilsS3 {
|
|
|
76
76
|
}
|
|
77
77
|
}
|
|
78
78
|
async get_stream(s3_id) {
|
|
79
|
-
const s3_input =
|
|
79
|
+
const s3_input = UtilsS3.get_s3_input(s3_id);
|
|
80
80
|
if (s3_input === undefined) {
|
|
81
81
|
return undefined;
|
|
82
82
|
}
|
|
@@ -100,7 +100,7 @@ export class UtilsS3 {
|
|
|
100
100
|
}
|
|
101
101
|
async query_prefix(s3_id_prefix, options = {}) {
|
|
102
102
|
const compile = options.compile !== undefined ? options.compile : false;
|
|
103
|
-
const s3_input =
|
|
103
|
+
const s3_input = UtilsS3.get_s3_input(s3_id_prefix);
|
|
104
104
|
if (s3_input === undefined) {
|
|
105
105
|
return [];
|
|
106
106
|
}
|
|
@@ -140,7 +140,7 @@ export class UtilsS3 {
|
|
|
140
140
|
if (headers !== undefined) {
|
|
141
141
|
return undefined;
|
|
142
142
|
}
|
|
143
|
-
const s3_input =
|
|
143
|
+
const s3_input = UtilsS3.get_s3_input(s3_id);
|
|
144
144
|
if (s3_input === undefined) {
|
|
145
145
|
return undefined;
|
|
146
146
|
}
|
|
@@ -153,7 +153,7 @@ export class UtilsS3 {
|
|
|
153
153
|
async upload(s3_id, content, options = {}) {
|
|
154
154
|
const content_type_id = options.content_type_id !== undefined ? options.content_type_id : undefined;
|
|
155
155
|
const part_size = options.part_size !== undefined ? options.part_size : undefined;
|
|
156
|
-
const s3_input =
|
|
156
|
+
const s3_input = UtilsS3.get_s3_input(s3_id);
|
|
157
157
|
if (s3_input === undefined) {
|
|
158
158
|
return undefined;
|
|
159
159
|
}
|
|
@@ -169,7 +169,7 @@ export class UtilsS3 {
|
|
|
169
169
|
await upload.done();
|
|
170
170
|
}
|
|
171
171
|
async delete(s3_id) {
|
|
172
|
-
const s3_input =
|
|
172
|
+
const s3_input = UtilsS3.get_s3_input(s3_id);
|
|
173
173
|
if (s3_input === undefined) {
|
|
174
174
|
return undefined;
|
|
175
175
|
}
|
|
@@ -180,7 +180,7 @@ export class UtilsS3 {
|
|
|
180
180
|
if (headers === undefined) {
|
|
181
181
|
return undefined;
|
|
182
182
|
}
|
|
183
|
-
const s3_input =
|
|
183
|
+
const s3_input = UtilsS3.get_s3_input(s3_id);
|
|
184
184
|
if (s3_input === undefined) {
|
|
185
185
|
return undefined;
|
|
186
186
|
}
|
|
@@ -190,7 +190,7 @@ export class UtilsS3 {
|
|
|
190
190
|
}
|
|
191
191
|
async get_upload_url(s3_id, options = {}) {
|
|
192
192
|
const content_type_id = options.content_type_id !== undefined ? options.content_type_id : undefined;
|
|
193
|
-
const s3_input =
|
|
193
|
+
const s3_input = UtilsS3.get_s3_input(s3_id);
|
|
194
194
|
if (s3_input === undefined) {
|
|
195
195
|
return undefined;
|
|
196
196
|
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { Textract } from "@aws-sdk/client-textract";
|
|
2
|
+
import { UtilsMisc } from "./UtilsMisc.js";
|
|
3
|
+
import { UtilsS3 } from "./UtilsS3.js";
|
|
4
|
+
export class UtilsTextract {
|
|
5
|
+
textract;
|
|
6
|
+
constructor(region) {
|
|
7
|
+
this.textract = new Textract({ region: region });
|
|
8
|
+
}
|
|
9
|
+
async get_text(s3_id) {
|
|
10
|
+
const s3_input = UtilsS3.get_s3_input(s3_id);
|
|
11
|
+
if (s3_input === undefined) {
|
|
12
|
+
return undefined;
|
|
13
|
+
}
|
|
14
|
+
const request = {
|
|
15
|
+
DocumentLocation: {
|
|
16
|
+
S3Object: {
|
|
17
|
+
Bucket: s3_input.Bucket,
|
|
18
|
+
Name: s3_input.Key
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
};
|
|
22
|
+
const job_id = await this.textract.startDocumentTextDetection(request)
|
|
23
|
+
.then(response => response.JobId);
|
|
24
|
+
if (job_id === undefined) {
|
|
25
|
+
return;
|
|
26
|
+
}
|
|
27
|
+
let token = undefined;
|
|
28
|
+
const blocks = [];
|
|
29
|
+
while (true) {
|
|
30
|
+
const result = await this.textract.getDocumentTextDetection({
|
|
31
|
+
JobId: job_id,
|
|
32
|
+
NextToken: token
|
|
33
|
+
});
|
|
34
|
+
if (result.JobStatus === "IN_PROGRESS") {
|
|
35
|
+
UtilsMisc.wait(1000);
|
|
36
|
+
continue;
|
|
37
|
+
}
|
|
38
|
+
blocks.push(...(result.Blocks || []));
|
|
39
|
+
if (result.NextToken === undefined) {
|
|
40
|
+
return blocks;
|
|
41
|
+
}
|
|
42
|
+
token = result.NextToken;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
}
|
package/dist/src/f.js
CHANGED
|
@@ -74,4 +74,10 @@ const utils = new TriangleUtils(config);
|
|
|
74
74
|
// target_ids : ["TURNOUT-2026-11-03"]
|
|
75
75
|
// }, { is_verbose : true }))
|
|
76
76
|
// console.log(await utils.rds.exec("SELECT * FROM prisms", { is_verbose : true }))\
|
|
77
|
-
|
|
77
|
+
// await utils.rds.connect()
|
|
78
|
+
// console.log(await utils.rds.scan("web_articles", { is_verbose : true, is_forced : true, max_num_items : 100 }))
|
|
79
|
+
// await utils.rds.disconnect()
|
|
80
|
+
// await utils.twitter.get_user("1509572215251124226")
|
|
81
|
+
// .then(user => console.log(user))
|
|
82
|
+
// await utils.bedrock.cohere_invoke("cohere.embed-v4", "yyyyyuuuuuurrrrr")
|
|
83
|
+
// .then(response => console.log(response))
|
package/dist/src/index.d.ts
CHANGED
|
@@ -13,12 +13,14 @@ import { UtilsTwitter } from "./UtilsTwitter.js";
|
|
|
13
13
|
import { GlobalConfig } from "./types/GlobalConfig.js";
|
|
14
14
|
import { UtilsRDS } from "./UtilsRDS.js";
|
|
15
15
|
import { ApplicationConfig } from "./types/ApplicationConfig.js";
|
|
16
|
+
import { UtilsTextract } from "./UtilsTextract.js";
|
|
16
17
|
export declare class TriangleUtils extends UtilsMisc {
|
|
17
18
|
readonly dynamodb: UtilsDynamoDB;
|
|
18
19
|
readonly rds: UtilsRDS;
|
|
19
20
|
readonly s3: UtilsS3;
|
|
20
21
|
readonly s3vectors: UtilsS3Vectors;
|
|
21
22
|
readonly bedrock: UtilsBedrock;
|
|
23
|
+
readonly textract: UtilsTextract;
|
|
22
24
|
readonly cognito: UtilsCognito;
|
|
23
25
|
readonly ses: UtilsSES;
|
|
24
26
|
readonly bee: UtilsBee;
|
package/dist/src/index.js
CHANGED
|
@@ -11,12 +11,14 @@ import { UtilsXAI } from "./UtilsXAI.js";
|
|
|
11
11
|
import { UtilsAnthropic } from "./UtilsAnthropic.js";
|
|
12
12
|
import { UtilsTwitter } from "./UtilsTwitter.js";
|
|
13
13
|
import { UtilsRDS } from "./UtilsRDS.js";
|
|
14
|
+
import { UtilsTextract } from "./UtilsTextract.js";
|
|
14
15
|
export class TriangleUtils extends UtilsMisc {
|
|
15
16
|
dynamodb;
|
|
16
17
|
rds;
|
|
17
18
|
s3;
|
|
18
19
|
s3vectors;
|
|
19
20
|
bedrock;
|
|
21
|
+
textract;
|
|
20
22
|
cognito;
|
|
21
23
|
ses;
|
|
22
24
|
bee;
|
|
@@ -31,6 +33,7 @@ export class TriangleUtils extends UtilsMisc {
|
|
|
31
33
|
this.s3 = new UtilsS3(config.region);
|
|
32
34
|
this.s3vectors = new UtilsS3Vectors(config.region);
|
|
33
35
|
this.bedrock = new UtilsBedrock(config.region);
|
|
36
|
+
this.textract = new UtilsTextract(config.region);
|
|
34
37
|
this.cognito = new UtilsCognito(config.region);
|
|
35
38
|
this.ses = new UtilsSES(config.region);
|
|
36
39
|
this.bee = new UtilsBee(config.scraping_bee_api_key);
|
package/f.txt
ADDED
|
Binary file
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "triangle-utils",
|
|
3
|
-
"version": "1.4.
|
|
3
|
+
"version": "1.4.150",
|
|
4
4
|
"main": "dist/src/index.js",
|
|
5
5
|
"types": "dist/src/index.d.ts",
|
|
6
6
|
"directories": {
|
|
@@ -26,6 +26,7 @@
|
|
|
26
26
|
"@aws-sdk/client-s3vectors": "^3.953.0",
|
|
27
27
|
"@aws-sdk/client-secrets-manager": "^3.965.0",
|
|
28
28
|
"@aws-sdk/client-ses": "^3.1032.0",
|
|
29
|
+
"@aws-sdk/client-textract": "^3.1117.0",
|
|
29
30
|
"@aws-sdk/lib-storage": "^3.1106.0",
|
|
30
31
|
"@aws-sdk/s3-request-presigner": "^3.1106.0",
|
|
31
32
|
"@types/jsdom": "^28.0.1",
|
package/src/UtilsRDS.ts
CHANGED
|
@@ -100,14 +100,16 @@ export class UtilsRDS {
|
|
|
100
100
|
async exec(
|
|
101
101
|
sql : string,
|
|
102
102
|
options : {
|
|
103
|
-
is_verbose? : boolean
|
|
103
|
+
is_verbose? : boolean,
|
|
104
|
+
is_forced? : boolean
|
|
104
105
|
} = {}
|
|
105
106
|
) {
|
|
106
107
|
const is_verbose = Boolean(options.is_verbose)
|
|
108
|
+
const is_forced = Boolean(options.is_forced)
|
|
107
109
|
if (is_verbose) {
|
|
108
110
|
console.log(sql)
|
|
109
111
|
}
|
|
110
|
-
const items = sql.length <= 60000 ?
|
|
112
|
+
const items = (sql.length <= 60000 && !is_forced) ?
|
|
111
113
|
await this.rds_data.executeStatement({
|
|
112
114
|
resourceArn : this.rds_cluster_arn,
|
|
113
115
|
secretArn : this.rds_secret_arn,
|
|
@@ -136,12 +138,43 @@ export class UtilsRDS {
|
|
|
136
138
|
return items
|
|
137
139
|
}
|
|
138
140
|
|
|
141
|
+
async scan(
|
|
142
|
+
table_id : string,
|
|
143
|
+
options : {
|
|
144
|
+
filters? : Record<string, any>,
|
|
145
|
+
undefined_attribute_names? : string[],
|
|
146
|
+
defined_attribute_names? : string[],
|
|
147
|
+
attribute_names? : string[],
|
|
148
|
+
max_num_items? : number,
|
|
149
|
+
is_verbose? : boolean,
|
|
150
|
+
is_forced? : boolean
|
|
151
|
+
} = {}
|
|
152
|
+
) {
|
|
153
|
+
const undefined_attribute_names = options.undefined_attribute_names || []
|
|
154
|
+
const defined_attribute_names = options.defined_attribute_names || []
|
|
155
|
+
const filters = options.filters || {}
|
|
156
|
+
const clauses_where = [
|
|
157
|
+
...Object.entries(filters).map(([key_name, value]) => key_name + " = " + convert_rds_input(value)),
|
|
158
|
+
...undefined_attribute_names.map(undefined_attribute_name => undefined_attribute_name + " IS NULL"),
|
|
159
|
+
...defined_attribute_names.map(undefined_attribute_name => undefined_attribute_name + " IS NOT NULL")
|
|
160
|
+
]
|
|
161
|
+
const sql = [
|
|
162
|
+
"SELECT " + (options.attribute_names === undefined ? "*" : options.attribute_names.join(", ")),
|
|
163
|
+
"FROM " + table_id,
|
|
164
|
+
(clauses_where.length !== 0 ? ("WHERE " + clauses_where.join(" AND ")) : ""),
|
|
165
|
+
(options.max_num_items !== undefined ? ("LIMIT " + options.max_num_items) : "")
|
|
166
|
+
].join("\n")
|
|
167
|
+
const items = await this.exec(sql, options)
|
|
168
|
+
return items
|
|
169
|
+
}
|
|
170
|
+
|
|
139
171
|
async get(
|
|
140
172
|
table_id : string,
|
|
141
173
|
primary_key : Record<string, any>,
|
|
142
174
|
options : {
|
|
143
175
|
attribute_names? : string[],
|
|
144
|
-
is_verbose? : boolean
|
|
176
|
+
is_verbose? : boolean,
|
|
177
|
+
is_forced? : boolean
|
|
145
178
|
} = {}
|
|
146
179
|
) {
|
|
147
180
|
const sql = "SELECT " + (options.attribute_names === undefined ? "*" : options.attribute_names.join(", ")) + " FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, value]) => key_name + " = " + convert_rds_input(value)).join(" AND ")
|
|
@@ -155,7 +188,8 @@ export class UtilsRDS {
|
|
|
155
188
|
primary_key : Record<string, any[]>,
|
|
156
189
|
options : {
|
|
157
190
|
attribute_names? : string[],
|
|
158
|
-
is_verbose? : boolean
|
|
191
|
+
is_verbose? : boolean,
|
|
192
|
+
is_forced? : boolean
|
|
159
193
|
} = {}
|
|
160
194
|
) {
|
|
161
195
|
const sql = "SELECT " + (options.attribute_names === undefined ? "*" : options.attribute_names.join(", ")) + " FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, values]) => key_name + " IN " + "(" + values.map(value => convert_rds_input(value)).join(", ") + ")").join(" AND ")
|
|
@@ -167,7 +201,8 @@ export class UtilsRDS {
|
|
|
167
201
|
table_id : string,
|
|
168
202
|
item : Record<string, any>,
|
|
169
203
|
options : {
|
|
170
|
-
is_verbose? : boolean
|
|
204
|
+
is_verbose? : boolean,
|
|
205
|
+
is_forced? : boolean
|
|
171
206
|
} = {}
|
|
172
207
|
) {
|
|
173
208
|
const attribute_names = Object.keys(item).sort()
|
|
@@ -186,7 +221,8 @@ export class UtilsRDS {
|
|
|
186
221
|
table_id : string,
|
|
187
222
|
items : Record<string, any>[],
|
|
188
223
|
options : {
|
|
189
|
-
is_verbose? : boolean
|
|
224
|
+
is_verbose? : boolean,
|
|
225
|
+
is_forced? : boolean
|
|
190
226
|
} = {}
|
|
191
227
|
) {
|
|
192
228
|
if (items.length === 0) {
|
|
@@ -208,7 +244,8 @@ export class UtilsRDS {
|
|
|
208
244
|
table_id : string,
|
|
209
245
|
item : Record<string, any>,
|
|
210
246
|
options : {
|
|
211
|
-
is_verbose? : boolean
|
|
247
|
+
is_verbose? : boolean,
|
|
248
|
+
is_forced? : boolean
|
|
212
249
|
} = {}
|
|
213
250
|
) {
|
|
214
251
|
const attribute_names = Object.keys(item).sort()
|
|
@@ -227,7 +264,8 @@ export class UtilsRDS {
|
|
|
227
264
|
table_id : string,
|
|
228
265
|
items : Record<string, any>[],
|
|
229
266
|
options : {
|
|
230
|
-
is_verbose? : boolean
|
|
267
|
+
is_verbose? : boolean,
|
|
268
|
+
is_forced? : boolean
|
|
231
269
|
} = {}
|
|
232
270
|
) {
|
|
233
271
|
if (items.length === 0) {
|
|
@@ -249,7 +287,8 @@ export class UtilsRDS {
|
|
|
249
287
|
table_id : string,
|
|
250
288
|
primary_key : Record<string, any>,
|
|
251
289
|
options : {
|
|
252
|
-
is_verbose? : boolean
|
|
290
|
+
is_verbose? : boolean,
|
|
291
|
+
is_forced? : boolean
|
|
253
292
|
} = {}
|
|
254
293
|
) {
|
|
255
294
|
const sql = "DELETE FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, value]) => key_name + " = " + convert_rds_input(value)).join(" AND ")
|
|
@@ -260,7 +299,8 @@ export class UtilsRDS {
|
|
|
260
299
|
table_id : string,
|
|
261
300
|
primary_key : Record<string, any[]>,
|
|
262
301
|
options : {
|
|
263
|
-
is_verbose? : boolean
|
|
302
|
+
is_verbose? : boolean,
|
|
303
|
+
is_forced? : boolean
|
|
264
304
|
} = {}
|
|
265
305
|
) {
|
|
266
306
|
const sql = "DELETE FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, values]) => key_name + " IN " + "(" + values.map(value => convert_rds_input(value)).join(", ") + ")").join(" AND ")
|
|
@@ -272,7 +312,8 @@ export class UtilsRDS {
|
|
|
272
312
|
primary_key : Record<string, any>,
|
|
273
313
|
attributes : Record<string, any>,
|
|
274
314
|
options : {
|
|
275
|
-
is_verbose? : boolean
|
|
315
|
+
is_verbose? : boolean,
|
|
316
|
+
is_forced? : boolean
|
|
276
317
|
} = {}
|
|
277
318
|
) {
|
|
278
319
|
const attribute_names = Object.keys(attributes).sort()
|
package/src/UtilsS3.ts
CHANGED
|
@@ -20,7 +20,7 @@ export class UtilsS3 {
|
|
|
20
20
|
return this.s3
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
-
get_s3_input(s3_id : string) : S3Input | undefined {
|
|
23
|
+
static get_s3_input(s3_id : string) : S3Input | undefined {
|
|
24
24
|
const [bucket, path] = (s3_id.match(/^s3\:\/\/([^\/]+)\/([^$]+)$/) || []).slice(1, 3)
|
|
25
25
|
if (bucket === undefined || path === undefined) {
|
|
26
26
|
return undefined
|
|
@@ -29,7 +29,7 @@ export class UtilsS3 {
|
|
|
29
29
|
}
|
|
30
30
|
|
|
31
31
|
async get_headers(s3_id : string) {
|
|
32
|
-
const s3_input =
|
|
32
|
+
const s3_input = UtilsS3.get_s3_input(s3_id)
|
|
33
33
|
if (s3_input === undefined) {
|
|
34
34
|
return undefined
|
|
35
35
|
}
|
|
@@ -45,7 +45,7 @@ export class UtilsS3 {
|
|
|
45
45
|
}
|
|
46
46
|
|
|
47
47
|
async get(s3_id : string, encoding : string = "utf-8") : Promise<string | undefined> {
|
|
48
|
-
const s3_input =
|
|
48
|
+
const s3_input = UtilsS3.get_s3_input(s3_id)
|
|
49
49
|
if (s3_input === undefined) {
|
|
50
50
|
return undefined
|
|
51
51
|
}
|
|
@@ -66,7 +66,7 @@ export class UtilsS3 {
|
|
|
66
66
|
}
|
|
67
67
|
|
|
68
68
|
async get_buffer(s3_id : string) : Promise<Buffer | undefined> {
|
|
69
|
-
const s3_input =
|
|
69
|
+
const s3_input = UtilsS3.get_s3_input(s3_id)
|
|
70
70
|
if (s3_input === undefined) {
|
|
71
71
|
return undefined
|
|
72
72
|
}
|
|
@@ -87,7 +87,7 @@ export class UtilsS3 {
|
|
|
87
87
|
}
|
|
88
88
|
|
|
89
89
|
async get_stream(s3_id : string) : Promise<Readable | undefined> {
|
|
90
|
-
const s3_input =
|
|
90
|
+
const s3_input = UtilsS3.get_s3_input(s3_id)
|
|
91
91
|
if (s3_input === undefined) {
|
|
92
92
|
return undefined
|
|
93
93
|
}
|
|
@@ -111,7 +111,7 @@ export class UtilsS3 {
|
|
|
111
111
|
|
|
112
112
|
async query_prefix(s3_id_prefix : string, options : { compile? : boolean } = {}) {
|
|
113
113
|
const compile = options.compile !== undefined ? options.compile : false
|
|
114
|
-
const s3_input =
|
|
114
|
+
const s3_input = UtilsS3.get_s3_input(s3_id_prefix)
|
|
115
115
|
if (s3_input === undefined) {
|
|
116
116
|
return []
|
|
117
117
|
}
|
|
@@ -155,7 +155,7 @@ export class UtilsS3 {
|
|
|
155
155
|
if (headers !== undefined) {
|
|
156
156
|
return undefined
|
|
157
157
|
}
|
|
158
|
-
const s3_input =
|
|
158
|
+
const s3_input = UtilsS3.get_s3_input(s3_id)
|
|
159
159
|
if (s3_input === undefined) {
|
|
160
160
|
return undefined
|
|
161
161
|
}
|
|
@@ -172,7 +172,7 @@ export class UtilsS3 {
|
|
|
172
172
|
} = {}) : Promise<void> {
|
|
173
173
|
const content_type_id = options.content_type_id !== undefined ? options.content_type_id : undefined
|
|
174
174
|
const part_size = options.part_size !== undefined ? options.part_size : undefined
|
|
175
|
-
const s3_input =
|
|
175
|
+
const s3_input = UtilsS3.get_s3_input(s3_id)
|
|
176
176
|
if (s3_input === undefined) {
|
|
177
177
|
return undefined
|
|
178
178
|
}
|
|
@@ -189,7 +189,7 @@ export class UtilsS3 {
|
|
|
189
189
|
}
|
|
190
190
|
|
|
191
191
|
async delete(s3_id : string) : Promise<void>{
|
|
192
|
-
const s3_input =
|
|
192
|
+
const s3_input = UtilsS3.get_s3_input(s3_id)
|
|
193
193
|
if (s3_input === undefined) {
|
|
194
194
|
return undefined
|
|
195
195
|
}
|
|
@@ -201,7 +201,7 @@ export class UtilsS3 {
|
|
|
201
201
|
if (headers === undefined) {
|
|
202
202
|
return undefined
|
|
203
203
|
}
|
|
204
|
-
const s3_input =
|
|
204
|
+
const s3_input = UtilsS3.get_s3_input(s3_id)
|
|
205
205
|
if (s3_input === undefined) {
|
|
206
206
|
return undefined
|
|
207
207
|
}
|
|
@@ -214,7 +214,7 @@ export class UtilsS3 {
|
|
|
214
214
|
content_type_id? : string
|
|
215
215
|
} = {}) : Promise<string | undefined> {
|
|
216
216
|
const content_type_id = options.content_type_id !== undefined ? options.content_type_id : undefined
|
|
217
|
-
const s3_input =
|
|
217
|
+
const s3_input = UtilsS3.get_s3_input(s3_id)
|
|
218
218
|
if (s3_input === undefined) {
|
|
219
219
|
return undefined
|
|
220
220
|
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { Block, GetDocumentAnalysisCommandOutput, StartDocumentTextDetectionRequest, Textract } from "@aws-sdk/client-textract"
|
|
2
|
+
import { UtilsMisc } from "./UtilsMisc"
|
|
3
|
+
import { UtilsS3 } from "./UtilsS3"
|
|
4
|
+
|
|
5
|
+
export class UtilsTextract {
|
|
6
|
+
|
|
7
|
+
private readonly textract : Textract
|
|
8
|
+
|
|
9
|
+
constructor(region : string) {
|
|
10
|
+
this.textract = new Textract({ region : region })
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
async get_text(s3_id : string) {
|
|
14
|
+
const s3_input = UtilsS3.get_s3_input(s3_id)
|
|
15
|
+
if (s3_input === undefined) {
|
|
16
|
+
return undefined
|
|
17
|
+
}
|
|
18
|
+
const request : StartDocumentTextDetectionRequest = {
|
|
19
|
+
DocumentLocation : {
|
|
20
|
+
S3Object : {
|
|
21
|
+
Bucket : s3_input.Bucket,
|
|
22
|
+
Name : s3_input.Key
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
const job_id = await this.textract.startDocumentTextDetection(request)
|
|
27
|
+
.then(response => response.JobId)
|
|
28
|
+
if (job_id === undefined) {
|
|
29
|
+
return
|
|
30
|
+
}
|
|
31
|
+
let token : string | undefined = undefined
|
|
32
|
+
const blocks : Block[] = []
|
|
33
|
+
while (true) {
|
|
34
|
+
const result : GetDocumentAnalysisCommandOutput = await this.textract.getDocumentTextDetection({
|
|
35
|
+
JobId : job_id,
|
|
36
|
+
NextToken : token
|
|
37
|
+
})
|
|
38
|
+
if (result.JobStatus === "IN_PROGRESS") {
|
|
39
|
+
UtilsMisc.wait(1000)
|
|
40
|
+
continue
|
|
41
|
+
}
|
|
42
|
+
blocks.push(...(result.Blocks || []))
|
|
43
|
+
if (result.NextToken === undefined) {
|
|
44
|
+
return blocks
|
|
45
|
+
}
|
|
46
|
+
token = result.NextToken
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
}
|
package/src/f.ts
CHANGED
|
@@ -93,5 +93,12 @@ const utils = new TriangleUtils(config)
|
|
|
93
93
|
|
|
94
94
|
// console.log(await utils.rds.exec("SELECT * FROM prisms", { is_verbose : true }))\
|
|
95
95
|
|
|
96
|
+
// await utils.rds.connect()
|
|
97
|
+
// console.log(await utils.rds.scan("web_articles", { is_verbose : true, is_forced : true, max_num_items : 100 }))
|
|
98
|
+
// await utils.rds.disconnect()
|
|
96
99
|
|
|
97
|
-
|
|
100
|
+
// await utils.twitter.get_user("1509572215251124226")
|
|
101
|
+
// .then(user => console.log(user))
|
|
102
|
+
|
|
103
|
+
// await utils.bedrock.cohere_invoke("cohere.embed-v4", "yyyyyuuuuuurrrrr")
|
|
104
|
+
// .then(response => console.log(response))
|
package/src/index.ts
CHANGED
|
@@ -13,6 +13,7 @@ import { UtilsTwitter } from "./UtilsTwitter"
|
|
|
13
13
|
import { GlobalConfig } from "./types/GlobalConfig"
|
|
14
14
|
import { UtilsRDS } from "./UtilsRDS"
|
|
15
15
|
import { ApplicationConfig } from "./types/ApplicationConfig"
|
|
16
|
+
import { UtilsTextract } from "./UtilsTextract"
|
|
16
17
|
|
|
17
18
|
|
|
18
19
|
|
|
@@ -23,6 +24,7 @@ export class TriangleUtils extends UtilsMisc {
|
|
|
23
24
|
readonly s3 : UtilsS3
|
|
24
25
|
readonly s3vectors : UtilsS3Vectors
|
|
25
26
|
readonly bedrock : UtilsBedrock
|
|
27
|
+
readonly textract : UtilsTextract
|
|
26
28
|
readonly cognito : UtilsCognito
|
|
27
29
|
readonly ses : UtilsSES
|
|
28
30
|
readonly bee : UtilsBee
|
|
@@ -38,6 +40,7 @@ export class TriangleUtils extends UtilsMisc {
|
|
|
38
40
|
this.s3 = new UtilsS3(config.region)
|
|
39
41
|
this.s3vectors = new UtilsS3Vectors(config.region)
|
|
40
42
|
this.bedrock = new UtilsBedrock(config.region)
|
|
43
|
+
this.textract = new UtilsTextract(config.region)
|
|
41
44
|
this.cognito = new UtilsCognito(config.region)
|
|
42
45
|
this.ses = new UtilsSES(config.region)
|
|
43
46
|
this.bee = new UtilsBee(config.scraping_bee_api_key)
|