triangle-utils 1.4.120 → 1.4.122
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/UtilsMisc.d.ts +1 -1
- package/dist/src/UtilsMisc.js +3 -3
- package/dist/src/UtilsRDS.d.ts +34 -0
- package/dist/src/UtilsRDS.js +128 -0
- package/dist/src/f.js +9 -223
- package/dist/src/index.d.ts +5 -1
- package/dist/src/index.js +4 -0
- package/dist/src/types/GlobalConfig.d.ts +2 -0
- package/dist/src/types/GlobalConfig.js +6 -0
- package/package.json +3 -1
- package/src/UtilsMisc.ts +4 -4
- package/src/UtilsRDS.ts +172 -0
- package/src/f.ts +9 -250
- package/src/index.ts +6 -1
- package/src/types/GlobalConfig.ts +6 -0
package/dist/src/UtilsMisc.d.ts
CHANGED
|
@@ -12,7 +12,7 @@ export declare class UtilsMisc {
|
|
|
12
12
|
verbosity?: number | boolean;
|
|
13
13
|
}): Promise<(U | undefined)[]>;
|
|
14
14
|
batch_iterate<T, U>(inputs: T[], f: (inputs: T[], iterator_id: number, index: number) => Promise<U>, options: {
|
|
15
|
-
|
|
15
|
+
batch_num_items?: number;
|
|
16
16
|
num_iterators?: number;
|
|
17
17
|
verbosity?: number | boolean;
|
|
18
18
|
}): Promise<(U | undefined)[]>;
|
package/dist/src/UtilsMisc.js
CHANGED
|
@@ -44,7 +44,7 @@ export class UtilsMisc {
|
|
|
44
44
|
return outputs;
|
|
45
45
|
}
|
|
46
46
|
async batch_iterate(inputs, f, options) {
|
|
47
|
-
const
|
|
47
|
+
const batch_num_items = options.batch_num_items || 1;
|
|
48
48
|
const num_iterators = options.num_iterators || 1;
|
|
49
49
|
const verbosity = typeof options.verbosity === "boolean" ? (options.verbosity === true ? 1 : 0) : (options.verbosity || 0);
|
|
50
50
|
let index = 0;
|
|
@@ -54,11 +54,11 @@ export class UtilsMisc {
|
|
|
54
54
|
iterators.push((async () => {
|
|
55
55
|
while (index < inputs.length) {
|
|
56
56
|
const local_index = index;
|
|
57
|
-
index +=
|
|
57
|
+
index += batch_num_items;
|
|
58
58
|
if (verbosity !== undefined && (local_index % verbosity === 0)) {
|
|
59
59
|
console.log(iterator_id + ":" + local_index + "/" + inputs.length);
|
|
60
60
|
}
|
|
61
|
-
const output = await this.safe_run(() => f(inputs.slice(local_index, local_index +
|
|
61
|
+
const output = await this.safe_run(() => f(inputs.slice(local_index, local_index + batch_num_items), iterator_id, local_index));
|
|
62
62
|
outputs[local_index] = output;
|
|
63
63
|
}
|
|
64
64
|
})());
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { Field } from "@aws-sdk/client-rds-data";
|
|
2
|
+
export declare class UtilsRDS {
|
|
3
|
+
private readonly rds;
|
|
4
|
+
private readonly rds_cluster_arn;
|
|
5
|
+
private readonly rds_secret_arn;
|
|
6
|
+
constructor(region: string, rds_cluster_arn: string, rds_secret_arn: string);
|
|
7
|
+
exec(sql: string): Promise<Field[][]>;
|
|
8
|
+
get(table_id: string, primary_key: Record<string, any>, attribute_names: string[], options?: {
|
|
9
|
+
is_verbose?: boolean;
|
|
10
|
+
}): Promise<{
|
|
11
|
+
[k: string]: string | number | boolean | undefined;
|
|
12
|
+
} | undefined>;
|
|
13
|
+
batch_get(table_id: string, primary_key: Record<string, any[]>, attribute_names: string[], options?: {
|
|
14
|
+
is_verbose?: boolean;
|
|
15
|
+
}): Promise<{
|
|
16
|
+
[k: string]: string | number | boolean | undefined;
|
|
17
|
+
}[]>;
|
|
18
|
+
put(table_id: string, item: Record<string, any>, options?: {
|
|
19
|
+
is_verbose?: boolean;
|
|
20
|
+
}): Promise<{
|
|
21
|
+
[k: string]: string | number | boolean | undefined;
|
|
22
|
+
}[] | undefined>;
|
|
23
|
+
batch_put(table_id: string, items: Record<string, any>[], options?: {
|
|
24
|
+
is_verbose?: boolean;
|
|
25
|
+
}): Promise<{
|
|
26
|
+
[k: string]: string | number | boolean | undefined;
|
|
27
|
+
}[] | undefined>;
|
|
28
|
+
delete(table_id: string, primary_key: Record<string, any>, options?: {
|
|
29
|
+
is_verbose?: boolean;
|
|
30
|
+
}): Promise<undefined>;
|
|
31
|
+
batch_delete(table_id: string, primary_key: Record<string, any[]>, options?: {
|
|
32
|
+
is_verbose?: boolean;
|
|
33
|
+
}): Promise<void>;
|
|
34
|
+
}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import { RDSData } from "@aws-sdk/client-rds-data";
|
|
2
|
+
function convert_rds_output(rds_output) {
|
|
3
|
+
if (rds_output.isNull) {
|
|
4
|
+
return undefined;
|
|
5
|
+
}
|
|
6
|
+
else if (rds_output.stringValue !== undefined) {
|
|
7
|
+
return rds_output.stringValue;
|
|
8
|
+
}
|
|
9
|
+
else if (rds_output.longValue !== undefined) {
|
|
10
|
+
return rds_output.longValue;
|
|
11
|
+
}
|
|
12
|
+
else if (rds_output.doubleValue !== undefined) {
|
|
13
|
+
return rds_output.doubleValue;
|
|
14
|
+
}
|
|
15
|
+
else if (rds_output.booleanValue !== undefined) {
|
|
16
|
+
return rds_output.booleanValue;
|
|
17
|
+
}
|
|
18
|
+
return undefined;
|
|
19
|
+
}
|
|
20
|
+
function convert_rds_input(input) {
|
|
21
|
+
if (input === undefined) {
|
|
22
|
+
return "NULL";
|
|
23
|
+
}
|
|
24
|
+
else if (typeof input === "string") {
|
|
25
|
+
return "'" + input + "'";
|
|
26
|
+
}
|
|
27
|
+
else if (typeof input === "number") {
|
|
28
|
+
return input.toString();
|
|
29
|
+
}
|
|
30
|
+
else if (typeof input === "boolean") {
|
|
31
|
+
return input.toString().toUpperCase();
|
|
32
|
+
}
|
|
33
|
+
return "NULL";
|
|
34
|
+
}
|
|
35
|
+
export class UtilsRDS {
|
|
36
|
+
rds;
|
|
37
|
+
rds_cluster_arn;
|
|
38
|
+
rds_secret_arn;
|
|
39
|
+
constructor(region, rds_cluster_arn, rds_secret_arn) {
|
|
40
|
+
this.rds = new RDSData({ region: region });
|
|
41
|
+
this.rds_cluster_arn = rds_cluster_arn,
|
|
42
|
+
this.rds_secret_arn = rds_secret_arn;
|
|
43
|
+
}
|
|
44
|
+
async exec(sql) {
|
|
45
|
+
const response = await this.rds.executeStatement({
|
|
46
|
+
resourceArn: this.rds_cluster_arn,
|
|
47
|
+
secretArn: this.rds_secret_arn,
|
|
48
|
+
database: "postgres",
|
|
49
|
+
sql: sql
|
|
50
|
+
});
|
|
51
|
+
return response.records || [];
|
|
52
|
+
}
|
|
53
|
+
async get(table_id, primary_key, attribute_names, options = {}) {
|
|
54
|
+
const is_verbose = Boolean(options.is_verbose);
|
|
55
|
+
const sql = "SELECT " + attribute_names.join(", ") + " FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, value]) => key_name + " = " + convert_rds_input(value)).join(" AND ");
|
|
56
|
+
if (is_verbose) {
|
|
57
|
+
console.log(sql);
|
|
58
|
+
}
|
|
59
|
+
const items = await this.exec(sql);
|
|
60
|
+
const item = items[0];
|
|
61
|
+
if (item === undefined) {
|
|
62
|
+
return undefined;
|
|
63
|
+
}
|
|
64
|
+
return Object.fromEntries(attribute_names.map((attribute_name, i) => [attribute_name, convert_rds_output(item[i])]));
|
|
65
|
+
}
|
|
66
|
+
async batch_get(table_id, primary_key, attribute_names, options = {}) {
|
|
67
|
+
const is_verbose = Boolean(options.is_verbose);
|
|
68
|
+
const sql = "SELECT " + attribute_names.join(", ") + " FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, values]) => key_name + " IN " + "(" + values.map(value => convert_rds_input(value)).join(", ") + ")").join(" AND ");
|
|
69
|
+
if (is_verbose) {
|
|
70
|
+
console.log(sql);
|
|
71
|
+
}
|
|
72
|
+
const items = await this.exec(sql);
|
|
73
|
+
return items.map(item => Object.fromEntries(attribute_names.map((attribute_name, i) => [attribute_name, convert_rds_output(item[i])])));
|
|
74
|
+
}
|
|
75
|
+
async put(table_id, item, options = {}) {
|
|
76
|
+
const is_verbose = Boolean(options.is_verbose);
|
|
77
|
+
const attribute_names = Object.keys(item).sort();
|
|
78
|
+
const sql = "INSERT INTO " + table_id + " (" + attribute_names.join(", ") + ")" + " VALUES " + "(" + attribute_names.map(attribute_name => convert_rds_input(item[attribute_name])).join(", ") + ")" + " ON CONFLICT ON CONSTRAINT " + table_id + "_pkey DO UPDATE SET " + attribute_names.map(attribute_name => attribute_name + " = " + "EXCLUDED." + attribute_name).join(", ") + ";";
|
|
79
|
+
if (is_verbose) {
|
|
80
|
+
console.log(sql);
|
|
81
|
+
}
|
|
82
|
+
try {
|
|
83
|
+
const items = await this.exec(sql);
|
|
84
|
+
return items.map(item => Object.fromEntries(attribute_names.map((attribute_name, i) => [attribute_name, convert_rds_output(item[i])])));
|
|
85
|
+
}
|
|
86
|
+
catch (error) {
|
|
87
|
+
console.log(error.stack);
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
async batch_put(table_id, items, options = {}) {
|
|
92
|
+
const is_verbose = Boolean(options.is_verbose);
|
|
93
|
+
const attribute_names = Array.from(new Set(items.map(item => Object.keys(item)).flat())).sort();
|
|
94
|
+
const sql = "INSERT INTO " + table_id + " (" + attribute_names.join(", ") + ")" + " VALUES " + items.map(item => "(" + attribute_names.map(attribute_name => convert_rds_input(item[attribute_name])).join(", ") + ")").join(", ") + " ON CONFLICT ON CONSTRAINT " + table_id + "_pkey DO UPDATE SET " + attribute_names.map(attribute_name => attribute_name + " = " + "EXCLUDED." + attribute_name).join(", ") + ";";
|
|
95
|
+
if (is_verbose) {
|
|
96
|
+
console.log(sql);
|
|
97
|
+
}
|
|
98
|
+
try {
|
|
99
|
+
const items = await this.exec(sql);
|
|
100
|
+
return items.map(item => Object.fromEntries(attribute_names.map((attribute_name, i) => [attribute_name, convert_rds_output(item[i])])));
|
|
101
|
+
}
|
|
102
|
+
catch (error) {
|
|
103
|
+
console.log(error.stack);
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
async delete(table_id, primary_key, options = {}) {
|
|
108
|
+
const is_verbose = Boolean(options.is_verbose);
|
|
109
|
+
const sql = "DELETE FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, value]) => key_name + " = " + convert_rds_input(value)).join(" AND ");
|
|
110
|
+
if (is_verbose) {
|
|
111
|
+
console.log(sql);
|
|
112
|
+
}
|
|
113
|
+
const items = await this.exec(sql);
|
|
114
|
+
const item = items[0];
|
|
115
|
+
if (item === undefined) {
|
|
116
|
+
return undefined;
|
|
117
|
+
}
|
|
118
|
+
return undefined;
|
|
119
|
+
}
|
|
120
|
+
async batch_delete(table_id, primary_key, options = {}) {
|
|
121
|
+
const is_verbose = Boolean(options.is_verbose);
|
|
122
|
+
const sql = "DELETE FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, values]) => key_name + " IN " + "(" + values.map(value => convert_rds_input(value)).join(", ") + ")").join(" AND ");
|
|
123
|
+
if (is_verbose) {
|
|
124
|
+
console.log(sql);
|
|
125
|
+
}
|
|
126
|
+
await this.exec(sql);
|
|
127
|
+
}
|
|
128
|
+
}
|
package/dist/src/f.js
CHANGED
|
@@ -15,226 +15,12 @@ const config = {
|
|
|
15
15
|
};
|
|
16
16
|
console.log(config);
|
|
17
17
|
const utils = new TriangleUtils(config);
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
// console.log(
|
|
21
|
-
//
|
|
22
|
-
//
|
|
23
|
-
//
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
// type: "string"
|
|
28
|
-
// },
|
|
29
|
-
// explanation : {
|
|
30
|
-
// type: "string"
|
|
31
|
-
// }
|
|
32
|
-
// },
|
|
33
|
-
// required : [
|
|
34
|
-
// "username",
|
|
35
|
-
// "explanation"
|
|
36
|
-
// ],
|
|
37
|
-
// additionalProperties : false
|
|
38
|
-
// }
|
|
39
|
-
// })
|
|
40
|
-
// console.log(text)
|
|
41
|
-
// const foods = await utils.dynamodb.query("triage_docket_documents:register_document_id", { register_document_id : "E6-17065" })
|
|
42
|
-
// console.log(foods)
|
|
43
|
-
// const text = await utils.bee.get("https://www.doyourjobs.org", { return_page_text : true })
|
|
44
|
-
// console.log(text)
|
|
45
|
-
// const text = await utils.bedrock.claude_invoke("global.anthropic.claude-opus-4-6-v1", "hi", undefined, { print_usage : true })
|
|
46
|
-
// console.log(text)
|
|
47
|
-
// const url = "https://dataviewers.tdec.tn.gov/dataviewers/f?p=2005:34051:3300341444471:::34051:P34051_PERMIT_NUMBER:TNR136379"
|
|
48
|
-
// const html = await utils.bee.get(url, { render_js : false })
|
|
49
|
-
// console.log(html)
|
|
50
|
-
// const prompt = "Is Mark Kelly a veteran?"
|
|
51
|
-
// console.log(prompt)
|
|
52
|
-
// const output = await utils.anthropic.claude_query("claude-opus-4-8",
|
|
53
|
-
// prompt,
|
|
54
|
-
// undefined,
|
|
55
|
-
// {
|
|
56
|
-
// print_usage : true,
|
|
57
|
-
// max_tokens : 20000,
|
|
58
|
-
// json_format : {
|
|
59
|
-
// type : "object",
|
|
60
|
-
// properties : {
|
|
61
|
-
// is_veteran : {
|
|
62
|
-
// type : "string"
|
|
63
|
-
// },
|
|
64
|
-
// explanation : {
|
|
65
|
-
// type : "string"
|
|
66
|
-
// }
|
|
67
|
-
// },
|
|
68
|
-
// required : ["is_veteran", "explanation"],
|
|
69
|
-
// additionalProperties : false
|
|
70
|
-
// },
|
|
71
|
-
// web : true
|
|
72
|
-
// }
|
|
73
|
-
// )
|
|
74
|
-
// console.log(output)
|
|
75
|
-
// import { Client } from "@xdevplatform/xdk"
|
|
76
|
-
// const x = new Client({ bearerToken : "AAAAAAAAAAAAAAAAAAAAAKoQ%2BQEAAAAAaej7PP534%2Fd9dKmieHRY6vQkv2s%3DyY2VerXmrA7w794ACFr9ALvoaJfuAHSlWwZdiDeVaOVpA6HCub" })
|
|
77
|
-
// const user = await x.users.getByUsername("umichvoter", { userFields : [
|
|
78
|
-
// "affiliation",
|
|
79
|
-
// // "confirmed_email",
|
|
80
|
-
// "connection_status",
|
|
81
|
-
// "created_at",
|
|
82
|
-
// "description",
|
|
83
|
-
// "entities",
|
|
84
|
-
// "id",
|
|
85
|
-
// "is_identity_verified",
|
|
86
|
-
// "location",
|
|
87
|
-
// "most_recent_tweet_id",
|
|
88
|
-
// "name",
|
|
89
|
-
// "parody",
|
|
90
|
-
// "pinned_tweet_id",
|
|
91
|
-
// "profile_banner_url",
|
|
92
|
-
// "profile_image_url",
|
|
93
|
-
// "protected",
|
|
94
|
-
// "public_metrics",
|
|
95
|
-
// // "receives_your_dm",
|
|
96
|
-
// "subscription",
|
|
97
|
-
// "subscription_type",
|
|
98
|
-
// "url",
|
|
99
|
-
// "username",
|
|
100
|
-
// "verified",
|
|
101
|
-
// "verified_followers_count",
|
|
102
|
-
// "verified_type",
|
|
103
|
-
// "withheld"
|
|
104
|
-
// ] })
|
|
105
|
-
// console.log(JSON.stringify(user, null, 4))
|
|
106
|
-
// const tweets = await x.users.getPosts("1603470469608374272", {
|
|
107
|
-
// tweetFields : [
|
|
108
|
-
// "article",
|
|
109
|
-
// "attachments",
|
|
110
|
-
// "author_id",
|
|
111
|
-
// "card_uri",
|
|
112
|
-
// "community_id",
|
|
113
|
-
// "context_annotations",
|
|
114
|
-
// "conversation_id",
|
|
115
|
-
// "created_at",
|
|
116
|
-
// "display_text_range",
|
|
117
|
-
// "edit_controls",
|
|
118
|
-
// "edit_history_tweet_ids",
|
|
119
|
-
// "entities",
|
|
120
|
-
// "geo",
|
|
121
|
-
// "id",
|
|
122
|
-
// "in_reply_to_user_id",
|
|
123
|
-
// "lang",
|
|
124
|
-
// "matched_media_notes",
|
|
125
|
-
// "media_metadata",
|
|
126
|
-
// "note_request_suggestions",
|
|
127
|
-
// "note_tweet",
|
|
128
|
-
// "paid_partnership",
|
|
129
|
-
// "possibly_sensitive",
|
|
130
|
-
// "referenced_tweets",
|
|
131
|
-
// "reply_settings",
|
|
132
|
-
// "scopes",
|
|
133
|
-
// "source",
|
|
134
|
-
// "suggested_source_links",
|
|
135
|
-
// "suggested_source_links_with_counts",
|
|
136
|
-
// "text",
|
|
137
|
-
// "withheld"
|
|
138
|
-
// ], expansions : [
|
|
139
|
-
// "article.cover_media",
|
|
140
|
-
// "article.media_entities",
|
|
141
|
-
// "attachments.media_keys",
|
|
142
|
-
// "attachments.media_source_tweet",
|
|
143
|
-
// "attachments.poll_ids",
|
|
144
|
-
// "author_id",
|
|
145
|
-
// "edit_history_tweet_ids",
|
|
146
|
-
// "entities.mentions.username",
|
|
147
|
-
// "geo.place_id",
|
|
148
|
-
// "in_reply_to_user_id",
|
|
149
|
-
// "entities.note.mentions.username",
|
|
150
|
-
// "referenced_tweets.id",
|
|
151
|
-
// "referenced_tweets.id.attachments.media_keys",
|
|
152
|
-
// "referenced_tweets.id.author_id"
|
|
153
|
-
// ]})
|
|
154
|
-
// console.log(JSON.stringify(tweets, null, 4))
|
|
155
|
-
// for (const tweet of tweets.data || []) {
|
|
156
|
-
// console.log(JSON.stringify(tweet, undefined, 4))
|
|
157
|
-
// }
|
|
158
|
-
// const tweet = await x.posts.getById("2061603004248154195", { tweetFields : [
|
|
159
|
-
// "article",
|
|
160
|
-
// "attachments",
|
|
161
|
-
// "author_id",
|
|
162
|
-
// "card_uri",
|
|
163
|
-
// "community_id",
|
|
164
|
-
// "context_annotations",
|
|
165
|
-
// "conversation_id",
|
|
166
|
-
// "created_at",
|
|
167
|
-
// "display_text_range",
|
|
168
|
-
// "edit_controls",
|
|
169
|
-
// "edit_history_tweet_ids",
|
|
170
|
-
// "entities",
|
|
171
|
-
// "geo",
|
|
172
|
-
// "id",
|
|
173
|
-
// "in_reply_to_user_id",
|
|
174
|
-
// "lang",
|
|
175
|
-
// "matched_media_notes",
|
|
176
|
-
// "media_metadata",
|
|
177
|
-
// // "non_public_metrics",
|
|
178
|
-
// "note_request_suggestions",
|
|
179
|
-
// "note_tweet",
|
|
180
|
-
// // "organic_metrics",
|
|
181
|
-
// "paid_partnership",
|
|
182
|
-
// "possibly_sensitive",
|
|
183
|
-
// // "promoted_metrics",
|
|
184
|
-
// "public_metrics",
|
|
185
|
-
// "referenced_tweets",
|
|
186
|
-
// "reply_settings",
|
|
187
|
-
// "scopes",
|
|
188
|
-
// "source",
|
|
189
|
-
// "suggested_source_links",
|
|
190
|
-
// "suggested_source_links_with_counts",
|
|
191
|
-
// "text",
|
|
192
|
-
// "withheld"
|
|
193
|
-
// ], expansions : [
|
|
194
|
-
// "article.cover_media",
|
|
195
|
-
// "article.media_entities",
|
|
196
|
-
// "attachments.media_keys",
|
|
197
|
-
// "attachments.media_source_tweet",
|
|
198
|
-
// "attachments.poll_ids",
|
|
199
|
-
// "author_id",
|
|
200
|
-
// "edit_history_tweet_ids",
|
|
201
|
-
// "entities.mentions.username",
|
|
202
|
-
// "geo.place_id",
|
|
203
|
-
// "in_reply_to_user_id",
|
|
204
|
-
// "entities.note.mentions.username",
|
|
205
|
-
// "referenced_tweets.id",
|
|
206
|
-
// "referenced_tweets.id.attachments.media_keys",
|
|
207
|
-
// "referenced_tweets.id.author_id"
|
|
208
|
-
// ], pollFields : [
|
|
209
|
-
// "duration_minutes",
|
|
210
|
-
// "end_datetime",
|
|
211
|
-
// "id",
|
|
212
|
-
// "options",
|
|
213
|
-
// "voting_status"
|
|
214
|
-
// ]
|
|
215
|
-
// })
|
|
216
|
-
// console.log(JSON.stringify(tweet, undefined, 4))
|
|
217
|
-
// const results = await utils.twitter.get_tweets("1603470469608374272", {
|
|
218
|
-
// compile : true,
|
|
219
|
-
// min_twitter_tweet_id : "2061610563679985813"
|
|
220
|
-
// })
|
|
221
|
-
// console.log(results)
|
|
222
|
-
// const f = await utils.dynamodb.query_range("federal_election_donations:donor_address_state_id.federal_election_committee_id-receipt_date-federal_election_donation_id", { donor_address_state_id : "NH", federal_election_committee_id : "C00694323" }, { receipt_date : ["2021", "2023"] }, { compile : false })
|
|
223
|
-
// console.log(f)
|
|
224
|
-
// console.log("YERP")
|
|
225
|
-
// const response = await utils.bedrock.claude_converse(
|
|
226
|
-
// "anthropic.claude-opus-4-8",
|
|
227
|
-
// [
|
|
228
|
-
// { role : "user", text : "Hi" }
|
|
229
|
-
// ],
|
|
230
|
-
// {
|
|
231
|
-
// print_usage : true
|
|
232
|
-
// }
|
|
233
|
-
// )
|
|
234
|
-
// console.log(response)
|
|
235
|
-
// const coordinates = await utils.bedrock.cohere_invoke("global.cohere.embed-v4:0", "yerrp")
|
|
236
|
-
// console.log(coordinates)
|
|
237
|
-
const scout_ids = await utils.dynamodb.scan("scouts")
|
|
238
|
-
.then(scouts => scouts.map(scout => scout.scout_id));
|
|
239
|
-
const scouts = await utils.dynamodb.batch_get("scouts", scout_ids.slice(0, 150).map(scout_id => ({ scout_id: scout_id })));
|
|
240
|
-
console.log(scouts.length);
|
|
18
|
+
const federal_censuses = await utils.dynamodb.scan("federal_censuses");
|
|
19
|
+
federal_censuses.sort((a, b) => a.federal_census_id - b.federal_census_id);
|
|
20
|
+
// console.log(JSON.stringify(response, undefined, 4))
|
|
21
|
+
// await utils.rds.batch_delete("federal_censuses", {
|
|
22
|
+
// federal_census_id : federal_censuses.map(federal_census => federal_census.federal_census_id)
|
|
23
|
+
// }, { is_verbose : true })
|
|
24
|
+
await utils.rds.batch_put("federal_censuses", federal_censuses);
|
|
25
|
+
const rds_federal_censuses = await utils.rds.batch_get("federal_censuses", { federal_census_id: federal_censuses.map(federal_census => federal_census.federal_census_id) }, ["federal_census_id", "is_decennial"], { is_verbose: true });
|
|
26
|
+
console.log(rds_federal_censuses);
|
package/dist/src/index.d.ts
CHANGED
|
@@ -11,8 +11,11 @@ import { UtilsXAI } from "./UtilsXAI.js";
|
|
|
11
11
|
import { UtilsAnthropic } from "./UtilsAnthropic.js";
|
|
12
12
|
import { UtilsTwitter } from "./UtilsTwitter.js";
|
|
13
13
|
import { GlobalConfig } from "./types/GlobalConfig.js";
|
|
14
|
+
import { UtilsRDS } from "./UtilsRDS.js";
|
|
15
|
+
import { ApplicationConfig } from "./types/ApplicationConfig.js";
|
|
14
16
|
export declare class TriangleUtils extends UtilsMisc {
|
|
15
17
|
readonly dynamodb: UtilsDynamoDB;
|
|
18
|
+
readonly rds: UtilsRDS;
|
|
16
19
|
readonly s3: UtilsS3;
|
|
17
20
|
readonly s3vectors: UtilsS3Vectors;
|
|
18
21
|
readonly bedrock: UtilsBedrock;
|
|
@@ -23,13 +26,14 @@ export declare class TriangleUtils extends UtilsMisc {
|
|
|
23
26
|
readonly anthropic: UtilsAnthropic;
|
|
24
27
|
readonly xai: UtilsXAI;
|
|
25
28
|
readonly twitter: UtilsTwitter;
|
|
26
|
-
constructor(config: GlobalConfig);
|
|
29
|
+
constructor(config: ApplicationConfig & GlobalConfig);
|
|
27
30
|
}
|
|
28
31
|
export * from "./types/ApplicationConfig.js";
|
|
29
32
|
export * from "./types/GlobalConfig.js";
|
|
30
33
|
export * from "./UtilsMisc.js";
|
|
31
34
|
export * from "./UtilsAnthropic.js";
|
|
32
35
|
export * from "./UtilsDynamoDB.js";
|
|
36
|
+
export * from "./UtilsRDS.js";
|
|
33
37
|
export * from "./UtilsS3.js";
|
|
34
38
|
export * from "./UtilsBedrock.js";
|
|
35
39
|
export * from "./UtilsBee.js";
|
package/dist/src/index.js
CHANGED
|
@@ -10,8 +10,10 @@ import { UtilsYoutube } from "./UtilsYoutube.js";
|
|
|
10
10
|
import { UtilsXAI } from "./UtilsXAI.js";
|
|
11
11
|
import { UtilsAnthropic } from "./UtilsAnthropic.js";
|
|
12
12
|
import { UtilsTwitter } from "./UtilsTwitter.js";
|
|
13
|
+
import { UtilsRDS } from "./UtilsRDS.js";
|
|
13
14
|
export class TriangleUtils extends UtilsMisc {
|
|
14
15
|
dynamodb;
|
|
16
|
+
rds;
|
|
15
17
|
s3;
|
|
16
18
|
s3vectors;
|
|
17
19
|
bedrock;
|
|
@@ -25,6 +27,7 @@ export class TriangleUtils extends UtilsMisc {
|
|
|
25
27
|
constructor(config) {
|
|
26
28
|
super(config);
|
|
27
29
|
this.dynamodb = new UtilsDynamoDB(config.region);
|
|
30
|
+
this.rds = new UtilsRDS(config.region, config.rds_cluster_arn, config.rds_secret_arn);
|
|
28
31
|
this.s3 = new UtilsS3(config.region);
|
|
29
32
|
this.s3vectors = new UtilsS3Vectors(config.region);
|
|
30
33
|
this.bedrock = new UtilsBedrock(config.region);
|
|
@@ -42,6 +45,7 @@ export * from "./types/GlobalConfig.js";
|
|
|
42
45
|
export * from "./UtilsMisc.js";
|
|
43
46
|
export * from "./UtilsAnthropic.js";
|
|
44
47
|
export * from "./UtilsDynamoDB.js";
|
|
48
|
+
export * from "./UtilsRDS.js";
|
|
45
49
|
export * from "./UtilsS3.js";
|
|
46
50
|
export * from "./UtilsBedrock.js";
|
|
47
51
|
export * from "./UtilsBee.js";
|
|
@@ -8,6 +8,8 @@ export declare class GlobalConfig {
|
|
|
8
8
|
readonly s3_prism: string;
|
|
9
9
|
readonly s3_scout: string;
|
|
10
10
|
readonly s3_triage: string;
|
|
11
|
+
readonly rds_cluster_arn: string;
|
|
12
|
+
readonly rds_secret_arn: string;
|
|
11
13
|
readonly super_federal_gov_api_keys: string;
|
|
12
14
|
readonly federal_gov_api_keys: string;
|
|
13
15
|
readonly federal_census_api_key: string;
|
|
@@ -8,6 +8,8 @@ export class GlobalConfig {
|
|
|
8
8
|
s3_prism;
|
|
9
9
|
s3_scout;
|
|
10
10
|
s3_triage;
|
|
11
|
+
rds_cluster_arn;
|
|
12
|
+
rds_secret_arn;
|
|
11
13
|
super_federal_gov_api_keys;
|
|
12
14
|
federal_gov_api_keys;
|
|
13
15
|
federal_census_api_key;
|
|
@@ -29,6 +31,8 @@ export class GlobalConfig {
|
|
|
29
31
|
this.s3_prism = global_config.s3_prism;
|
|
30
32
|
this.s3_scout = global_config.s3_scout;
|
|
31
33
|
this.s3_triage = global_config.s3_triage;
|
|
34
|
+
this.rds_cluster_arn = global_config.rds_cluster_arn;
|
|
35
|
+
this.rds_secret_arn = global_config.rds_secret_arn;
|
|
32
36
|
this.super_federal_gov_api_keys = global_config.super_federal_gov_api_keys;
|
|
33
37
|
this.federal_gov_api_keys = global_config.federal_gov_api_keys;
|
|
34
38
|
this.federal_census_api_key = global_config.federal_census_api_key;
|
|
@@ -49,6 +53,8 @@ export class GlobalConfig {
|
|
|
49
53
|
typeof global_config.s3_prism === "string" &&
|
|
50
54
|
typeof global_config.s3_scout === "string" &&
|
|
51
55
|
typeof global_config.s3_triage === "string" &&
|
|
56
|
+
typeof global_config.rds_cluster_arn === "string" &&
|
|
57
|
+
typeof global_config.rds_secret_arn === "string" &&
|
|
52
58
|
typeof global_config.super_federal_gov_api_keys === "string" &&
|
|
53
59
|
typeof global_config.federal_gov_api_keys === "string" &&
|
|
54
60
|
typeof global_config.federal_census_api_key === "string" &&
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "triangle-utils",
|
|
3
|
-
"version": "1.4.
|
|
3
|
+
"version": "1.4.122",
|
|
4
4
|
"main": "dist/src/index.js",
|
|
5
5
|
"types": "dist/src/index.d.ts",
|
|
6
6
|
"directories": {
|
|
@@ -20,6 +20,8 @@
|
|
|
20
20
|
"@aws-sdk/client-bedrock-runtime": "^3.953.0",
|
|
21
21
|
"@aws-sdk/client-cognito-identity-provider": "^3.1004.0",
|
|
22
22
|
"@aws-sdk/client-dynamodb": "^3.953.0",
|
|
23
|
+
"@aws-sdk/client-rds": "^3.1111.0",
|
|
24
|
+
"@aws-sdk/client-rds-data": "^3.1111.0",
|
|
23
25
|
"@aws-sdk/client-s3": "^3.1106.0",
|
|
24
26
|
"@aws-sdk/client-s3vectors": "^3.953.0",
|
|
25
27
|
"@aws-sdk/client-secrets-manager": "^3.965.0",
|
package/src/UtilsMisc.ts
CHANGED
|
@@ -55,8 +55,8 @@ export class UtilsMisc {
|
|
|
55
55
|
}
|
|
56
56
|
|
|
57
57
|
async batch_iterate<T, U>(inputs : T[], f : (inputs : T[], iterator_id : number, index : number) => Promise<U>,
|
|
58
|
-
options : {
|
|
59
|
-
const
|
|
58
|
+
options : { batch_num_items? : number, num_iterators? : number, verbosity? : number | boolean }) {
|
|
59
|
+
const batch_num_items = options.batch_num_items || 1
|
|
60
60
|
const num_iterators = options.num_iterators || 1
|
|
61
61
|
const verbosity : number = typeof options.verbosity === "boolean" ? (options.verbosity === true ? 1 : 0) : (options.verbosity || 0)
|
|
62
62
|
let index = 0
|
|
@@ -66,11 +66,11 @@ export class UtilsMisc {
|
|
|
66
66
|
iterators.push((async () => {
|
|
67
67
|
while (index < inputs.length) {
|
|
68
68
|
const local_index = index
|
|
69
|
-
index +=
|
|
69
|
+
index += batch_num_items
|
|
70
70
|
if (verbosity !== undefined && (local_index % verbosity === 0)) {
|
|
71
71
|
console.log(iterator_id + ":" + local_index + "/" + inputs.length)
|
|
72
72
|
}
|
|
73
|
-
const output = await this.safe_run(() => f(inputs.slice(local_index, local_index +
|
|
73
|
+
const output = await this.safe_run(() => f(inputs.slice(local_index, local_index + batch_num_items), iterator_id, local_index))
|
|
74
74
|
outputs[local_index] = output
|
|
75
75
|
}
|
|
76
76
|
})())
|
package/src/UtilsRDS.ts
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
import { Field, RDSData } from "@aws-sdk/client-rds-data"
|
|
2
|
+
|
|
3
|
+
function convert_rds_output(rds_output : Field) : string | boolean | number | undefined {
|
|
4
|
+
if (rds_output.isNull) {
|
|
5
|
+
return undefined
|
|
6
|
+
} else if (rds_output.stringValue !== undefined) {
|
|
7
|
+
return rds_output.stringValue
|
|
8
|
+
} else if (rds_output.longValue !== undefined) {
|
|
9
|
+
return rds_output.longValue
|
|
10
|
+
} else if (rds_output.doubleValue !== undefined) {
|
|
11
|
+
return rds_output.doubleValue
|
|
12
|
+
} else if (rds_output.booleanValue !== undefined) {
|
|
13
|
+
return rds_output.booleanValue
|
|
14
|
+
}
|
|
15
|
+
return undefined
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function convert_rds_input(input : string | boolean | number | undefined) : string {
|
|
19
|
+
if (input === undefined) {
|
|
20
|
+
return "NULL"
|
|
21
|
+
} else if (typeof input === "string") {
|
|
22
|
+
return "'" + input + "'"
|
|
23
|
+
} else if (typeof input === "number") {
|
|
24
|
+
return input.toString()
|
|
25
|
+
} else if (typeof input === "boolean") {
|
|
26
|
+
return input.toString().toUpperCase()
|
|
27
|
+
}
|
|
28
|
+
return "NULL"
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export class UtilsRDS {
|
|
32
|
+
|
|
33
|
+
private readonly rds : RDSData
|
|
34
|
+
private readonly rds_cluster_arn : string
|
|
35
|
+
private readonly rds_secret_arn : string
|
|
36
|
+
|
|
37
|
+
constructor(region : string, rds_cluster_arn : string, rds_secret_arn : string) {
|
|
38
|
+
this.rds = new RDSData({ region : region })
|
|
39
|
+
this.rds_cluster_arn = rds_cluster_arn,
|
|
40
|
+
this.rds_secret_arn = rds_secret_arn
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
async exec(sql : string) {
|
|
44
|
+
const response = await this.rds.executeStatement({
|
|
45
|
+
resourceArn : this.rds_cluster_arn,
|
|
46
|
+
secretArn : this.rds_secret_arn,
|
|
47
|
+
database : "postgres",
|
|
48
|
+
sql : sql
|
|
49
|
+
})
|
|
50
|
+
return response.records || []
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
async get(
|
|
54
|
+
table_id : string,
|
|
55
|
+
primary_key : Record<string, any>,
|
|
56
|
+
attribute_names : string[],
|
|
57
|
+
options : {
|
|
58
|
+
is_verbose? : boolean
|
|
59
|
+
} = {}
|
|
60
|
+
) {
|
|
61
|
+
const is_verbose = Boolean(options.is_verbose)
|
|
62
|
+
const sql = "SELECT " + attribute_names.join(", ") + " FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, value]) => key_name + " = " + convert_rds_input(value)).join(" AND ")
|
|
63
|
+
if (is_verbose) {
|
|
64
|
+
console.log(sql)
|
|
65
|
+
}
|
|
66
|
+
const items = await this.exec(sql)
|
|
67
|
+
const item = items[0]
|
|
68
|
+
if (item === undefined) {
|
|
69
|
+
return undefined
|
|
70
|
+
}
|
|
71
|
+
return Object.fromEntries(attribute_names.map((attribute_name, i) => [attribute_name, convert_rds_output(item[i])]))
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
async batch_get(
|
|
75
|
+
table_id : string,
|
|
76
|
+
primary_key : Record<string, any[]>,
|
|
77
|
+
attribute_names : string[],
|
|
78
|
+
options : {
|
|
79
|
+
is_verbose? : boolean
|
|
80
|
+
} = {}
|
|
81
|
+
) {
|
|
82
|
+
const is_verbose = Boolean(options.is_verbose)
|
|
83
|
+
const sql = "SELECT " + attribute_names.join(", ") + " FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, values]) => key_name + " IN " + "(" + values.map(value => convert_rds_input(value)).join(", ") + ")").join(" AND ")
|
|
84
|
+
if (is_verbose) {
|
|
85
|
+
console.log(sql)
|
|
86
|
+
}
|
|
87
|
+
const items = await this.exec(sql)
|
|
88
|
+
return items.map(item => Object.fromEntries(attribute_names.map((attribute_name, i) => [attribute_name, convert_rds_output(item[i])])))
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
async put(
|
|
92
|
+
table_id : string,
|
|
93
|
+
item : Record<string, any>,
|
|
94
|
+
options : {
|
|
95
|
+
is_verbose? : boolean
|
|
96
|
+
} = {}
|
|
97
|
+
) {
|
|
98
|
+
const is_verbose = Boolean(options.is_verbose)
|
|
99
|
+
const attribute_names = Object.keys(item).sort()
|
|
100
|
+
const sql = "INSERT INTO " + table_id + " (" + attribute_names.join(", ") + ")" + " VALUES " + "(" + attribute_names.map(attribute_name => convert_rds_input(item[attribute_name])).join(", ") + ")" + " ON CONFLICT ON CONSTRAINT " + table_id + "_pkey DO UPDATE SET " + attribute_names.map(attribute_name => attribute_name + " = " + "EXCLUDED." + attribute_name).join(", ") + ";"
|
|
101
|
+
if (is_verbose) {
|
|
102
|
+
console.log(sql)
|
|
103
|
+
}
|
|
104
|
+
try {
|
|
105
|
+
const items = await this.exec(sql)
|
|
106
|
+
return items.map(item => Object.fromEntries(attribute_names.map((attribute_name, i) => [attribute_name, convert_rds_output(item[i])])))
|
|
107
|
+
} catch (error : any) {
|
|
108
|
+
console.log(error.stack)
|
|
109
|
+
return
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
async batch_put(
|
|
114
|
+
table_id : string,
|
|
115
|
+
items : Record<string, any>[],
|
|
116
|
+
options : {
|
|
117
|
+
is_verbose? : boolean
|
|
118
|
+
} = {}
|
|
119
|
+
) {
|
|
120
|
+
const is_verbose = Boolean(options.is_verbose)
|
|
121
|
+
const attribute_names = Array.from(new Set(items.map(item => Object.keys(item)).flat())).sort()
|
|
122
|
+
const sql = "INSERT INTO " + table_id + " (" + attribute_names.join(", ") + ")" + " VALUES " + items.map(item => "(" + attribute_names.map(attribute_name => convert_rds_input(item[attribute_name])).join(", ") + ")").join(", ") + " ON CONFLICT ON CONSTRAINT " + table_id + "_pkey DO UPDATE SET " + attribute_names.map(attribute_name => attribute_name + " = " + "EXCLUDED." + attribute_name).join(", ") + ";"
|
|
123
|
+
if (is_verbose) {
|
|
124
|
+
console.log(sql)
|
|
125
|
+
}
|
|
126
|
+
try {
|
|
127
|
+
const items = await this.exec(sql)
|
|
128
|
+
return items.map(item => Object.fromEntries(attribute_names.map((attribute_name, i) => [attribute_name, convert_rds_output(item[i])])))
|
|
129
|
+
} catch (error : any) {
|
|
130
|
+
console.log(error.stack)
|
|
131
|
+
return
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
async delete(
|
|
136
|
+
table_id : string,
|
|
137
|
+
primary_key : Record<string, any>,
|
|
138
|
+
options : {
|
|
139
|
+
is_verbose? : boolean
|
|
140
|
+
} = {}
|
|
141
|
+
) {
|
|
142
|
+
const is_verbose = Boolean(options.is_verbose)
|
|
143
|
+
const sql = "DELETE FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, value]) => key_name + " = " + convert_rds_input(value)).join(" AND ")
|
|
144
|
+
if (is_verbose) {
|
|
145
|
+
console.log(sql)
|
|
146
|
+
}
|
|
147
|
+
const items = await this.exec(sql)
|
|
148
|
+
const item = items[0]
|
|
149
|
+
if (item === undefined) {
|
|
150
|
+
return undefined
|
|
151
|
+
}
|
|
152
|
+
return undefined
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
async batch_delete(
|
|
156
|
+
table_id : string,
|
|
157
|
+
primary_key : Record<string, any[]>,
|
|
158
|
+
options : {
|
|
159
|
+
is_verbose? : boolean
|
|
160
|
+
} = {}
|
|
161
|
+
) {
|
|
162
|
+
const is_verbose = Boolean(options.is_verbose)
|
|
163
|
+
const sql = "DELETE FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, values]) => key_name + " IN " + "(" + values.map(value => convert_rds_input(value)).join(", ") + ")").join(" AND ")
|
|
164
|
+
if (is_verbose) {
|
|
165
|
+
console.log(sql)
|
|
166
|
+
}
|
|
167
|
+
await this.exec(sql)
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
}
|
package/src/f.ts
CHANGED
|
@@ -24,258 +24,17 @@ console.log(config)
|
|
|
24
24
|
|
|
25
25
|
const utils = new TriangleUtils(config)
|
|
26
26
|
|
|
27
|
-
|
|
28
|
-
|
|
27
|
+
const federal_censuses = await utils.dynamodb.scan("federal_censuses")
|
|
28
|
+
federal_censuses.sort((a, b) => a.federal_census_id - b.federal_census_id)
|
|
29
29
|
|
|
30
|
-
// console.log(
|
|
30
|
+
// console.log(JSON.stringify(response, undefined, 4))
|
|
31
31
|
|
|
32
|
-
//
|
|
33
|
-
//
|
|
34
|
-
//
|
|
35
|
-
// type : "object",
|
|
36
|
-
// properties : {
|
|
37
|
-
// username : {
|
|
38
|
-
// type: "string"
|
|
39
|
-
// },
|
|
40
|
-
// explanation : {
|
|
41
|
-
// type: "string"
|
|
42
|
-
// }
|
|
43
|
-
// },
|
|
44
|
-
// required : [
|
|
45
|
-
// "username",
|
|
46
|
-
// "explanation"
|
|
47
|
-
// ],
|
|
48
|
-
// additionalProperties : false
|
|
49
|
-
// }
|
|
50
|
-
// })
|
|
51
|
-
// console.log(text)
|
|
32
|
+
// await utils.rds.batch_delete("federal_censuses", {
|
|
33
|
+
// federal_census_id : federal_censuses.map(federal_census => federal_census.federal_census_id)
|
|
34
|
+
// }, { is_verbose : true })
|
|
52
35
|
|
|
53
|
-
|
|
36
|
+
await utils.rds.batch_put("federal_censuses", federal_censuses)
|
|
54
37
|
|
|
55
|
-
|
|
38
|
+
const rds_federal_censuses = await utils.rds.batch_get("federal_censuses", { federal_census_id : federal_censuses.map(federal_census => federal_census.federal_census_id) }, ["federal_census_id", "is_decennial"], { is_verbose : true })
|
|
56
39
|
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
// console.log(text)
|
|
60
|
-
|
|
61
|
-
// const text = await utils.bedrock.claude_invoke("global.anthropic.claude-opus-4-6-v1", "hi", undefined, { print_usage : true })
|
|
62
|
-
// console.log(text)
|
|
63
|
-
|
|
64
|
-
// const url = "https://dataviewers.tdec.tn.gov/dataviewers/f?p=2005:34051:3300341444471:::34051:P34051_PERMIT_NUMBER:TNR136379"
|
|
65
|
-
|
|
66
|
-
// const html = await utils.bee.get(url, { render_js : false })
|
|
67
|
-
|
|
68
|
-
// console.log(html)
|
|
69
|
-
|
|
70
|
-
// const prompt = "Is Mark Kelly a veteran?"
|
|
71
|
-
// console.log(prompt)
|
|
72
|
-
// const output = await utils.anthropic.claude_query("claude-opus-4-8",
|
|
73
|
-
// prompt,
|
|
74
|
-
// undefined,
|
|
75
|
-
// {
|
|
76
|
-
// print_usage : true,
|
|
77
|
-
// max_tokens : 20000,
|
|
78
|
-
// json_format : {
|
|
79
|
-
// type : "object",
|
|
80
|
-
// properties : {
|
|
81
|
-
// is_veteran : {
|
|
82
|
-
// type : "string"
|
|
83
|
-
// },
|
|
84
|
-
// explanation : {
|
|
85
|
-
// type : "string"
|
|
86
|
-
// }
|
|
87
|
-
// },
|
|
88
|
-
// required : ["is_veteran", "explanation"],
|
|
89
|
-
// additionalProperties : false
|
|
90
|
-
// },
|
|
91
|
-
// web : true
|
|
92
|
-
// }
|
|
93
|
-
// )
|
|
94
|
-
|
|
95
|
-
// console.log(output)
|
|
96
|
-
|
|
97
|
-
// import { Client } from "@xdevplatform/xdk"
|
|
98
|
-
|
|
99
|
-
// const x = new Client({ bearerToken : "AAAAAAAAAAAAAAAAAAAAAKoQ%2BQEAAAAAaej7PP534%2Fd9dKmieHRY6vQkv2s%3DyY2VerXmrA7w794ACFr9ALvoaJfuAHSlWwZdiDeVaOVpA6HCub" })
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
// const user = await x.users.getByUsername("umichvoter", { userFields : [
|
|
103
|
-
// "affiliation",
|
|
104
|
-
// // "confirmed_email",
|
|
105
|
-
// "connection_status",
|
|
106
|
-
// "created_at",
|
|
107
|
-
// "description",
|
|
108
|
-
// "entities",
|
|
109
|
-
// "id",
|
|
110
|
-
// "is_identity_verified",
|
|
111
|
-
// "location",
|
|
112
|
-
// "most_recent_tweet_id",
|
|
113
|
-
// "name",
|
|
114
|
-
// "parody",
|
|
115
|
-
// "pinned_tweet_id",
|
|
116
|
-
// "profile_banner_url",
|
|
117
|
-
// "profile_image_url",
|
|
118
|
-
// "protected",
|
|
119
|
-
// "public_metrics",
|
|
120
|
-
// // "receives_your_dm",
|
|
121
|
-
// "subscription",
|
|
122
|
-
// "subscription_type",
|
|
123
|
-
// "url",
|
|
124
|
-
// "username",
|
|
125
|
-
// "verified",
|
|
126
|
-
// "verified_followers_count",
|
|
127
|
-
// "verified_type",
|
|
128
|
-
// "withheld"
|
|
129
|
-
// ] })
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
// console.log(JSON.stringify(user, null, 4))
|
|
133
|
-
|
|
134
|
-
// const tweets = await x.users.getPosts("1603470469608374272", {
|
|
135
|
-
|
|
136
|
-
// tweetFields : [
|
|
137
|
-
// "article",
|
|
138
|
-
// "attachments",
|
|
139
|
-
// "author_id",
|
|
140
|
-
// "card_uri",
|
|
141
|
-
// "community_id",
|
|
142
|
-
// "context_annotations",
|
|
143
|
-
// "conversation_id",
|
|
144
|
-
// "created_at",
|
|
145
|
-
// "display_text_range",
|
|
146
|
-
// "edit_controls",
|
|
147
|
-
// "edit_history_tweet_ids",
|
|
148
|
-
// "entities",
|
|
149
|
-
// "geo",
|
|
150
|
-
// "id",
|
|
151
|
-
// "in_reply_to_user_id",
|
|
152
|
-
// "lang",
|
|
153
|
-
// "matched_media_notes",
|
|
154
|
-
// "media_metadata",
|
|
155
|
-
// "note_request_suggestions",
|
|
156
|
-
// "note_tweet",
|
|
157
|
-
// "paid_partnership",
|
|
158
|
-
// "possibly_sensitive",
|
|
159
|
-
// "referenced_tweets",
|
|
160
|
-
// "reply_settings",
|
|
161
|
-
// "scopes",
|
|
162
|
-
// "source",
|
|
163
|
-
// "suggested_source_links",
|
|
164
|
-
// "suggested_source_links_with_counts",
|
|
165
|
-
// "text",
|
|
166
|
-
// "withheld"
|
|
167
|
-
// ], expansions : [
|
|
168
|
-
// "article.cover_media",
|
|
169
|
-
// "article.media_entities",
|
|
170
|
-
// "attachments.media_keys",
|
|
171
|
-
// "attachments.media_source_tweet",
|
|
172
|
-
// "attachments.poll_ids",
|
|
173
|
-
// "author_id",
|
|
174
|
-
// "edit_history_tweet_ids",
|
|
175
|
-
// "entities.mentions.username",
|
|
176
|
-
// "geo.place_id",
|
|
177
|
-
// "in_reply_to_user_id",
|
|
178
|
-
// "entities.note.mentions.username",
|
|
179
|
-
// "referenced_tweets.id",
|
|
180
|
-
// "referenced_tweets.id.attachments.media_keys",
|
|
181
|
-
// "referenced_tweets.id.author_id"
|
|
182
|
-
// ]})
|
|
183
|
-
|
|
184
|
-
// console.log(JSON.stringify(tweets, null, 4))
|
|
185
|
-
|
|
186
|
-
// for (const tweet of tweets.data || []) {
|
|
187
|
-
// console.log(JSON.stringify(tweet, undefined, 4))
|
|
188
|
-
// }
|
|
189
|
-
|
|
190
|
-
// const tweet = await x.posts.getById("2061603004248154195", { tweetFields : [
|
|
191
|
-
// "article",
|
|
192
|
-
// "attachments",
|
|
193
|
-
// "author_id",
|
|
194
|
-
// "card_uri",
|
|
195
|
-
// "community_id",
|
|
196
|
-
// "context_annotations",
|
|
197
|
-
// "conversation_id",
|
|
198
|
-
// "created_at",
|
|
199
|
-
// "display_text_range",
|
|
200
|
-
// "edit_controls",
|
|
201
|
-
// "edit_history_tweet_ids",
|
|
202
|
-
// "entities",
|
|
203
|
-
// "geo",
|
|
204
|
-
// "id",
|
|
205
|
-
// "in_reply_to_user_id",
|
|
206
|
-
// "lang",
|
|
207
|
-
// "matched_media_notes",
|
|
208
|
-
// "media_metadata",
|
|
209
|
-
// // "non_public_metrics",
|
|
210
|
-
// "note_request_suggestions",
|
|
211
|
-
// "note_tweet",
|
|
212
|
-
// // "organic_metrics",
|
|
213
|
-
// "paid_partnership",
|
|
214
|
-
// "possibly_sensitive",
|
|
215
|
-
// // "promoted_metrics",
|
|
216
|
-
// "public_metrics",
|
|
217
|
-
// "referenced_tweets",
|
|
218
|
-
// "reply_settings",
|
|
219
|
-
// "scopes",
|
|
220
|
-
// "source",
|
|
221
|
-
// "suggested_source_links",
|
|
222
|
-
// "suggested_source_links_with_counts",
|
|
223
|
-
// "text",
|
|
224
|
-
// "withheld"
|
|
225
|
-
// ], expansions : [
|
|
226
|
-
// "article.cover_media",
|
|
227
|
-
// "article.media_entities",
|
|
228
|
-
// "attachments.media_keys",
|
|
229
|
-
// "attachments.media_source_tweet",
|
|
230
|
-
// "attachments.poll_ids",
|
|
231
|
-
// "author_id",
|
|
232
|
-
// "edit_history_tweet_ids",
|
|
233
|
-
// "entities.mentions.username",
|
|
234
|
-
// "geo.place_id",
|
|
235
|
-
// "in_reply_to_user_id",
|
|
236
|
-
// "entities.note.mentions.username",
|
|
237
|
-
// "referenced_tweets.id",
|
|
238
|
-
// "referenced_tweets.id.attachments.media_keys",
|
|
239
|
-
// "referenced_tweets.id.author_id"
|
|
240
|
-
// ], pollFields : [
|
|
241
|
-
// "duration_minutes",
|
|
242
|
-
// "end_datetime",
|
|
243
|
-
// "id",
|
|
244
|
-
// "options",
|
|
245
|
-
// "voting_status"
|
|
246
|
-
// ]
|
|
247
|
-
// })
|
|
248
|
-
|
|
249
|
-
// console.log(JSON.stringify(tweet, undefined, 4))
|
|
250
|
-
|
|
251
|
-
// const results = await utils.twitter.get_tweets("1603470469608374272", {
|
|
252
|
-
// compile : true,
|
|
253
|
-
// min_twitter_tweet_id : "2061610563679985813"
|
|
254
|
-
// })
|
|
255
|
-
|
|
256
|
-
// console.log(results)
|
|
257
|
-
|
|
258
|
-
// const f = await utils.dynamodb.query_range("federal_election_donations:donor_address_state_id.federal_election_committee_id-receipt_date-federal_election_donation_id", { donor_address_state_id : "NH", federal_election_committee_id : "C00694323" }, { receipt_date : ["2021", "2023"] }, { compile : false })
|
|
259
|
-
|
|
260
|
-
// console.log(f)
|
|
261
|
-
// console.log("YERP")
|
|
262
|
-
// const response = await utils.bedrock.claude_converse(
|
|
263
|
-
// "anthropic.claude-opus-4-8",
|
|
264
|
-
// [
|
|
265
|
-
// { role : "user", text : "Hi" }
|
|
266
|
-
// ],
|
|
267
|
-
// {
|
|
268
|
-
// print_usage : true
|
|
269
|
-
// }
|
|
270
|
-
// )
|
|
271
|
-
// console.log(response)
|
|
272
|
-
|
|
273
|
-
// const coordinates = await utils.bedrock.cohere_invoke("global.cohere.embed-v4:0", "yerrp")
|
|
274
|
-
|
|
275
|
-
// console.log(coordinates)
|
|
276
|
-
|
|
277
|
-
const scout_ids = await utils.dynamodb.scan("scouts")
|
|
278
|
-
.then(scouts => scouts.map(scout => scout.scout_id))
|
|
279
|
-
|
|
280
|
-
const scouts = await utils.dynamodb.batch_get("scouts", scout_ids.slice(0, 150).map(scout_id => ({ scout_id : scout_id })))
|
|
281
|
-
console.log(scouts.length)
|
|
40
|
+
console.log(rds_federal_censuses)
|
package/src/index.ts
CHANGED
|
@@ -11,12 +11,15 @@ import { UtilsXAI } from "./UtilsXAI"
|
|
|
11
11
|
import { UtilsAnthropic } from "./UtilsAnthropic"
|
|
12
12
|
import { UtilsTwitter } from "./UtilsTwitter"
|
|
13
13
|
import { GlobalConfig } from "./types/GlobalConfig"
|
|
14
|
+
import { UtilsRDS } from "./UtilsRDS"
|
|
15
|
+
import { ApplicationConfig } from "./types/ApplicationConfig"
|
|
14
16
|
|
|
15
17
|
|
|
16
18
|
|
|
17
19
|
export class TriangleUtils extends UtilsMisc {
|
|
18
20
|
|
|
19
21
|
readonly dynamodb : UtilsDynamoDB
|
|
22
|
+
readonly rds : UtilsRDS
|
|
20
23
|
readonly s3 : UtilsS3
|
|
21
24
|
readonly s3vectors : UtilsS3Vectors
|
|
22
25
|
readonly bedrock : UtilsBedrock
|
|
@@ -28,9 +31,10 @@ export class TriangleUtils extends UtilsMisc {
|
|
|
28
31
|
readonly xai : UtilsXAI
|
|
29
32
|
readonly twitter : UtilsTwitter
|
|
30
33
|
|
|
31
|
-
constructor(config : GlobalConfig) {
|
|
34
|
+
constructor(config : ApplicationConfig & GlobalConfig) {
|
|
32
35
|
super(config)
|
|
33
36
|
this.dynamodb = new UtilsDynamoDB(config.region)
|
|
37
|
+
this.rds = new UtilsRDS(config.region, config.rds_cluster_arn, config.rds_secret_arn)
|
|
34
38
|
this.s3 = new UtilsS3(config.region)
|
|
35
39
|
this.s3vectors = new UtilsS3Vectors(config.region)
|
|
36
40
|
this.bedrock = new UtilsBedrock(config.region)
|
|
@@ -49,6 +53,7 @@ export * from "./types/GlobalConfig"
|
|
|
49
53
|
export * from "./UtilsMisc"
|
|
50
54
|
export * from "./UtilsAnthropic"
|
|
51
55
|
export * from "./UtilsDynamoDB"
|
|
56
|
+
export * from "./UtilsRDS"
|
|
52
57
|
export * from "./UtilsS3"
|
|
53
58
|
export * from "./UtilsBedrock"
|
|
54
59
|
export * from "./UtilsBee"
|
|
@@ -8,6 +8,8 @@ export class GlobalConfig {
|
|
|
8
8
|
readonly s3_prism : string
|
|
9
9
|
readonly s3_scout : string
|
|
10
10
|
readonly s3_triage : string
|
|
11
|
+
readonly rds_cluster_arn : string
|
|
12
|
+
readonly rds_secret_arn : string
|
|
11
13
|
readonly super_federal_gov_api_keys : string
|
|
12
14
|
readonly federal_gov_api_keys : string
|
|
13
15
|
readonly federal_census_api_key : string
|
|
@@ -31,6 +33,8 @@ export class GlobalConfig {
|
|
|
31
33
|
this.s3_prism = global_config.s3_prism
|
|
32
34
|
this.s3_scout = global_config.s3_scout
|
|
33
35
|
this.s3_triage = global_config.s3_triage
|
|
36
|
+
this.rds_cluster_arn = global_config.rds_cluster_arn
|
|
37
|
+
this.rds_secret_arn = global_config.rds_secret_arn
|
|
34
38
|
this.super_federal_gov_api_keys = global_config.super_federal_gov_api_keys
|
|
35
39
|
this.federal_gov_api_keys = global_config.federal_gov_api_keys
|
|
36
40
|
this.federal_census_api_key = global_config.federal_census_api_key
|
|
@@ -53,6 +57,8 @@ export class GlobalConfig {
|
|
|
53
57
|
typeof global_config.s3_prism === "string" &&
|
|
54
58
|
typeof global_config.s3_scout === "string" &&
|
|
55
59
|
typeof global_config.s3_triage === "string" &&
|
|
60
|
+
typeof global_config.rds_cluster_arn === "string" &&
|
|
61
|
+
typeof global_config.rds_secret_arn === "string" &&
|
|
56
62
|
typeof global_config.super_federal_gov_api_keys === "string" &&
|
|
57
63
|
typeof global_config.federal_gov_api_keys === "string" &&
|
|
58
64
|
typeof global_config.federal_census_api_key === "string" &&
|