triangle-utils 1.4.121 → 1.4.123
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/UtilsRDS.d.ts +40 -0
- package/dist/src/UtilsRDS.js +156 -0
- package/dist/src/f.js +50 -221
- package/dist/src/index.d.ts +5 -1
- package/dist/src/index.js +4 -0
- package/dist/src/types/GlobalConfig.d.ts +5 -0
- package/dist/src/types/GlobalConfig.js +15 -0
- package/package.json +5 -1
- package/src/UtilsRDS.ts +234 -0
- package/src/f.ts +51 -247
- package/src/index.ts +6 -1
- package/src/types/GlobalConfig.ts +15 -0
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { Client } from "pg";
|
|
2
|
+
export declare class UtilsRDS {
|
|
3
|
+
private readonly postgres;
|
|
4
|
+
private readonly rds_data;
|
|
5
|
+
private readonly rds_cluster_arn;
|
|
6
|
+
private readonly rds_secret_arn;
|
|
7
|
+
constructor(region: string, rds_cluster_arn: string, rds_secret_arn: string, rds_host: string, rds_username: string, rds_password: string);
|
|
8
|
+
connect(): Promise<Client>;
|
|
9
|
+
disconnect(): Promise<void>;
|
|
10
|
+
query(sql: string, options?: {
|
|
11
|
+
is_verbose?: boolean;
|
|
12
|
+
}): Promise<any[]>;
|
|
13
|
+
exec(sql: string, options?: {
|
|
14
|
+
is_verbose?: boolean;
|
|
15
|
+
}): Promise<any[]>;
|
|
16
|
+
get(table_id: string, primary_key: Record<string, any>, attribute_names?: string[], options?: {
|
|
17
|
+
is_verbose?: boolean;
|
|
18
|
+
}): Promise<Record<string, any> | undefined>;
|
|
19
|
+
batch_get(table_id: string, primary_key: Record<string, any[]>, attribute_names?: string[], options?: {
|
|
20
|
+
is_verbose?: boolean;
|
|
21
|
+
}): Promise<any[]>;
|
|
22
|
+
create(table_id: string, item: Record<string, any>, options?: {
|
|
23
|
+
is_verbose?: boolean;
|
|
24
|
+
}): Promise<any[] | undefined>;
|
|
25
|
+
batch_create(table_id: string, items: Record<string, any>[], options?: {
|
|
26
|
+
is_verbose?: boolean;
|
|
27
|
+
}): Promise<any[] | undefined>;
|
|
28
|
+
put(table_id: string, item: Record<string, any>, options?: {
|
|
29
|
+
is_verbose?: boolean;
|
|
30
|
+
}): Promise<any[] | undefined>;
|
|
31
|
+
batch_put(table_id: string, items: Record<string, any>[], options?: {
|
|
32
|
+
is_verbose?: boolean;
|
|
33
|
+
}): Promise<any[] | undefined>;
|
|
34
|
+
delete(table_id: string, primary_key: Record<string, any>, options?: {
|
|
35
|
+
is_verbose?: boolean;
|
|
36
|
+
}): Promise<any[]>;
|
|
37
|
+
batch_delete(table_id: string, primary_key: Record<string, any[]>, options?: {
|
|
38
|
+
is_verbose?: boolean;
|
|
39
|
+
}): Promise<any[]>;
|
|
40
|
+
}
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
import { RDSData } from "@aws-sdk/client-rds-data";
|
|
2
|
+
import { Client } from "pg";
|
|
3
|
+
function convert_rds_input(input) {
|
|
4
|
+
if (input === undefined) {
|
|
5
|
+
return "NULL";
|
|
6
|
+
}
|
|
7
|
+
else if (typeof input === "string") {
|
|
8
|
+
return "'" + input + "'";
|
|
9
|
+
}
|
|
10
|
+
else if (typeof input === "number") {
|
|
11
|
+
return input.toString();
|
|
12
|
+
}
|
|
13
|
+
else if (typeof input === "boolean") {
|
|
14
|
+
return input.toString().toUpperCase();
|
|
15
|
+
}
|
|
16
|
+
else if (typeof input === "object") {
|
|
17
|
+
if (input.expression !== undefined) {
|
|
18
|
+
return input.expression;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
return "NULL";
|
|
22
|
+
}
|
|
23
|
+
export class UtilsRDS {
|
|
24
|
+
postgres;
|
|
25
|
+
rds_data;
|
|
26
|
+
rds_cluster_arn;
|
|
27
|
+
rds_secret_arn;
|
|
28
|
+
constructor(region, rds_cluster_arn, rds_secret_arn, rds_host, rds_username, rds_password) {
|
|
29
|
+
this.postgres = new Client({
|
|
30
|
+
host: rds_host,
|
|
31
|
+
user: rds_username,
|
|
32
|
+
password: rds_password,
|
|
33
|
+
database: "postgres",
|
|
34
|
+
port: 5432,
|
|
35
|
+
ssl: {
|
|
36
|
+
rejectUnauthorized: false
|
|
37
|
+
}
|
|
38
|
+
});
|
|
39
|
+
this.rds_data = new RDSData({ region: region });
|
|
40
|
+
this.rds_cluster_arn = rds_cluster_arn,
|
|
41
|
+
this.rds_secret_arn = rds_secret_arn;
|
|
42
|
+
}
|
|
43
|
+
async connect() {
|
|
44
|
+
return await this.postgres.connect();
|
|
45
|
+
}
|
|
46
|
+
async disconnect() {
|
|
47
|
+
return await this.postgres.end();
|
|
48
|
+
}
|
|
49
|
+
async query(sql, options = {}) {
|
|
50
|
+
const is_verbose = Boolean(options.is_verbose);
|
|
51
|
+
if (is_verbose) {
|
|
52
|
+
console.log(sql);
|
|
53
|
+
}
|
|
54
|
+
const response_connect = await this.connect();
|
|
55
|
+
if (is_verbose) {
|
|
56
|
+
console.log(response_connect);
|
|
57
|
+
}
|
|
58
|
+
const response = await this.postgres.query(sql);
|
|
59
|
+
const response_disconnect = await this.disconnect();
|
|
60
|
+
if (is_verbose) {
|
|
61
|
+
console.log(response_disconnect);
|
|
62
|
+
}
|
|
63
|
+
return response.rows;
|
|
64
|
+
}
|
|
65
|
+
async exec(sql, options = {}) {
|
|
66
|
+
const is_verbose = Boolean(options.is_verbose);
|
|
67
|
+
if (is_verbose) {
|
|
68
|
+
console.log(sql);
|
|
69
|
+
}
|
|
70
|
+
const items = sql.length <= 60000 ?
|
|
71
|
+
await this.rds_data.executeStatement({
|
|
72
|
+
resourceArn: this.rds_cluster_arn,
|
|
73
|
+
secretArn: this.rds_secret_arn,
|
|
74
|
+
database: "postgres",
|
|
75
|
+
sql: sql,
|
|
76
|
+
formatRecordsAs: "JSON"
|
|
77
|
+
})
|
|
78
|
+
.then(response => JSON.parse(response.formattedRecords || "[]"))
|
|
79
|
+
:
|
|
80
|
+
await this.query(sql, options);
|
|
81
|
+
if (!Array.isArray(items)) {
|
|
82
|
+
if (is_verbose) {
|
|
83
|
+
console.log("Bad items:", items);
|
|
84
|
+
}
|
|
85
|
+
return [];
|
|
86
|
+
}
|
|
87
|
+
return items;
|
|
88
|
+
}
|
|
89
|
+
async get(table_id, primary_key, attribute_names, options = {}) {
|
|
90
|
+
const sql = "SELECT " + (attribute_names === undefined ? "*" : attribute_names.join(", ")) + " FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, value]) => key_name + " = " + convert_rds_input(value)).join(" AND ");
|
|
91
|
+
const items = await this.exec(sql, options);
|
|
92
|
+
const item = items[0];
|
|
93
|
+
return item;
|
|
94
|
+
}
|
|
95
|
+
async batch_get(table_id, primary_key, attribute_names, options = {}) {
|
|
96
|
+
const sql = "SELECT " + (attribute_names === undefined ? "*" : attribute_names.join(", ")) + " FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, values]) => key_name + " IN " + "(" + values.map(value => convert_rds_input(value)).join(", ") + ")").join(" AND ");
|
|
97
|
+
const items = await this.exec(sql, options);
|
|
98
|
+
return items;
|
|
99
|
+
}
|
|
100
|
+
async create(table_id, item, options = {}) {
|
|
101
|
+
const attribute_names = Object.keys(item).sort();
|
|
102
|
+
const sql = "INSERT INTO " + table_id + " (" + attribute_names.join(", ") + ")" + " VALUES " + "(" + attribute_names.map(attribute_name => convert_rds_input(item[attribute_name])).join(", ") + ")" + " ON CONFLICT ON CONSTRAINT " + table_id + "_pkey IGNORE;";
|
|
103
|
+
try {
|
|
104
|
+
const items = await this.exec(sql, options);
|
|
105
|
+
return items;
|
|
106
|
+
}
|
|
107
|
+
catch (error) {
|
|
108
|
+
console.log(error.stack);
|
|
109
|
+
return;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
async batch_create(table_id, items, options = {}) {
|
|
113
|
+
const attribute_names = Array.from(new Set(items.map(item => Object.keys(item)).flat())).sort();
|
|
114
|
+
const sql = "INSERT INTO " + table_id + " (" + attribute_names.join(", ") + ")" + " VALUES " + items.map(item => "(" + attribute_names.map(attribute_name => convert_rds_input(item[attribute_name])).join(", ") + ")").join(", ") + " ON CONFLICT ON CONSTRAINT " + table_id + "_pkey IGNORE;";
|
|
115
|
+
try {
|
|
116
|
+
const items = await this.exec(sql, options);
|
|
117
|
+
return items;
|
|
118
|
+
}
|
|
119
|
+
catch (error) {
|
|
120
|
+
console.log(error.stack);
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
async put(table_id, item, options = {}) {
|
|
125
|
+
const attribute_names = Object.keys(item).sort();
|
|
126
|
+
const sql = "INSERT INTO " + table_id + " (" + attribute_names.join(", ") + ")" + " VALUES " + "(" + attribute_names.map(attribute_name => convert_rds_input(item[attribute_name])).join(", ") + ")" + " ON CONFLICT ON CONSTRAINT " + table_id + "_pkey DO UPDATE SET " + attribute_names.map(attribute_name => attribute_name + " = " + "EXCLUDED." + attribute_name).join(", ") + ";";
|
|
127
|
+
try {
|
|
128
|
+
const items = await this.exec(sql, options);
|
|
129
|
+
return items;
|
|
130
|
+
}
|
|
131
|
+
catch (error) {
|
|
132
|
+
console.log(error.stack);
|
|
133
|
+
return;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
async batch_put(table_id, items, options = {}) {
|
|
137
|
+
const attribute_names = Array.from(new Set(items.map(item => Object.keys(item)).flat())).sort();
|
|
138
|
+
const sql = "INSERT INTO " + table_id + " (" + attribute_names.join(", ") + ")" + " VALUES " + items.map(item => "(" + attribute_names.map(attribute_name => convert_rds_input(item[attribute_name])).join(", ") + ")").join(", ") + " ON CONFLICT ON CONSTRAINT " + table_id + "_pkey DO UPDATE SET " + attribute_names.map(attribute_name => attribute_name + " = " + "EXCLUDED." + attribute_name).join(", ") + ";";
|
|
139
|
+
try {
|
|
140
|
+
const items = await this.exec(sql, options);
|
|
141
|
+
return items;
|
|
142
|
+
}
|
|
143
|
+
catch (error) {
|
|
144
|
+
console.log(error.stack);
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
async delete(table_id, primary_key, options = {}) {
|
|
149
|
+
const sql = "DELETE FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, value]) => key_name + " = " + convert_rds_input(value)).join(" AND ");
|
|
150
|
+
return await this.exec(sql, options);
|
|
151
|
+
}
|
|
152
|
+
async batch_delete(table_id, primary_key, options = {}) {
|
|
153
|
+
const sql = "DELETE FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, values]) => key_name + " IN " + "(" + values.map(value => convert_rds_input(value)).join(", ") + ")").join(" AND ");
|
|
154
|
+
return await this.exec(sql, options);
|
|
155
|
+
}
|
|
156
|
+
}
|
package/dist/src/f.js
CHANGED
|
@@ -15,226 +15,55 @@ const config = {
|
|
|
15
15
|
};
|
|
16
16
|
console.log(config);
|
|
17
17
|
const utils = new TriangleUtils(config);
|
|
18
|
-
// const
|
|
19
|
-
//
|
|
20
|
-
// console.log(
|
|
21
|
-
//
|
|
22
|
-
//
|
|
23
|
-
//
|
|
24
|
-
//
|
|
25
|
-
//
|
|
26
|
-
//
|
|
27
|
-
//
|
|
28
|
-
//
|
|
29
|
-
//
|
|
30
|
-
//
|
|
31
|
-
//
|
|
32
|
-
//
|
|
33
|
-
//
|
|
34
|
-
//
|
|
35
|
-
//
|
|
36
|
-
//
|
|
37
|
-
//
|
|
18
|
+
// const federal_censuses = await utils.dynamodb.scan("federal_censuses")
|
|
19
|
+
// federal_censuses.sort((a, b) => a.federal_census_id - b.federal_census_id)
|
|
20
|
+
// console.log(JSON.stringify(response, undefined, 4))
|
|
21
|
+
// await utils.rds.batch_delete("federal_censuses", {
|
|
22
|
+
// federal_census_id : federal_censuses.map(federal_census => federal_census.federal_census_id)
|
|
23
|
+
// }, { is_verbose : true })
|
|
24
|
+
// await utils.rds.batch_put("federal_censuses", federal_censuses)
|
|
25
|
+
// const rds_federal_censuses = await utils.rds.batch_get("federal_censuses", { federal_census_id : federal_censuses.map(federal_census => federal_census.federal_census_id) }, ["federal_census_id", "is_decennial"], { is_verbose : true })
|
|
26
|
+
// console.log(rds_federal_censuses)
|
|
27
|
+
// const voters = await utils.dynamodb.scan("voters", { compile : false, max_num_items : 100 })
|
|
28
|
+
// for (const voter of voters) {
|
|
29
|
+
// const new_lal_file_voter = {
|
|
30
|
+
// lal_file_voter_id : voter.state_id + "-2026-08-15" + "|" + voter.voter_id,
|
|
31
|
+
// lal_file_id : voter.state_id + "-2026-08-15",
|
|
32
|
+
// lal_voter_id : voter.voter_id,
|
|
33
|
+
// voter_id : voter.state_voter_id,
|
|
34
|
+
// state_id : voter.state_id,
|
|
35
|
+
// address_residence_latitude : voter.address_residence_latitude,
|
|
36
|
+
// address_residence_longitude : voter.address_residence_longitude,
|
|
37
|
+
// address_residence_coordinates : "ST_GeomFromText('POINT(" + voter.address_residence_latitude + " " + voter.address_residence_longitude + ")', 4326)"
|
|
38
38
|
// }
|
|
39
|
-
//
|
|
40
|
-
//
|
|
41
|
-
//
|
|
42
|
-
//
|
|
43
|
-
// const
|
|
44
|
-
// console.log(
|
|
45
|
-
//
|
|
46
|
-
//
|
|
47
|
-
// const
|
|
48
|
-
//
|
|
49
|
-
//
|
|
50
|
-
//
|
|
51
|
-
//
|
|
52
|
-
//
|
|
53
|
-
//
|
|
54
|
-
//
|
|
55
|
-
// {
|
|
56
|
-
//
|
|
57
|
-
//
|
|
58
|
-
//
|
|
59
|
-
//
|
|
60
|
-
//
|
|
61
|
-
//
|
|
62
|
-
//
|
|
63
|
-
//
|
|
64
|
-
//
|
|
65
|
-
//
|
|
66
|
-
//
|
|
67
|
-
//
|
|
68
|
-
//
|
|
69
|
-
// additionalProperties : false
|
|
70
|
-
// },
|
|
71
|
-
// web : true
|
|
72
|
-
// }
|
|
73
|
-
// )
|
|
74
|
-
// console.log(output)
|
|
75
|
-
// import { Client } from "@xdevplatform/xdk"
|
|
76
|
-
// const x = new Client({ bearerToken : "AAAAAAAAAAAAAAAAAAAAAKoQ%2BQEAAAAAaej7PP534%2Fd9dKmieHRY6vQkv2s%3DyY2VerXmrA7w794ACFr9ALvoaJfuAHSlWwZdiDeVaOVpA6HCub" })
|
|
77
|
-
// const user = await x.users.getByUsername("umichvoter", { userFields : [
|
|
78
|
-
// "affiliation",
|
|
79
|
-
// // "confirmed_email",
|
|
80
|
-
// "connection_status",
|
|
81
|
-
// "created_at",
|
|
82
|
-
// "description",
|
|
83
|
-
// "entities",
|
|
84
|
-
// "id",
|
|
85
|
-
// "is_identity_verified",
|
|
86
|
-
// "location",
|
|
87
|
-
// "most_recent_tweet_id",
|
|
88
|
-
// "name",
|
|
89
|
-
// "parody",
|
|
90
|
-
// "pinned_tweet_id",
|
|
91
|
-
// "profile_banner_url",
|
|
92
|
-
// "profile_image_url",
|
|
93
|
-
// "protected",
|
|
94
|
-
// "public_metrics",
|
|
95
|
-
// // "receives_your_dm",
|
|
96
|
-
// "subscription",
|
|
97
|
-
// "subscription_type",
|
|
98
|
-
// "url",
|
|
99
|
-
// "username",
|
|
100
|
-
// "verified",
|
|
101
|
-
// "verified_followers_count",
|
|
102
|
-
// "verified_type",
|
|
103
|
-
// "withheld"
|
|
104
|
-
// ] })
|
|
105
|
-
// console.log(JSON.stringify(user, null, 4))
|
|
106
|
-
// const tweets = await x.users.getPosts("1603470469608374272", {
|
|
107
|
-
// tweetFields : [
|
|
108
|
-
// "article",
|
|
109
|
-
// "attachments",
|
|
110
|
-
// "author_id",
|
|
111
|
-
// "card_uri",
|
|
112
|
-
// "community_id",
|
|
113
|
-
// "context_annotations",
|
|
114
|
-
// "conversation_id",
|
|
115
|
-
// "created_at",
|
|
116
|
-
// "display_text_range",
|
|
117
|
-
// "edit_controls",
|
|
118
|
-
// "edit_history_tweet_ids",
|
|
119
|
-
// "entities",
|
|
120
|
-
// "geo",
|
|
121
|
-
// "id",
|
|
122
|
-
// "in_reply_to_user_id",
|
|
123
|
-
// "lang",
|
|
124
|
-
// "matched_media_notes",
|
|
125
|
-
// "media_metadata",
|
|
126
|
-
// "note_request_suggestions",
|
|
127
|
-
// "note_tweet",
|
|
128
|
-
// "paid_partnership",
|
|
129
|
-
// "possibly_sensitive",
|
|
130
|
-
// "referenced_tweets",
|
|
131
|
-
// "reply_settings",
|
|
132
|
-
// "scopes",
|
|
133
|
-
// "source",
|
|
134
|
-
// "suggested_source_links",
|
|
135
|
-
// "suggested_source_links_with_counts",
|
|
136
|
-
// "text",
|
|
137
|
-
// "withheld"
|
|
138
|
-
// ], expansions : [
|
|
139
|
-
// "article.cover_media",
|
|
140
|
-
// "article.media_entities",
|
|
141
|
-
// "attachments.media_keys",
|
|
142
|
-
// "attachments.media_source_tweet",
|
|
143
|
-
// "attachments.poll_ids",
|
|
144
|
-
// "author_id",
|
|
145
|
-
// "edit_history_tweet_ids",
|
|
146
|
-
// "entities.mentions.username",
|
|
147
|
-
// "geo.place_id",
|
|
148
|
-
// "in_reply_to_user_id",
|
|
149
|
-
// "entities.note.mentions.username",
|
|
150
|
-
// "referenced_tweets.id",
|
|
151
|
-
// "referenced_tweets.id.attachments.media_keys",
|
|
152
|
-
// "referenced_tweets.id.author_id"
|
|
153
|
-
// ]})
|
|
154
|
-
// console.log(JSON.stringify(tweets, null, 4))
|
|
155
|
-
// for (const tweet of tweets.data || []) {
|
|
156
|
-
// console.log(JSON.stringify(tweet, undefined, 4))
|
|
39
|
+
// console.log(new_lal_file_voter)
|
|
40
|
+
// await utils.rds.put("lal_file_voters", new_lal_file_voter)
|
|
41
|
+
// }
|
|
42
|
+
// const federal_census_county_id = "2026-04021"
|
|
43
|
+
// const federal_census_county = await utils.rds.get("federal_census_counties", { federal_census_county_id : federal_census_county_id }, ["*"])
|
|
44
|
+
// console.log(federal_census_county)
|
|
45
|
+
// console.log(await utils.rds.exec("SELECT * FROM federal_censuses"))
|
|
46
|
+
// const federal_census_county = await utils.dynamodb.get("federal_census_counties", { federal_census_county_id : federal_census_county_id })
|
|
47
|
+
// const s3_id = config.s3_triangle + "/federal_census/2026/counties/" + federal_census_county_id + ".json"
|
|
48
|
+
// console.log(s3_id)
|
|
49
|
+
// const feature = await utils.s3.get(s3_id)
|
|
50
|
+
// // console.log(feature)
|
|
51
|
+
// if (feature !== undefined && federal_census_county !== undefined) {
|
|
52
|
+
// const geometry = "(" + JSON.parse(feature).geometry.coordinates[0].map((polygon : any) => "(" + polygon.map((vertex : any) => vertex[1] + " " + vertex[0]).join(", ") + ")").join(", ") + ")"
|
|
53
|
+
// const ewkt = "'MULTIPOLYGON(" + geometry + ")'"
|
|
54
|
+
// // const response = await utils.rds.exec("SELECT ST_AsText(address_residence_coordinates) FROM lal_file_voters", { is_verbose : true })
|
|
55
|
+
// // const response = await utils.rds.exec("SELECT lal_file_voter_id FROM lal_file_voters WHERE ST_Contains(ST_GEOMFROMTEXT(" + ewkt + ", 4326), lal_file_voters.address_residence_coordinates)", { is_verbose : true })
|
|
56
|
+
// const response = await utils.rds.put("federal_census_counties", {
|
|
57
|
+
// federal_census_county_id : federal_census_county.federal_census_county_id,
|
|
58
|
+
// federal_census_id : federal_census_county.federal_census_id,
|
|
59
|
+
// county_id : federal_census_county.county_id,
|
|
60
|
+
// state_id : federal_census_county.state_id,
|
|
61
|
+
// county_type_id : federal_census_county.county_type_id,
|
|
62
|
+
// function_type_id : federal_census_county.function_type_id,
|
|
63
|
+
// name : federal_census_county.name,
|
|
64
|
+
// geometry : {
|
|
65
|
+
// expression : "ST_GEOMFROMTEXT(" + ewkt + ", 4326)"
|
|
66
|
+
// }
|
|
67
|
+
// }, { is_verbose : true })
|
|
68
|
+
// console.log(response)
|
|
157
69
|
// }
|
|
158
|
-
// const tweet = await x.posts.getById("2061603004248154195", { tweetFields : [
|
|
159
|
-
// "article",
|
|
160
|
-
// "attachments",
|
|
161
|
-
// "author_id",
|
|
162
|
-
// "card_uri",
|
|
163
|
-
// "community_id",
|
|
164
|
-
// "context_annotations",
|
|
165
|
-
// "conversation_id",
|
|
166
|
-
// "created_at",
|
|
167
|
-
// "display_text_range",
|
|
168
|
-
// "edit_controls",
|
|
169
|
-
// "edit_history_tweet_ids",
|
|
170
|
-
// "entities",
|
|
171
|
-
// "geo",
|
|
172
|
-
// "id",
|
|
173
|
-
// "in_reply_to_user_id",
|
|
174
|
-
// "lang",
|
|
175
|
-
// "matched_media_notes",
|
|
176
|
-
// "media_metadata",
|
|
177
|
-
// // "non_public_metrics",
|
|
178
|
-
// "note_request_suggestions",
|
|
179
|
-
// "note_tweet",
|
|
180
|
-
// // "organic_metrics",
|
|
181
|
-
// "paid_partnership",
|
|
182
|
-
// "possibly_sensitive",
|
|
183
|
-
// // "promoted_metrics",
|
|
184
|
-
// "public_metrics",
|
|
185
|
-
// "referenced_tweets",
|
|
186
|
-
// "reply_settings",
|
|
187
|
-
// "scopes",
|
|
188
|
-
// "source",
|
|
189
|
-
// "suggested_source_links",
|
|
190
|
-
// "suggested_source_links_with_counts",
|
|
191
|
-
// "text",
|
|
192
|
-
// "withheld"
|
|
193
|
-
// ], expansions : [
|
|
194
|
-
// "article.cover_media",
|
|
195
|
-
// "article.media_entities",
|
|
196
|
-
// "attachments.media_keys",
|
|
197
|
-
// "attachments.media_source_tweet",
|
|
198
|
-
// "attachments.poll_ids",
|
|
199
|
-
// "author_id",
|
|
200
|
-
// "edit_history_tweet_ids",
|
|
201
|
-
// "entities.mentions.username",
|
|
202
|
-
// "geo.place_id",
|
|
203
|
-
// "in_reply_to_user_id",
|
|
204
|
-
// "entities.note.mentions.username",
|
|
205
|
-
// "referenced_tweets.id",
|
|
206
|
-
// "referenced_tweets.id.attachments.media_keys",
|
|
207
|
-
// "referenced_tweets.id.author_id"
|
|
208
|
-
// ], pollFields : [
|
|
209
|
-
// "duration_minutes",
|
|
210
|
-
// "end_datetime",
|
|
211
|
-
// "id",
|
|
212
|
-
// "options",
|
|
213
|
-
// "voting_status"
|
|
214
|
-
// ]
|
|
215
|
-
// })
|
|
216
|
-
// console.log(JSON.stringify(tweet, undefined, 4))
|
|
217
|
-
// const results = await utils.twitter.get_tweets("1603470469608374272", {
|
|
218
|
-
// compile : true,
|
|
219
|
-
// min_twitter_tweet_id : "2061610563679985813"
|
|
220
|
-
// })
|
|
221
|
-
// console.log(results)
|
|
222
|
-
// const f = await utils.dynamodb.query_range("federal_election_donations:donor_address_state_id.federal_election_committee_id-receipt_date-federal_election_donation_id", { donor_address_state_id : "NH", federal_election_committee_id : "C00694323" }, { receipt_date : ["2021", "2023"] }, { compile : false })
|
|
223
|
-
// console.log(f)
|
|
224
|
-
// console.log("YERP")
|
|
225
|
-
// const response = await utils.bedrock.claude_converse(
|
|
226
|
-
// "anthropic.claude-opus-4-8",
|
|
227
|
-
// [
|
|
228
|
-
// { role : "user", text : "Hi" }
|
|
229
|
-
// ],
|
|
230
|
-
// {
|
|
231
|
-
// print_usage : true
|
|
232
|
-
// }
|
|
233
|
-
// )
|
|
234
|
-
// console.log(response)
|
|
235
|
-
// const coordinates = await utils.bedrock.cohere_invoke("global.cohere.embed-v4:0", "yerrp")
|
|
236
|
-
// console.log(coordinates)
|
|
237
|
-
const scout_ids = await utils.dynamodb.scan("scouts")
|
|
238
|
-
.then(scouts => scouts.map(scout => scout.scout_id));
|
|
239
|
-
const scouts = await utils.dynamodb.batch_get("scouts", scout_ids.slice(0, 150).map(scout_id => ({ scout_id: scout_id })));
|
|
240
|
-
console.log(scouts.length);
|
package/dist/src/index.d.ts
CHANGED
|
@@ -11,8 +11,11 @@ import { UtilsXAI } from "./UtilsXAI.js";
|
|
|
11
11
|
import { UtilsAnthropic } from "./UtilsAnthropic.js";
|
|
12
12
|
import { UtilsTwitter } from "./UtilsTwitter.js";
|
|
13
13
|
import { GlobalConfig } from "./types/GlobalConfig.js";
|
|
14
|
+
import { UtilsRDS } from "./UtilsRDS.js";
|
|
15
|
+
import { ApplicationConfig } from "./types/ApplicationConfig.js";
|
|
14
16
|
export declare class TriangleUtils extends UtilsMisc {
|
|
15
17
|
readonly dynamodb: UtilsDynamoDB;
|
|
18
|
+
readonly rds: UtilsRDS;
|
|
16
19
|
readonly s3: UtilsS3;
|
|
17
20
|
readonly s3vectors: UtilsS3Vectors;
|
|
18
21
|
readonly bedrock: UtilsBedrock;
|
|
@@ -23,13 +26,14 @@ export declare class TriangleUtils extends UtilsMisc {
|
|
|
23
26
|
readonly anthropic: UtilsAnthropic;
|
|
24
27
|
readonly xai: UtilsXAI;
|
|
25
28
|
readonly twitter: UtilsTwitter;
|
|
26
|
-
constructor(config: GlobalConfig);
|
|
29
|
+
constructor(config: ApplicationConfig & GlobalConfig);
|
|
27
30
|
}
|
|
28
31
|
export * from "./types/ApplicationConfig.js";
|
|
29
32
|
export * from "./types/GlobalConfig.js";
|
|
30
33
|
export * from "./UtilsMisc.js";
|
|
31
34
|
export * from "./UtilsAnthropic.js";
|
|
32
35
|
export * from "./UtilsDynamoDB.js";
|
|
36
|
+
export * from "./UtilsRDS.js";
|
|
33
37
|
export * from "./UtilsS3.js";
|
|
34
38
|
export * from "./UtilsBedrock.js";
|
|
35
39
|
export * from "./UtilsBee.js";
|
package/dist/src/index.js
CHANGED
|
@@ -10,8 +10,10 @@ import { UtilsYoutube } from "./UtilsYoutube.js";
|
|
|
10
10
|
import { UtilsXAI } from "./UtilsXAI.js";
|
|
11
11
|
import { UtilsAnthropic } from "./UtilsAnthropic.js";
|
|
12
12
|
import { UtilsTwitter } from "./UtilsTwitter.js";
|
|
13
|
+
import { UtilsRDS } from "./UtilsRDS.js";
|
|
13
14
|
export class TriangleUtils extends UtilsMisc {
|
|
14
15
|
dynamodb;
|
|
16
|
+
rds;
|
|
15
17
|
s3;
|
|
16
18
|
s3vectors;
|
|
17
19
|
bedrock;
|
|
@@ -25,6 +27,7 @@ export class TriangleUtils extends UtilsMisc {
|
|
|
25
27
|
constructor(config) {
|
|
26
28
|
super(config);
|
|
27
29
|
this.dynamodb = new UtilsDynamoDB(config.region);
|
|
30
|
+
this.rds = new UtilsRDS(config.region, config.rds_cluster_arn, config.rds_secret_arn, config.rds_host, config.rds_username, config.rds_password);
|
|
28
31
|
this.s3 = new UtilsS3(config.region);
|
|
29
32
|
this.s3vectors = new UtilsS3Vectors(config.region);
|
|
30
33
|
this.bedrock = new UtilsBedrock(config.region);
|
|
@@ -42,6 +45,7 @@ export * from "./types/GlobalConfig.js";
|
|
|
42
45
|
export * from "./UtilsMisc.js";
|
|
43
46
|
export * from "./UtilsAnthropic.js";
|
|
44
47
|
export * from "./UtilsDynamoDB.js";
|
|
48
|
+
export * from "./UtilsRDS.js";
|
|
45
49
|
export * from "./UtilsS3.js";
|
|
46
50
|
export * from "./UtilsBedrock.js";
|
|
47
51
|
export * from "./UtilsBee.js";
|
|
@@ -8,6 +8,11 @@ export declare class GlobalConfig {
|
|
|
8
8
|
readonly s3_prism: string;
|
|
9
9
|
readonly s3_scout: string;
|
|
10
10
|
readonly s3_triage: string;
|
|
11
|
+
readonly rds_cluster_arn: string;
|
|
12
|
+
readonly rds_secret_arn: string;
|
|
13
|
+
readonly rds_host: string;
|
|
14
|
+
readonly rds_username: string;
|
|
15
|
+
readonly rds_password: string;
|
|
11
16
|
readonly super_federal_gov_api_keys: string;
|
|
12
17
|
readonly federal_gov_api_keys: string;
|
|
13
18
|
readonly federal_census_api_key: string;
|
|
@@ -8,6 +8,11 @@ export class GlobalConfig {
|
|
|
8
8
|
s3_prism;
|
|
9
9
|
s3_scout;
|
|
10
10
|
s3_triage;
|
|
11
|
+
rds_cluster_arn;
|
|
12
|
+
rds_secret_arn;
|
|
13
|
+
rds_host;
|
|
14
|
+
rds_username;
|
|
15
|
+
rds_password;
|
|
11
16
|
super_federal_gov_api_keys;
|
|
12
17
|
federal_gov_api_keys;
|
|
13
18
|
federal_census_api_key;
|
|
@@ -29,6 +34,11 @@ export class GlobalConfig {
|
|
|
29
34
|
this.s3_prism = global_config.s3_prism;
|
|
30
35
|
this.s3_scout = global_config.s3_scout;
|
|
31
36
|
this.s3_triage = global_config.s3_triage;
|
|
37
|
+
this.rds_cluster_arn = global_config.rds_cluster_arn;
|
|
38
|
+
this.rds_secret_arn = global_config.rds_secret_arn;
|
|
39
|
+
this.rds_host = global_config.rds_host;
|
|
40
|
+
this.rds_username = global_config.rds_username;
|
|
41
|
+
this.rds_password = global_config.rds_password;
|
|
32
42
|
this.super_federal_gov_api_keys = global_config.super_federal_gov_api_keys;
|
|
33
43
|
this.federal_gov_api_keys = global_config.federal_gov_api_keys;
|
|
34
44
|
this.federal_census_api_key = global_config.federal_census_api_key;
|
|
@@ -49,6 +59,11 @@ export class GlobalConfig {
|
|
|
49
59
|
typeof global_config.s3_prism === "string" &&
|
|
50
60
|
typeof global_config.s3_scout === "string" &&
|
|
51
61
|
typeof global_config.s3_triage === "string" &&
|
|
62
|
+
typeof global_config.rds_cluster_arn === "string" &&
|
|
63
|
+
typeof global_config.rds_secret_arn === "string" &&
|
|
64
|
+
typeof global_config.rds_host === "string" &&
|
|
65
|
+
typeof global_config.rds_username === "string" &&
|
|
66
|
+
typeof global_config.rds_password === "string" &&
|
|
52
67
|
typeof global_config.super_federal_gov_api_keys === "string" &&
|
|
53
68
|
typeof global_config.federal_gov_api_keys === "string" &&
|
|
54
69
|
typeof global_config.federal_census_api_key === "string" &&
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "triangle-utils",
|
|
3
|
-
"version": "1.4.
|
|
3
|
+
"version": "1.4.123",
|
|
4
4
|
"main": "dist/src/index.js",
|
|
5
5
|
"types": "dist/src/index.d.ts",
|
|
6
6
|
"directories": {
|
|
@@ -20,6 +20,8 @@
|
|
|
20
20
|
"@aws-sdk/client-bedrock-runtime": "^3.953.0",
|
|
21
21
|
"@aws-sdk/client-cognito-identity-provider": "^3.1004.0",
|
|
22
22
|
"@aws-sdk/client-dynamodb": "^3.953.0",
|
|
23
|
+
"@aws-sdk/client-rds": "^3.1111.0",
|
|
24
|
+
"@aws-sdk/client-rds-data": "^3.1111.0",
|
|
23
25
|
"@aws-sdk/client-s3": "^3.1106.0",
|
|
24
26
|
"@aws-sdk/client-s3vectors": "^3.953.0",
|
|
25
27
|
"@aws-sdk/client-secrets-manager": "^3.965.0",
|
|
@@ -29,9 +31,11 @@
|
|
|
29
31
|
"@types/jsdom": "^28.0.1",
|
|
30
32
|
"@types/node": "^25.2.3",
|
|
31
33
|
"@types/nodemailer": "^7.0.9",
|
|
34
|
+
"@types/pg": "^8.21.0",
|
|
32
35
|
"@xdevplatform/xdk": "^0.5.0",
|
|
33
36
|
"googleapis": "^170.0.0",
|
|
34
37
|
"jsdom": "^29.0.1",
|
|
38
|
+
"pg": "^8.23.0",
|
|
35
39
|
"scrapingbee": "^1.8.2"
|
|
36
40
|
},
|
|
37
41
|
"devDependencies": {
|
package/src/UtilsRDS.ts
ADDED
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
import { Field, RDSData } from "@aws-sdk/client-rds-data"
|
|
2
|
+
import { Client } from "pg"
|
|
3
|
+
|
|
4
|
+
function convert_rds_input(input : string | boolean | number | Record<string, any> | undefined) : string {
|
|
5
|
+
if (input === undefined) {
|
|
6
|
+
return "NULL"
|
|
7
|
+
} else if (typeof input === "string") {
|
|
8
|
+
return "'" + input + "'"
|
|
9
|
+
} else if (typeof input === "number") {
|
|
10
|
+
return input.toString()
|
|
11
|
+
} else if (typeof input === "boolean") {
|
|
12
|
+
return input.toString().toUpperCase()
|
|
13
|
+
} else if (typeof input === "object") {
|
|
14
|
+
if (input.expression !== undefined) {
|
|
15
|
+
return input.expression
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
return "NULL"
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export class UtilsRDS {
|
|
22
|
+
|
|
23
|
+
private readonly postgres : Client
|
|
24
|
+
private readonly rds_data : RDSData
|
|
25
|
+
private readonly rds_cluster_arn : string
|
|
26
|
+
private readonly rds_secret_arn : string
|
|
27
|
+
|
|
28
|
+
constructor(
|
|
29
|
+
region : string,
|
|
30
|
+
rds_cluster_arn : string,
|
|
31
|
+
rds_secret_arn : string,
|
|
32
|
+
rds_host : string,
|
|
33
|
+
rds_username : string,
|
|
34
|
+
rds_password : string
|
|
35
|
+
) {
|
|
36
|
+
this.postgres = new Client({
|
|
37
|
+
host : rds_host,
|
|
38
|
+
user : rds_username,
|
|
39
|
+
password : rds_password,
|
|
40
|
+
database : "postgres",
|
|
41
|
+
port : 5432,
|
|
42
|
+
ssl : {
|
|
43
|
+
rejectUnauthorized: false
|
|
44
|
+
}
|
|
45
|
+
})
|
|
46
|
+
this.rds_data = new RDSData({ region : region })
|
|
47
|
+
this.rds_cluster_arn = rds_cluster_arn,
|
|
48
|
+
this.rds_secret_arn = rds_secret_arn
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
async connect() {
|
|
52
|
+
return await this.postgres.connect()
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
async disconnect() {
|
|
56
|
+
return await this.postgres.end()
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
async query(
|
|
60
|
+
sql : string,
|
|
61
|
+
options : {
|
|
62
|
+
is_verbose? : boolean
|
|
63
|
+
} = {}
|
|
64
|
+
) {
|
|
65
|
+
const is_verbose = Boolean(options.is_verbose)
|
|
66
|
+
if (is_verbose) {
|
|
67
|
+
console.log(sql)
|
|
68
|
+
}
|
|
69
|
+
const response_connect = await this.connect()
|
|
70
|
+
if (is_verbose) {
|
|
71
|
+
console.log(response_connect)
|
|
72
|
+
}
|
|
73
|
+
const response = await this.postgres.query(sql)
|
|
74
|
+
const response_disconnect = await this.disconnect()
|
|
75
|
+
if (is_verbose) {
|
|
76
|
+
console.log(response_disconnect)
|
|
77
|
+
}
|
|
78
|
+
return response.rows
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
async exec(
|
|
82
|
+
sql : string,
|
|
83
|
+
options : {
|
|
84
|
+
is_verbose? : boolean
|
|
85
|
+
} = {}
|
|
86
|
+
) {
|
|
87
|
+
const is_verbose = Boolean(options.is_verbose)
|
|
88
|
+
if (is_verbose) {
|
|
89
|
+
console.log(sql)
|
|
90
|
+
}
|
|
91
|
+
const items = sql.length <= 60000 ?
|
|
92
|
+
await this.rds_data.executeStatement({
|
|
93
|
+
resourceArn : this.rds_cluster_arn,
|
|
94
|
+
secretArn : this.rds_secret_arn,
|
|
95
|
+
database : "postgres",
|
|
96
|
+
sql : sql,
|
|
97
|
+
formatRecordsAs : "JSON"
|
|
98
|
+
})
|
|
99
|
+
.then(response => JSON.parse(response.formattedRecords || "[]"))
|
|
100
|
+
:
|
|
101
|
+
await this.query(sql, options)
|
|
102
|
+
if (!Array.isArray(items)) {
|
|
103
|
+
if (is_verbose) {
|
|
104
|
+
console.log("Bad items:", items)
|
|
105
|
+
}
|
|
106
|
+
return []
|
|
107
|
+
}
|
|
108
|
+
return items
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
async get(
|
|
112
|
+
table_id : string,
|
|
113
|
+
primary_key : Record<string, any>,
|
|
114
|
+
attribute_names? : string[],
|
|
115
|
+
options : {
|
|
116
|
+
is_verbose? : boolean
|
|
117
|
+
} = {}
|
|
118
|
+
) {
|
|
119
|
+
const sql = "SELECT " + (attribute_names === undefined ? "*" : attribute_names.join(", ")) + " FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, value]) => key_name + " = " + convert_rds_input(value)).join(" AND ")
|
|
120
|
+
const items = await this.exec(sql, options)
|
|
121
|
+
const item : Record<string, any> | undefined = items[0]
|
|
122
|
+
return item
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
async batch_get(
|
|
126
|
+
table_id : string,
|
|
127
|
+
primary_key : Record<string, any[]>,
|
|
128
|
+
attribute_names? : string[],
|
|
129
|
+
options : {
|
|
130
|
+
is_verbose? : boolean
|
|
131
|
+
} = {}
|
|
132
|
+
) {
|
|
133
|
+
const sql = "SELECT " + (attribute_names === undefined ? "*" : attribute_names.join(", ")) + " FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, values]) => key_name + " IN " + "(" + values.map(value => convert_rds_input(value)).join(", ") + ")").join(" AND ")
|
|
134
|
+
const items = await this.exec(sql, options)
|
|
135
|
+
return items
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
async create(
|
|
139
|
+
table_id : string,
|
|
140
|
+
item : Record<string, any>,
|
|
141
|
+
options : {
|
|
142
|
+
is_verbose? : boolean
|
|
143
|
+
} = {}
|
|
144
|
+
) {
|
|
145
|
+
const attribute_names = Object.keys(item).sort()
|
|
146
|
+
const sql = "INSERT INTO " + table_id + " (" + attribute_names.join(", ") + ")" + " VALUES " + "(" + attribute_names.map(attribute_name => convert_rds_input(item[attribute_name])).join(", ") + ")" + " ON CONFLICT ON CONSTRAINT " + table_id + "_pkey IGNORE;"
|
|
147
|
+
try {
|
|
148
|
+
const items = await this.exec(sql, options)
|
|
149
|
+
return items
|
|
150
|
+
} catch (error : any) {
|
|
151
|
+
console.log(error.stack)
|
|
152
|
+
return
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
async batch_create(
|
|
157
|
+
table_id : string,
|
|
158
|
+
items : Record<string, any>[],
|
|
159
|
+
options : {
|
|
160
|
+
is_verbose? : boolean
|
|
161
|
+
} = {}
|
|
162
|
+
) {
|
|
163
|
+
const attribute_names = Array.from(new Set(items.map(item => Object.keys(item)).flat())).sort()
|
|
164
|
+
const sql = "INSERT INTO " + table_id + " (" + attribute_names.join(", ") + ")" + " VALUES " + items.map(item => "(" + attribute_names.map(attribute_name => convert_rds_input(item[attribute_name])).join(", ") + ")").join(", ") + " ON CONFLICT ON CONSTRAINT " + table_id + "_pkey IGNORE;"
|
|
165
|
+
try {
|
|
166
|
+
const items = await this.exec(sql, options)
|
|
167
|
+
return items
|
|
168
|
+
} catch (error : any) {
|
|
169
|
+
console.log(error.stack)
|
|
170
|
+
return
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
async put(
|
|
175
|
+
table_id : string,
|
|
176
|
+
item : Record<string, any>,
|
|
177
|
+
options : {
|
|
178
|
+
is_verbose? : boolean
|
|
179
|
+
} = {}
|
|
180
|
+
) {
|
|
181
|
+
const attribute_names = Object.keys(item).sort()
|
|
182
|
+
const sql = "INSERT INTO " + table_id + " (" + attribute_names.join(", ") + ")" + " VALUES " + "(" + attribute_names.map(attribute_name => convert_rds_input(item[attribute_name])).join(", ") + ")" + " ON CONFLICT ON CONSTRAINT " + table_id + "_pkey DO UPDATE SET " + attribute_names.map(attribute_name => attribute_name + " = " + "EXCLUDED." + attribute_name).join(", ") + ";"
|
|
183
|
+
try {
|
|
184
|
+
const items = await this.exec(sql, options)
|
|
185
|
+
return items
|
|
186
|
+
} catch (error : any) {
|
|
187
|
+
console.log(error.stack)
|
|
188
|
+
return
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
async batch_put(
|
|
193
|
+
table_id : string,
|
|
194
|
+
items : Record<string, any>[],
|
|
195
|
+
options : {
|
|
196
|
+
is_verbose? : boolean
|
|
197
|
+
} = {}
|
|
198
|
+
) {
|
|
199
|
+
const attribute_names = Array.from(new Set(items.map(item => Object.keys(item)).flat())).sort()
|
|
200
|
+
const sql = "INSERT INTO " + table_id + " (" + attribute_names.join(", ") + ")" + " VALUES " + items.map(item => "(" + attribute_names.map(attribute_name => convert_rds_input(item[attribute_name])).join(", ") + ")").join(", ") + " ON CONFLICT ON CONSTRAINT " + table_id + "_pkey DO UPDATE SET " + attribute_names.map(attribute_name => attribute_name + " = " + "EXCLUDED." + attribute_name).join(", ") + ";"
|
|
201
|
+
try {
|
|
202
|
+
const items = await this.exec(sql, options)
|
|
203
|
+
return items
|
|
204
|
+
} catch (error : any) {
|
|
205
|
+
console.log(error.stack)
|
|
206
|
+
return
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
async delete(
|
|
211
|
+
table_id : string,
|
|
212
|
+
primary_key : Record<string, any>,
|
|
213
|
+
options : {
|
|
214
|
+
is_verbose? : boolean
|
|
215
|
+
} = {}
|
|
216
|
+
) {
|
|
217
|
+
const sql = "DELETE FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, value]) => key_name + " = " + convert_rds_input(value)).join(" AND ")
|
|
218
|
+
return await this.exec(sql, options)
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
async batch_delete(
|
|
222
|
+
table_id : string,
|
|
223
|
+
primary_key : Record<string, any[]>,
|
|
224
|
+
options : {
|
|
225
|
+
is_verbose? : boolean
|
|
226
|
+
} = {}
|
|
227
|
+
) {
|
|
228
|
+
const sql = "DELETE FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, values]) => key_name + " IN " + "(" + values.map(value => convert_rds_input(value)).join(", ") + ")").join(" AND ")
|
|
229
|
+
return await this.exec(sql, options)
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
}
|
package/src/f.ts
CHANGED
|
@@ -24,258 +24,62 @@ console.log(config)
|
|
|
24
24
|
|
|
25
25
|
const utils = new TriangleUtils(config)
|
|
26
26
|
|
|
27
|
-
// const
|
|
28
|
-
//
|
|
27
|
+
// const federal_censuses = await utils.dynamodb.scan("federal_censuses")
|
|
28
|
+
// federal_censuses.sort((a, b) => a.federal_census_id - b.federal_census_id)
|
|
29
29
|
|
|
30
|
-
// console.log(
|
|
30
|
+
// console.log(JSON.stringify(response, undefined, 4))
|
|
31
31
|
|
|
32
|
-
//
|
|
33
|
-
//
|
|
34
|
-
//
|
|
35
|
-
// type : "object",
|
|
36
|
-
// properties : {
|
|
37
|
-
// username : {
|
|
38
|
-
// type: "string"
|
|
39
|
-
// },
|
|
40
|
-
// explanation : {
|
|
41
|
-
// type: "string"
|
|
42
|
-
// }
|
|
43
|
-
// },
|
|
44
|
-
// required : [
|
|
45
|
-
// "username",
|
|
46
|
-
// "explanation"
|
|
47
|
-
// ],
|
|
48
|
-
// additionalProperties : false
|
|
49
|
-
// }
|
|
50
|
-
// })
|
|
51
|
-
// console.log(text)
|
|
52
|
-
|
|
53
|
-
// const foods = await utils.dynamodb.query("triage_docket_documents:register_document_id", { register_document_id : "E6-17065" })
|
|
54
|
-
|
|
55
|
-
// console.log(foods)
|
|
56
|
-
|
|
57
|
-
// const text = await utils.bee.get("https://www.doyourjobs.org", { return_page_text : true })
|
|
32
|
+
// await utils.rds.batch_delete("federal_censuses", {
|
|
33
|
+
// federal_census_id : federal_censuses.map(federal_census => federal_census.federal_census_id)
|
|
34
|
+
// }, { is_verbose : true })
|
|
58
35
|
|
|
59
|
-
//
|
|
36
|
+
// await utils.rds.batch_put("federal_censuses", federal_censuses)
|
|
60
37
|
|
|
61
|
-
// const
|
|
62
|
-
// console.log(text)
|
|
38
|
+
// const rds_federal_censuses = await utils.rds.batch_get("federal_censuses", { federal_census_id : federal_censuses.map(federal_census => federal_census.federal_census_id) }, ["federal_census_id", "is_decennial"], { is_verbose : true })
|
|
63
39
|
|
|
64
|
-
//
|
|
40
|
+
// console.log(rds_federal_censuses)
|
|
65
41
|
|
|
66
|
-
// const
|
|
67
|
-
|
|
68
|
-
//
|
|
69
|
-
|
|
70
|
-
//
|
|
71
|
-
//
|
|
72
|
-
//
|
|
73
|
-
//
|
|
74
|
-
//
|
|
75
|
-
//
|
|
76
|
-
//
|
|
77
|
-
// max_tokens : 20000,
|
|
78
|
-
// json_format : {
|
|
79
|
-
// type : "object",
|
|
80
|
-
// properties : {
|
|
81
|
-
// is_veteran : {
|
|
82
|
-
// type : "string"
|
|
83
|
-
// },
|
|
84
|
-
// explanation : {
|
|
85
|
-
// type : "string"
|
|
86
|
-
// }
|
|
87
|
-
// },
|
|
88
|
-
// required : ["is_veteran", "explanation"],
|
|
89
|
-
// additionalProperties : false
|
|
90
|
-
// },
|
|
91
|
-
// web : true
|
|
42
|
+
// const voters = await utils.dynamodb.scan("voters", { compile : false, max_num_items : 100 })
|
|
43
|
+
// for (const voter of voters) {
|
|
44
|
+
// const new_lal_file_voter = {
|
|
45
|
+
// lal_file_voter_id : voter.state_id + "-2026-08-15" + "|" + voter.voter_id,
|
|
46
|
+
// lal_file_id : voter.state_id + "-2026-08-15",
|
|
47
|
+
// lal_voter_id : voter.voter_id,
|
|
48
|
+
// voter_id : voter.state_voter_id,
|
|
49
|
+
// state_id : voter.state_id,
|
|
50
|
+
// address_residence_latitude : voter.address_residence_latitude,
|
|
51
|
+
// address_residence_longitude : voter.address_residence_longitude,
|
|
52
|
+
// address_residence_coordinates : "ST_GeomFromText('POINT(" + voter.address_residence_latitude + " " + voter.address_residence_longitude + ")', 4326)"
|
|
92
53
|
// }
|
|
93
|
-
//
|
|
94
|
-
|
|
95
|
-
// console.log(output)
|
|
96
|
-
|
|
97
|
-
// import { Client } from "@xdevplatform/xdk"
|
|
98
|
-
|
|
99
|
-
// const x = new Client({ bearerToken : "AAAAAAAAAAAAAAAAAAAAAKoQ%2BQEAAAAAaej7PP534%2Fd9dKmieHRY6vQkv2s%3DyY2VerXmrA7w794ACFr9ALvoaJfuAHSlWwZdiDeVaOVpA6HCub" })
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
// const user = await x.users.getByUsername("umichvoter", { userFields : [
|
|
103
|
-
// "affiliation",
|
|
104
|
-
// // "confirmed_email",
|
|
105
|
-
// "connection_status",
|
|
106
|
-
// "created_at",
|
|
107
|
-
// "description",
|
|
108
|
-
// "entities",
|
|
109
|
-
// "id",
|
|
110
|
-
// "is_identity_verified",
|
|
111
|
-
// "location",
|
|
112
|
-
// "most_recent_tweet_id",
|
|
113
|
-
// "name",
|
|
114
|
-
// "parody",
|
|
115
|
-
// "pinned_tweet_id",
|
|
116
|
-
// "profile_banner_url",
|
|
117
|
-
// "profile_image_url",
|
|
118
|
-
// "protected",
|
|
119
|
-
// "public_metrics",
|
|
120
|
-
// // "receives_your_dm",
|
|
121
|
-
// "subscription",
|
|
122
|
-
// "subscription_type",
|
|
123
|
-
// "url",
|
|
124
|
-
// "username",
|
|
125
|
-
// "verified",
|
|
126
|
-
// "verified_followers_count",
|
|
127
|
-
// "verified_type",
|
|
128
|
-
// "withheld"
|
|
129
|
-
// ] })
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
// console.log(JSON.stringify(user, null, 4))
|
|
133
|
-
|
|
134
|
-
// const tweets = await x.users.getPosts("1603470469608374272", {
|
|
135
|
-
|
|
136
|
-
// tweetFields : [
|
|
137
|
-
// "article",
|
|
138
|
-
// "attachments",
|
|
139
|
-
// "author_id",
|
|
140
|
-
// "card_uri",
|
|
141
|
-
// "community_id",
|
|
142
|
-
// "context_annotations",
|
|
143
|
-
// "conversation_id",
|
|
144
|
-
// "created_at",
|
|
145
|
-
// "display_text_range",
|
|
146
|
-
// "edit_controls",
|
|
147
|
-
// "edit_history_tweet_ids",
|
|
148
|
-
// "entities",
|
|
149
|
-
// "geo",
|
|
150
|
-
// "id",
|
|
151
|
-
// "in_reply_to_user_id",
|
|
152
|
-
// "lang",
|
|
153
|
-
// "matched_media_notes",
|
|
154
|
-
// "media_metadata",
|
|
155
|
-
// "note_request_suggestions",
|
|
156
|
-
// "note_tweet",
|
|
157
|
-
// "paid_partnership",
|
|
158
|
-
// "possibly_sensitive",
|
|
159
|
-
// "referenced_tweets",
|
|
160
|
-
// "reply_settings",
|
|
161
|
-
// "scopes",
|
|
162
|
-
// "source",
|
|
163
|
-
// "suggested_source_links",
|
|
164
|
-
// "suggested_source_links_with_counts",
|
|
165
|
-
// "text",
|
|
166
|
-
// "withheld"
|
|
167
|
-
// ], expansions : [
|
|
168
|
-
// "article.cover_media",
|
|
169
|
-
// "article.media_entities",
|
|
170
|
-
// "attachments.media_keys",
|
|
171
|
-
// "attachments.media_source_tweet",
|
|
172
|
-
// "attachments.poll_ids",
|
|
173
|
-
// "author_id",
|
|
174
|
-
// "edit_history_tweet_ids",
|
|
175
|
-
// "entities.mentions.username",
|
|
176
|
-
// "geo.place_id",
|
|
177
|
-
// "in_reply_to_user_id",
|
|
178
|
-
// "entities.note.mentions.username",
|
|
179
|
-
// "referenced_tweets.id",
|
|
180
|
-
// "referenced_tweets.id.attachments.media_keys",
|
|
181
|
-
// "referenced_tweets.id.author_id"
|
|
182
|
-
// ]})
|
|
183
|
-
|
|
184
|
-
// console.log(JSON.stringify(tweets, null, 4))
|
|
185
|
-
|
|
186
|
-
// for (const tweet of tweets.data || []) {
|
|
187
|
-
// console.log(JSON.stringify(tweet, undefined, 4))
|
|
54
|
+
// console.log(new_lal_file_voter)
|
|
55
|
+
// await utils.rds.put("lal_file_voters", new_lal_file_voter)
|
|
188
56
|
// }
|
|
189
|
-
|
|
190
|
-
// const
|
|
191
|
-
//
|
|
192
|
-
//
|
|
193
|
-
//
|
|
194
|
-
//
|
|
195
|
-
//
|
|
196
|
-
//
|
|
197
|
-
//
|
|
198
|
-
//
|
|
199
|
-
// "
|
|
200
|
-
// "
|
|
201
|
-
|
|
202
|
-
// "
|
|
203
|
-
// "
|
|
204
|
-
// "
|
|
205
|
-
//
|
|
206
|
-
//
|
|
207
|
-
//
|
|
208
|
-
//
|
|
209
|
-
//
|
|
210
|
-
//
|
|
211
|
-
//
|
|
212
|
-
//
|
|
213
|
-
//
|
|
214
|
-
//
|
|
215
|
-
//
|
|
216
|
-
//
|
|
217
|
-
//
|
|
218
|
-
// "reply_settings",
|
|
219
|
-
// "scopes",
|
|
220
|
-
// "source",
|
|
221
|
-
// "suggested_source_links",
|
|
222
|
-
// "suggested_source_links_with_counts",
|
|
223
|
-
// "text",
|
|
224
|
-
// "withheld"
|
|
225
|
-
// ], expansions : [
|
|
226
|
-
// "article.cover_media",
|
|
227
|
-
// "article.media_entities",
|
|
228
|
-
// "attachments.media_keys",
|
|
229
|
-
// "attachments.media_source_tweet",
|
|
230
|
-
// "attachments.poll_ids",
|
|
231
|
-
// "author_id",
|
|
232
|
-
// "edit_history_tweet_ids",
|
|
233
|
-
// "entities.mentions.username",
|
|
234
|
-
// "geo.place_id",
|
|
235
|
-
// "in_reply_to_user_id",
|
|
236
|
-
// "entities.note.mentions.username",
|
|
237
|
-
// "referenced_tweets.id",
|
|
238
|
-
// "referenced_tweets.id.attachments.media_keys",
|
|
239
|
-
// "referenced_tweets.id.author_id"
|
|
240
|
-
// ], pollFields : [
|
|
241
|
-
// "duration_minutes",
|
|
242
|
-
// "end_datetime",
|
|
243
|
-
// "id",
|
|
244
|
-
// "options",
|
|
245
|
-
// "voting_status"
|
|
246
|
-
// ]
|
|
247
|
-
// })
|
|
248
|
-
|
|
249
|
-
// console.log(JSON.stringify(tweet, undefined, 4))
|
|
250
|
-
|
|
251
|
-
// const results = await utils.twitter.get_tweets("1603470469608374272", {
|
|
252
|
-
// compile : true,
|
|
253
|
-
// min_twitter_tweet_id : "2061610563679985813"
|
|
254
|
-
// })
|
|
255
|
-
|
|
256
|
-
// console.log(results)
|
|
257
|
-
|
|
258
|
-
// const f = await utils.dynamodb.query_range("federal_election_donations:donor_address_state_id.federal_election_committee_id-receipt_date-federal_election_donation_id", { donor_address_state_id : "NH", federal_election_committee_id : "C00694323" }, { receipt_date : ["2021", "2023"] }, { compile : false })
|
|
259
|
-
|
|
260
|
-
// console.log(f)
|
|
261
|
-
// console.log("YERP")
|
|
262
|
-
// const response = await utils.bedrock.claude_converse(
|
|
263
|
-
// "anthropic.claude-opus-4-8",
|
|
264
|
-
// [
|
|
265
|
-
// { role : "user", text : "Hi" }
|
|
266
|
-
// ],
|
|
267
|
-
// {
|
|
268
|
-
// print_usage : true
|
|
269
|
-
// }
|
|
270
|
-
// )
|
|
271
|
-
// console.log(response)
|
|
272
|
-
|
|
273
|
-
// const coordinates = await utils.bedrock.cohere_invoke("global.cohere.embed-v4:0", "yerrp")
|
|
274
|
-
|
|
275
|
-
// console.log(coordinates)
|
|
276
|
-
|
|
277
|
-
const scout_ids = await utils.dynamodb.scan("scouts")
|
|
278
|
-
.then(scouts => scouts.map(scout => scout.scout_id))
|
|
279
|
-
|
|
280
|
-
const scouts = await utils.dynamodb.batch_get("scouts", scout_ids.slice(0, 150).map(scout_id => ({ scout_id : scout_id })))
|
|
281
|
-
console.log(scouts.length)
|
|
57
|
+
// const federal_census_county_id = "2026-04021"
|
|
58
|
+
// const federal_census_county = await utils.rds.get("federal_census_counties", { federal_census_county_id : federal_census_county_id }, ["*"])
|
|
59
|
+
// console.log(federal_census_county)
|
|
60
|
+
// console.log(await utils.rds.exec("SELECT * FROM federal_censuses"))
|
|
61
|
+
// const federal_census_county = await utils.dynamodb.get("federal_census_counties", { federal_census_county_id : federal_census_county_id })
|
|
62
|
+
// const s3_id = config.s3_triangle + "/federal_census/2026/counties/" + federal_census_county_id + ".json"
|
|
63
|
+
// console.log(s3_id)
|
|
64
|
+
// const feature = await utils.s3.get(s3_id)
|
|
65
|
+
// // console.log(feature)
|
|
66
|
+
// if (feature !== undefined && federal_census_county !== undefined) {
|
|
67
|
+
// const geometry = "(" + JSON.parse(feature).geometry.coordinates[0].map((polygon : any) => "(" + polygon.map((vertex : any) => vertex[1] + " " + vertex[0]).join(", ") + ")").join(", ") + ")"
|
|
68
|
+
// const ewkt = "'MULTIPOLYGON(" + geometry + ")'"
|
|
69
|
+
|
|
70
|
+
// // const response = await utils.rds.exec("SELECT ST_AsText(address_residence_coordinates) FROM lal_file_voters", { is_verbose : true })
|
|
71
|
+
// // const response = await utils.rds.exec("SELECT lal_file_voter_id FROM lal_file_voters WHERE ST_Contains(ST_GEOMFROMTEXT(" + ewkt + ", 4326), lal_file_voters.address_residence_coordinates)", { is_verbose : true })
|
|
72
|
+
// const response = await utils.rds.put("federal_census_counties", {
|
|
73
|
+
// federal_census_county_id : federal_census_county.federal_census_county_id,
|
|
74
|
+
// federal_census_id : federal_census_county.federal_census_id,
|
|
75
|
+
// county_id : federal_census_county.county_id,
|
|
76
|
+
// state_id : federal_census_county.state_id,
|
|
77
|
+
// county_type_id : federal_census_county.county_type_id,
|
|
78
|
+
// function_type_id : federal_census_county.function_type_id,
|
|
79
|
+
// name : federal_census_county.name,
|
|
80
|
+
// geometry : {
|
|
81
|
+
// expression : "ST_GEOMFROMTEXT(" + ewkt + ", 4326)"
|
|
82
|
+
// }
|
|
83
|
+
// }, { is_verbose : true })
|
|
84
|
+
// console.log(response)
|
|
85
|
+
// }
|
package/src/index.ts
CHANGED
|
@@ -11,12 +11,15 @@ import { UtilsXAI } from "./UtilsXAI"
|
|
|
11
11
|
import { UtilsAnthropic } from "./UtilsAnthropic"
|
|
12
12
|
import { UtilsTwitter } from "./UtilsTwitter"
|
|
13
13
|
import { GlobalConfig } from "./types/GlobalConfig"
|
|
14
|
+
import { UtilsRDS } from "./UtilsRDS"
|
|
15
|
+
import { ApplicationConfig } from "./types/ApplicationConfig"
|
|
14
16
|
|
|
15
17
|
|
|
16
18
|
|
|
17
19
|
export class TriangleUtils extends UtilsMisc {
|
|
18
20
|
|
|
19
21
|
readonly dynamodb : UtilsDynamoDB
|
|
22
|
+
readonly rds : UtilsRDS
|
|
20
23
|
readonly s3 : UtilsS3
|
|
21
24
|
readonly s3vectors : UtilsS3Vectors
|
|
22
25
|
readonly bedrock : UtilsBedrock
|
|
@@ -28,9 +31,10 @@ export class TriangleUtils extends UtilsMisc {
|
|
|
28
31
|
readonly xai : UtilsXAI
|
|
29
32
|
readonly twitter : UtilsTwitter
|
|
30
33
|
|
|
31
|
-
constructor(config : GlobalConfig) {
|
|
34
|
+
constructor(config : ApplicationConfig & GlobalConfig) {
|
|
32
35
|
super(config)
|
|
33
36
|
this.dynamodb = new UtilsDynamoDB(config.region)
|
|
37
|
+
this.rds = new UtilsRDS(config.region, config.rds_cluster_arn, config.rds_secret_arn, config.rds_host, config.rds_username, config.rds_password)
|
|
34
38
|
this.s3 = new UtilsS3(config.region)
|
|
35
39
|
this.s3vectors = new UtilsS3Vectors(config.region)
|
|
36
40
|
this.bedrock = new UtilsBedrock(config.region)
|
|
@@ -49,6 +53,7 @@ export * from "./types/GlobalConfig"
|
|
|
49
53
|
export * from "./UtilsMisc"
|
|
50
54
|
export * from "./UtilsAnthropic"
|
|
51
55
|
export * from "./UtilsDynamoDB"
|
|
56
|
+
export * from "./UtilsRDS"
|
|
52
57
|
export * from "./UtilsS3"
|
|
53
58
|
export * from "./UtilsBedrock"
|
|
54
59
|
export * from "./UtilsBee"
|
|
@@ -8,6 +8,11 @@ export class GlobalConfig {
|
|
|
8
8
|
readonly s3_prism : string
|
|
9
9
|
readonly s3_scout : string
|
|
10
10
|
readonly s3_triage : string
|
|
11
|
+
readonly rds_cluster_arn : string
|
|
12
|
+
readonly rds_secret_arn : string
|
|
13
|
+
readonly rds_host : string
|
|
14
|
+
readonly rds_username : string
|
|
15
|
+
readonly rds_password : string
|
|
11
16
|
readonly super_federal_gov_api_keys : string
|
|
12
17
|
readonly federal_gov_api_keys : string
|
|
13
18
|
readonly federal_census_api_key : string
|
|
@@ -31,6 +36,11 @@ export class GlobalConfig {
|
|
|
31
36
|
this.s3_prism = global_config.s3_prism
|
|
32
37
|
this.s3_scout = global_config.s3_scout
|
|
33
38
|
this.s3_triage = global_config.s3_triage
|
|
39
|
+
this.rds_cluster_arn = global_config.rds_cluster_arn
|
|
40
|
+
this.rds_secret_arn = global_config.rds_secret_arn
|
|
41
|
+
this.rds_host = global_config.rds_host
|
|
42
|
+
this.rds_username = global_config.rds_username
|
|
43
|
+
this.rds_password = global_config.rds_password
|
|
34
44
|
this.super_federal_gov_api_keys = global_config.super_federal_gov_api_keys
|
|
35
45
|
this.federal_gov_api_keys = global_config.federal_gov_api_keys
|
|
36
46
|
this.federal_census_api_key = global_config.federal_census_api_key
|
|
@@ -53,6 +63,11 @@ export class GlobalConfig {
|
|
|
53
63
|
typeof global_config.s3_prism === "string" &&
|
|
54
64
|
typeof global_config.s3_scout === "string" &&
|
|
55
65
|
typeof global_config.s3_triage === "string" &&
|
|
66
|
+
typeof global_config.rds_cluster_arn === "string" &&
|
|
67
|
+
typeof global_config.rds_secret_arn === "string" &&
|
|
68
|
+
typeof global_config.rds_host === "string" &&
|
|
69
|
+
typeof global_config.rds_username === "string" &&
|
|
70
|
+
typeof global_config.rds_password === "string" &&
|
|
56
71
|
typeof global_config.super_federal_gov_api_keys === "string" &&
|
|
57
72
|
typeof global_config.federal_gov_api_keys === "string" &&
|
|
58
73
|
typeof global_config.federal_census_api_key === "string" &&
|