triangle-utils 1.4.122 → 1.4.124

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (11) hide show
  1. package/dist/src/UtilsRDS.d.ts +28 -22
  2. package/dist/src/UtilsRDS.js +101 -67
  3. package/dist/src/f.js +48 -5
  4. package/dist/src/index.js +1 -1
  5. package/dist/src/types/GlobalConfig.d.ts +3 -0
  6. package/dist/src/types/GlobalConfig.js +9 -0
  7. package/package.json +3 -1
  8. package/src/UtilsRDS.ts +139 -71
  9. package/src/f.ts +50 -5
  10. package/src/index.ts +1 -1
  11. package/src/types/GlobalConfig.ts +9 -0
package/dist/src/UtilsRDS.d.ts CHANGED
@@ -1,34 +1,40 @@
1
- import { Field } from "@aws-sdk/client-rds-data";
1
+ import { Client } from "pg";
2
2
  export declare class UtilsRDS {
3
- private readonly rds;
3
+ private readonly postgres;
4
+ private readonly rds_data;
4
5
  private readonly rds_cluster_arn;
5
6
  private readonly rds_secret_arn;
6
- constructor(region: string, rds_cluster_arn: string, rds_secret_arn: string);
7
- exec(sql: string): Promise<Field[][]>;
8
- get(table_id: string, primary_key: Record<string, any>, attribute_names: string[], options?: {
9
- is_verbose?: boolean;
10
- }): Promise<{
11
- [k: string]: string | number | boolean | undefined;
12
- } | undefined>;
13
- batch_get(table_id: string, primary_key: Record<string, any[]>, attribute_names: string[], options?: {
14
- is_verbose?: boolean;
15
- }): Promise<{
16
- [k: string]: string | number | boolean | undefined;
17
- }[]>;
7
+ constructor(region: string, rds_cluster_arn: string, rds_secret_arn: string, rds_host: string, rds_username: string, rds_password: string);
8
+ connect(): Promise<Client>;
9
+ disconnect(): Promise<void>;
10
+ query(sql: string, options?: {
11
+ is_verbose?: boolean;
12
+ }): Promise<any[]>;
13
+ exec(sql: string, options?: {
14
+ is_verbose?: boolean;
15
+ }): Promise<any[]>;
16
+ get(table_id: string, primary_key: Record<string, any>, attribute_names?: string[], options?: {
17
+ is_verbose?: boolean;
18
+ }): Promise<Record<string, any> | undefined>;
19
+ batch_get(table_id: string, primary_key: Record<string, any[]>, attribute_names?: string[], options?: {
20
+ is_verbose?: boolean;
21
+ }): Promise<any[]>;
22
+ create(table_id: string, item: Record<string, any>, options?: {
23
+ is_verbose?: boolean;
24
+ }): Promise<any[] | undefined>;
25
+ batch_create(table_id: string, items: Record<string, any>[], options?: {
26
+ is_verbose?: boolean;
27
+ }): Promise<any[] | undefined>;
18
28
  put(table_id: string, item: Record<string, any>, options?: {
19
29
  is_verbose?: boolean;
20
- }): Promise<{
21
- [k: string]: string | number | boolean | undefined;
22
- }[] | undefined>;
30
+ }): Promise<any[] | undefined>;
23
31
  batch_put(table_id: string, items: Record<string, any>[], options?: {
24
32
  is_verbose?: boolean;
25
- }): Promise<{
26
- [k: string]: string | number | boolean | undefined;
27
- }[] | undefined>;
33
+ }): Promise<any[] | undefined>;
28
34
  delete(table_id: string, primary_key: Record<string, any>, options?: {
29
35
  is_verbose?: boolean;
30
- }): Promise<undefined>;
36
+ }): Promise<any[]>;
31
37
  batch_delete(table_id: string, primary_key: Record<string, any[]>, options?: {
32
38
  is_verbose?: boolean;
33
- }): Promise<void>;
39
+ }): Promise<any[]>;
34
40
  }
package/dist/src/UtilsRDS.js CHANGED
@@ -1,22 +1,5 @@
1
1
  import { RDSData } from "@aws-sdk/client-rds-data";
2
- function convert_rds_output(rds_output) {
3
- if (rds_output.isNull) {
4
- return undefined;
5
- }
6
- else if (rds_output.stringValue !== undefined) {
7
- return rds_output.stringValue;
8
- }
9
- else if (rds_output.longValue !== undefined) {
10
- return rds_output.longValue;
11
- }
12
- else if (rds_output.doubleValue !== undefined) {
13
- return rds_output.doubleValue;
14
- }
15
- else if (rds_output.booleanValue !== undefined) {
16
- return rds_output.booleanValue;
17
- }
18
- return undefined;
19
- }
2
+ import { Client } from "pg";
20
3
  function convert_rds_input(input) {
21
4
  if (input === undefined) {
22
5
  return "NULL";
@@ -30,58 +13,123 @@ function convert_rds_input(input) {
30
13
  else if (typeof input === "boolean") {
31
14
  return input.toString().toUpperCase();
32
15
  }
16
+ else if (typeof input === "object") {
17
+ if (input.expression !== undefined) {
18
+ return input.expression;
19
+ }
20
+ }
33
21
  return "NULL";
34
22
  }
35
23
  export class UtilsRDS {
36
- rds;
24
+ postgres;
25
+ rds_data;
37
26
  rds_cluster_arn;
38
27
  rds_secret_arn;
39
- constructor(region, rds_cluster_arn, rds_secret_arn) {
40
- this.rds = new RDSData({ region: region });
28
+ constructor(region, rds_cluster_arn, rds_secret_arn, rds_host, rds_username, rds_password) {
29
+ this.postgres = new Client({
30
+ host: rds_host,
31
+ user: rds_username,
32
+ password: rds_password,
33
+ database: "postgres",
34
+ port: 5432,
35
+ ssl: {
36
+ rejectUnauthorized: false
37
+ }
38
+ });
39
+ this.rds_data = new RDSData({ region: region });
41
40
  this.rds_cluster_arn = rds_cluster_arn,
42
41
  this.rds_secret_arn = rds_secret_arn;
43
42
  }
44
- async exec(sql) {
45
- const response = await this.rds.executeStatement({
46
- resourceArn: this.rds_cluster_arn,
47
- secretArn: this.rds_secret_arn,
48
- database: "postgres",
49
- sql: sql
50
- });
51
- return response.records || [];
43
+ async connect() {
44
+ return await this.postgres.connect();
52
45
  }
53
- async get(table_id, primary_key, attribute_names, options = {}) {
46
+ async disconnect() {
47
+ return await this.postgres.end();
48
+ }
49
+ async query(sql, options = {}) {
54
50
  const is_verbose = Boolean(options.is_verbose);
55
- const sql = "SELECT " + attribute_names.join(", ") + " FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, value]) => key_name + " = " + convert_rds_input(value)).join(" AND ");
56
51
  if (is_verbose) {
57
52
  console.log(sql);
58
53
  }
59
- const items = await this.exec(sql);
60
- const item = items[0];
61
- if (item === undefined) {
62
- return undefined;
54
+ const response_connect = await this.connect();
55
+ if (is_verbose) {
56
+ console.log(response_connect);
63
57
  }
64
- return Object.fromEntries(attribute_names.map((attribute_name, i) => [attribute_name, convert_rds_output(item[i])]));
58
+ const response = await this.postgres.query(sql);
59
+ const response_disconnect = await this.disconnect();
60
+ if (is_verbose) {
61
+ console.log(response_disconnect);
62
+ }
63
+ return response.rows;
65
64
  }
66
- async batch_get(table_id, primary_key, attribute_names, options = {}) {
65
+ async exec(sql, options = {}) {
67
66
  const is_verbose = Boolean(options.is_verbose);
68
- const sql = "SELECT " + attribute_names.join(", ") + " FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, values]) => key_name + " IN " + "(" + values.map(value => convert_rds_input(value)).join(", ") + ")").join(" AND ");
69
67
  if (is_verbose) {
70
68
  console.log(sql);
71
69
  }
72
- const items = await this.exec(sql);
73
- return items.map(item => Object.fromEntries(attribute_names.map((attribute_name, i) => [attribute_name, convert_rds_output(item[i])])));
70
+ const items = sql.length <= 60000 ?
71
+ await this.rds_data.executeStatement({
72
+ resourceArn: this.rds_cluster_arn,
73
+ secretArn: this.rds_secret_arn,
74
+ database: "postgres",
75
+ sql: sql,
76
+ formatRecordsAs: "JSON"
77
+ })
78
+ .then(response => JSON.parse(response.formattedRecords || "[]"))
79
+ :
80
+ await this.query(sql, options);
81
+ if (!Array.isArray(items)) {
82
+ if (is_verbose) {
83
+ console.log("Bad items:", items);
84
+ }
85
+ return [];
86
+ }
87
+ return items;
88
+ }
89
+ async get(table_id, primary_key, attribute_names, options = {}) {
90
+ const sql = "SELECT " + (attribute_names === undefined ? "*" : attribute_names.join(", ")) + " FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, value]) => key_name + " = " + convert_rds_input(value)).join(" AND ");
91
+ const items = await this.exec(sql, options);
92
+ const item = items[0];
93
+ return item;
94
+ }
95
+ async batch_get(table_id, primary_key, attribute_names, options = {}) {
96
+ const sql = "SELECT " + (attribute_names === undefined ? "*" : attribute_names.join(", ")) + " FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, values]) => key_name + " IN " + "(" + values.map(value => convert_rds_input(value)).join(", ") + ")").join(" AND ");
97
+ const items = await this.exec(sql, options);
98
+ return items;
99
+ }
100
+ async create(table_id, item, options = {}) {
101
+ const attribute_names = Object.keys(item).sort();
102
+ const sql = "INSERT INTO " + table_id + " (" + attribute_names.join(", ") + ")" + " VALUES " + "(" + attribute_names.map(attribute_name => convert_rds_input(item[attribute_name])).join(", ") + ")" + " ON CONFLICT ON CONSTRAINT " + table_id + "_pkey IGNORE;";
103
+ try {
104
+ const items = await this.exec(sql, options);
105
+ return items;
106
+ }
107
+ catch (error) {
108
+ console.log(error.stack);
109
+ return;
110
+ }
111
+ }
112
+ async batch_create(table_id, items, options = {}) {
113
+ if (items.length === 0) {
114
+ return;
115
+ }
116
+ const attribute_names = Array.from(new Set(items.map(item => Object.keys(item)).flat())).sort();
117
+ const sql = "INSERT INTO " + table_id + " (" + attribute_names.join(", ") + ")" + " VALUES " + items.map(item => "(" + attribute_names.map(attribute_name => convert_rds_input(item[attribute_name])).join(", ") + ")").join(", ") + " ON CONFLICT ON CONSTRAINT " + table_id + "_pkey IGNORE;";
118
+ try {
119
+ const items = await this.exec(sql, options);
120
+ return items;
121
+ }
122
+ catch (error) {
123
+ console.log(error.stack);
124
+ return;
125
+ }
74
126
  }
75
127
  async put(table_id, item, options = {}) {
76
- const is_verbose = Boolean(options.is_verbose);
77
128
  const attribute_names = Object.keys(item).sort();
78
129
  const sql = "INSERT INTO " + table_id + " (" + attribute_names.join(", ") + ")" + " VALUES " + "(" + attribute_names.map(attribute_name => convert_rds_input(item[attribute_name])).join(", ") + ")" + " ON CONFLICT ON CONSTRAINT " + table_id + "_pkey DO UPDATE SET " + attribute_names.map(attribute_name => attribute_name + " = " + "EXCLUDED." + attribute_name).join(", ") + ";";
79
- if (is_verbose) {
80
- console.log(sql);
81
- }
82
130
  try {
83
- const items = await this.exec(sql);
84
- return items.map(item => Object.fromEntries(attribute_names.map((attribute_name, i) => [attribute_name, convert_rds_output(item[i])])));
131
+ const items = await this.exec(sql, options);
132
+ return items;
85
133
  }
86
134
  catch (error) {
87
135
  console.log(error.stack);
@@ -89,15 +137,14 @@ export class UtilsRDS {
89
137
  }
90
138
  }
91
139
  async batch_put(table_id, items, options = {}) {
92
- const is_verbose = Boolean(options.is_verbose);
140
+ if (items.length === 0) {
141
+ return;
142
+ }
93
143
  const attribute_names = Array.from(new Set(items.map(item => Object.keys(item)).flat())).sort();
94
144
  const sql = "INSERT INTO " + table_id + " (" + attribute_names.join(", ") + ")" + " VALUES " + items.map(item => "(" + attribute_names.map(attribute_name => convert_rds_input(item[attribute_name])).join(", ") + ")").join(", ") + " ON CONFLICT ON CONSTRAINT " + table_id + "_pkey DO UPDATE SET " + attribute_names.map(attribute_name => attribute_name + " = " + "EXCLUDED." + attribute_name).join(", ") + ";";
95
- if (is_verbose) {
96
- console.log(sql);
97
- }
98
145
  try {
99
- const items = await this.exec(sql);
100
- return items.map(item => Object.fromEntries(attribute_names.map((attribute_name, i) => [attribute_name, convert_rds_output(item[i])])));
146
+ const items = await this.exec(sql, options);
147
+ return items;
101
148
  }
102
149
  catch (error) {
103
150
  console.log(error.stack);
@@ -105,24 +152,11 @@ export class UtilsRDS {
105
152
  }
106
153
  }
107
154
  async delete(table_id, primary_key, options = {}) {
108
- const is_verbose = Boolean(options.is_verbose);
109
155
  const sql = "DELETE FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, value]) => key_name + " = " + convert_rds_input(value)).join(" AND ");
110
- if (is_verbose) {
111
- console.log(sql);
112
- }
113
- const items = await this.exec(sql);
114
- const item = items[0];
115
- if (item === undefined) {
116
- return undefined;
117
- }
118
- return undefined;
156
+ return await this.exec(sql, options);
119
157
  }
120
158
  async batch_delete(table_id, primary_key, options = {}) {
121
- const is_verbose = Boolean(options.is_verbose);
122
159
  const sql = "DELETE FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, values]) => key_name + " IN " + "(" + values.map(value => convert_rds_input(value)).join(", ") + ")").join(" AND ");
123
- if (is_verbose) {
124
- console.log(sql);
125
- }
126
- await this.exec(sql);
160
+ return await this.exec(sql, options);
127
161
  }
128
162
  }
package/dist/src/f.js CHANGED
@@ -15,12 +15,55 @@ const config = {
15
15
  };
16
16
  console.log(config);
17
17
  const utils = new TriangleUtils(config);
18
- const federal_censuses = await utils.dynamodb.scan("federal_censuses");
19
- federal_censuses.sort((a, b) => a.federal_census_id - b.federal_census_id);
18
+ // const federal_censuses = await utils.dynamodb.scan("federal_censuses")
19
+ // federal_censuses.sort((a, b) => a.federal_census_id - b.federal_census_id)
20
20
  // console.log(JSON.stringify(response, undefined, 4))
21
21
  // await utils.rds.batch_delete("federal_censuses", {
22
22
  // federal_census_id : federal_censuses.map(federal_census => federal_census.federal_census_id)
23
23
  // }, { is_verbose : true })
24
- await utils.rds.batch_put("federal_censuses", federal_censuses);
25
- const rds_federal_censuses = await utils.rds.batch_get("federal_censuses", { federal_census_id: federal_censuses.map(federal_census => federal_census.federal_census_id) }, ["federal_census_id", "is_decennial"], { is_verbose: true });
26
- console.log(rds_federal_censuses);
24
+ // await utils.rds.batch_put("federal_censuses", federal_censuses)
25
+ // const rds_federal_censuses = await utils.rds.batch_get("federal_censuses", { federal_census_id : federal_censuses.map(federal_census => federal_census.federal_census_id) }, ["federal_census_id", "is_decennial"], { is_verbose : true })
26
+ // console.log(rds_federal_censuses)
27
+ // const voters = await utils.dynamodb.scan("voters", { compile : false, max_num_items : 100 })
28
+ // for (const voter of voters) {
29
+ // const new_lal_file_voter = {
30
+ // lal_file_voter_id : voter.state_id + "-2026-08-15" + "|" + voter.voter_id,
31
+ // lal_file_id : voter.state_id + "-2026-08-15",
32
+ // lal_voter_id : voter.voter_id,
33
+ // voter_id : voter.state_voter_id,
34
+ // state_id : voter.state_id,
35
+ // address_residence_latitude : voter.address_residence_latitude,
36
+ // address_residence_longitude : voter.address_residence_longitude,
37
+ // address_residence_coordinates : "ST_GeomFromText('POINT(" + voter.address_residence_latitude + " " + voter.address_residence_longitude + ")', 4326)"
38
+ // }
39
+ // console.log(new_lal_file_voter)
40
+ // await utils.rds.put("lal_file_voters", new_lal_file_voter)
41
+ // }
42
+ // const federal_census_county_id = "2026-04021"
43
+ // const federal_census_county = await utils.rds.get("federal_census_counties", { federal_census_county_id : federal_census_county_id }, ["*"])
44
+ // console.log(federal_census_county)
45
+ // console.log(await utils.rds.exec("SELECT * FROM federal_censuses"))
46
+ // const federal_census_county = await utils.dynamodb.get("federal_census_counties", { federal_census_county_id : federal_census_county_id })
47
+ // const s3_id = config.s3_triangle + "/federal_census/2026/counties/" + federal_census_county_id + ".json"
48
+ // console.log(s3_id)
49
+ // const feature = await utils.s3.get(s3_id)
50
+ // // console.log(feature)
51
+ // if (feature !== undefined && federal_census_county !== undefined) {
52
+ // const geometry = "(" + JSON.parse(feature).geometry.coordinates[0].map((polygon : any) => "(" + polygon.map((vertex : any) => vertex[1] + " " + vertex[0]).join(", ") + ")").join(", ") + ")"
53
+ // const ewkt = "'MULTIPOLYGON(" + geometry + ")'"
54
+ // // const response = await utils.rds.exec("SELECT ST_AsText(address_residence_coordinates) FROM lal_file_voters", { is_verbose : true })
55
+ // // const response = await utils.rds.exec("SELECT lal_file_voter_id FROM lal_file_voters WHERE ST_Contains(ST_GEOMFROMTEXT(" + ewkt + ", 4326), lal_file_voters.address_residence_coordinates)", { is_verbose : true })
56
+ // const response = await utils.rds.put("federal_census_counties", {
57
+ // federal_census_county_id : federal_census_county.federal_census_county_id,
58
+ // federal_census_id : federal_census_county.federal_census_id,
59
+ // county_id : federal_census_county.county_id,
60
+ // state_id : federal_census_county.state_id,
61
+ // county_type_id : federal_census_county.county_type_id,
62
+ // function_type_id : federal_census_county.function_type_id,
63
+ // name : federal_census_county.name,
64
+ // geometry : {
65
+ // expression : "ST_GEOMFROMTEXT(" + ewkt + ", 4326)"
66
+ // }
67
+ // }, { is_verbose : true })
68
+ // console.log(response)
69
+ // }
package/dist/src/index.js CHANGED
@@ -27,7 +27,7 @@ export class TriangleUtils extends UtilsMisc {
27
27
  constructor(config) {
28
28
  super(config);
29
29
  this.dynamodb = new UtilsDynamoDB(config.region);
30
- this.rds = new UtilsRDS(config.region, config.rds_cluster_arn, config.rds_secret_arn);
30
+ this.rds = new UtilsRDS(config.region, config.rds_cluster_arn, config.rds_secret_arn, config.rds_host, config.rds_username, config.rds_password);
31
31
  this.s3 = new UtilsS3(config.region);
32
32
  this.s3vectors = new UtilsS3Vectors(config.region);
33
33
  this.bedrock = new UtilsBedrock(config.region);
package/dist/src/types/GlobalConfig.d.ts CHANGED
@@ -10,6 +10,9 @@ export declare class GlobalConfig {
10
10
  readonly s3_triage: string;
11
11
  readonly rds_cluster_arn: string;
12
12
  readonly rds_secret_arn: string;
13
+ readonly rds_host: string;
14
+ readonly rds_username: string;
15
+ readonly rds_password: string;
13
16
  readonly super_federal_gov_api_keys: string;
14
17
  readonly federal_gov_api_keys: string;
15
18
  readonly federal_census_api_key: string;
package/dist/src/types/GlobalConfig.js CHANGED
@@ -10,6 +10,9 @@ export class GlobalConfig {
10
10
  s3_triage;
11
11
  rds_cluster_arn;
12
12
  rds_secret_arn;
13
+ rds_host;
14
+ rds_username;
15
+ rds_password;
13
16
  super_federal_gov_api_keys;
14
17
  federal_gov_api_keys;
15
18
  federal_census_api_key;
@@ -33,6 +36,9 @@ export class GlobalConfig {
33
36
  this.s3_triage = global_config.s3_triage;
34
37
  this.rds_cluster_arn = global_config.rds_cluster_arn;
35
38
  this.rds_secret_arn = global_config.rds_secret_arn;
39
+ this.rds_host = global_config.rds_host;
40
+ this.rds_username = global_config.rds_username;
41
+ this.rds_password = global_config.rds_password;
36
42
  this.super_federal_gov_api_keys = global_config.super_federal_gov_api_keys;
37
43
  this.federal_gov_api_keys = global_config.federal_gov_api_keys;
38
44
  this.federal_census_api_key = global_config.federal_census_api_key;
@@ -55,6 +61,9 @@ export class GlobalConfig {
55
61
  typeof global_config.s3_triage === "string" &&
56
62
  typeof global_config.rds_cluster_arn === "string" &&
57
63
  typeof global_config.rds_secret_arn === "string" &&
64
+ typeof global_config.rds_host === "string" &&
65
+ typeof global_config.rds_username === "string" &&
66
+ typeof global_config.rds_password === "string" &&
58
67
  typeof global_config.super_federal_gov_api_keys === "string" &&
59
68
  typeof global_config.federal_gov_api_keys === "string" &&
60
69
  typeof global_config.federal_census_api_key === "string" &&
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "triangle-utils",
3
- "version": "1.4.122",
3
+ "version": "1.4.124",
4
4
  "main": "dist/src/index.js",
5
5
  "types": "dist/src/index.d.ts",
6
6
  "directories": {
@@ -31,9 +31,11 @@
31
31
  "@types/jsdom": "^28.0.1",
32
32
  "@types/node": "^25.2.3",
33
33
  "@types/nodemailer": "^7.0.9",
34
+ "@types/pg": "^8.21.0",
34
35
  "@xdevplatform/xdk": "^0.5.0",
35
36
  "googleapis": "^170.0.0",
36
37
  "jsdom": "^29.0.1",
38
+ "pg": "^8.23.0",
37
39
  "scrapingbee": "^1.8.2"
38
40
  },
39
41
  "devDependencies": {
package/src/UtilsRDS.ts CHANGED
@@ -1,21 +1,7 @@
1
1
  import { Field, RDSData } from "@aws-sdk/client-rds-data"
2
+ import { Client } from "pg"
2
3
 
3
- function convert_rds_output(rds_output : Field) : string | boolean | number | undefined {
4
- if (rds_output.isNull) {
5
- return undefined
6
- } else if (rds_output.stringValue !== undefined) {
7
- return rds_output.stringValue
8
- } else if (rds_output.longValue !== undefined) {
9
- return rds_output.longValue
10
- } else if (rds_output.doubleValue !== undefined) {
11
- return rds_output.doubleValue
12
- } else if (rds_output.booleanValue !== undefined) {
13
- return rds_output.booleanValue
14
- }
15
- return undefined
16
- }
17
-
18
- function convert_rds_input(input : string | boolean | number | undefined) : string {
4
+ function convert_rds_input(input : string | boolean | number | Record<string, any> | undefined) : string {
19
5
  if (input === undefined) {
20
6
  return "NULL"
21
7
  } else if (typeof input === "string") {
@@ -24,68 +10,168 @@ function convert_rds_input(input : string | boolean | number | undefined) : stri
24
10
  return input.toString()
25
11
  } else if (typeof input === "boolean") {
26
12
  return input.toString().toUpperCase()
13
+ } else if (typeof input === "object") {
14
+ if (input.expression !== undefined) {
15
+ return input.expression
16
+ }
27
17
  }
28
18
  return "NULL"
29
19
  }
30
20
 
31
21
  export class UtilsRDS {
32
22
 
33
- private readonly rds : RDSData
23
+ private readonly postgres : Client
24
+ private readonly rds_data : RDSData
34
25
  private readonly rds_cluster_arn : string
35
26
  private readonly rds_secret_arn : string
36
27
 
37
- constructor(region : string, rds_cluster_arn : string, rds_secret_arn : string) {
38
- this.rds = new RDSData({ region : region })
28
+ constructor(
29
+ region : string,
30
+ rds_cluster_arn : string,
31
+ rds_secret_arn : string,
32
+ rds_host : string,
33
+ rds_username : string,
34
+ rds_password : string
35
+ ) {
36
+ this.postgres = new Client({
37
+ host : rds_host,
38
+ user : rds_username,
39
+ password : rds_password,
40
+ database : "postgres",
41
+ port : 5432,
42
+ ssl : {
43
+ rejectUnauthorized: false
44
+ }
45
+ })
46
+ this.rds_data = new RDSData({ region : region })
39
47
  this.rds_cluster_arn = rds_cluster_arn,
40
48
  this.rds_secret_arn = rds_secret_arn
41
49
  }
42
50
 
43
- async exec(sql : string) {
44
- const response = await this.rds.executeStatement({
45
- resourceArn : this.rds_cluster_arn,
46
- secretArn : this.rds_secret_arn,
47
- database : "postgres",
48
- sql : sql
49
- })
50
- return response.records || []
51
+ async connect() {
52
+ return await this.postgres.connect()
51
53
  }
52
54
 
53
- async get(
54
- table_id : string,
55
- primary_key : Record<string, any>,
56
- attribute_names : string[],
55
+ async disconnect() {
56
+ return await this.postgres.end()
57
+ }
58
+
59
+ async query(
60
+ sql : string,
57
61
  options : {
58
62
  is_verbose? : boolean
59
63
  } = {}
60
64
  ) {
61
65
  const is_verbose = Boolean(options.is_verbose)
62
- const sql = "SELECT " + attribute_names.join(", ") + " FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, value]) => key_name + " = " + convert_rds_input(value)).join(" AND ")
63
66
  if (is_verbose) {
64
67
  console.log(sql)
65
68
  }
66
- const items = await this.exec(sql)
67
- const item = items[0]
68
- if (item === undefined) {
69
- return undefined
69
+ const response_connect = await this.connect()
70
+ if (is_verbose) {
71
+ console.log(response_connect)
72
+ }
73
+ const response = await this.postgres.query(sql)
74
+ const response_disconnect = await this.disconnect()
75
+ if (is_verbose) {
76
+ console.log(response_disconnect)
70
77
  }
71
- return Object.fromEntries(attribute_names.map((attribute_name, i) => [attribute_name, convert_rds_output(item[i])]))
78
+ return response.rows
72
79
  }
73
80
 
74
- async batch_get(
75
- table_id : string,
76
- primary_key : Record<string, any[]>,
77
- attribute_names : string[],
81
+ async exec(
82
+ sql : string,
78
83
  options : {
79
84
  is_verbose? : boolean
80
85
  } = {}
81
86
  ) {
82
87
  const is_verbose = Boolean(options.is_verbose)
83
- const sql = "SELECT " + attribute_names.join(", ") + " FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, values]) => key_name + " IN " + "(" + values.map(value => convert_rds_input(value)).join(", ") + ")").join(" AND ")
84
88
  if (is_verbose) {
85
89
  console.log(sql)
86
90
  }
87
- const items = await this.exec(sql)
88
- return items.map(item => Object.fromEntries(attribute_names.map((attribute_name, i) => [attribute_name, convert_rds_output(item[i])])))
91
+ const items = sql.length <= 60000 ?
92
+ await this.rds_data.executeStatement({
93
+ resourceArn : this.rds_cluster_arn,
94
+ secretArn : this.rds_secret_arn,
95
+ database : "postgres",
96
+ sql : sql,
97
+ formatRecordsAs : "JSON"
98
+ })
99
+ .then(response => JSON.parse(response.formattedRecords || "[]"))
100
+ :
101
+ await this.query(sql, options)
102
+ if (!Array.isArray(items)) {
103
+ if (is_verbose) {
104
+ console.log("Bad items:", items)
105
+ }
106
+ return []
107
+ }
108
+ return items
109
+ }
110
+
111
+ async get(
112
+ table_id : string,
113
+ primary_key : Record<string, any>,
114
+ attribute_names? : string[],
115
+ options : {
116
+ is_verbose? : boolean
117
+ } = {}
118
+ ) {
119
+ const sql = "SELECT " + (attribute_names === undefined ? "*" : attribute_names.join(", ")) + " FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, value]) => key_name + " = " + convert_rds_input(value)).join(" AND ")
120
+ const items = await this.exec(sql, options)
121
+ const item : Record<string, any> | undefined = items[0]
122
+ return item
123
+ }
124
+
125
+ async batch_get(
126
+ table_id : string,
127
+ primary_key : Record<string, any[]>,
128
+ attribute_names? : string[],
129
+ options : {
130
+ is_verbose? : boolean
131
+ } = {}
132
+ ) {
133
+ const sql = "SELECT " + (attribute_names === undefined ? "*" : attribute_names.join(", ")) + " FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, values]) => key_name + " IN " + "(" + values.map(value => convert_rds_input(value)).join(", ") + ")").join(" AND ")
134
+ const items = await this.exec(sql, options)
135
+ return items
136
+ }
137
+
138
+ async create(
139
+ table_id : string,
140
+ item : Record<string, any>,
141
+ options : {
142
+ is_verbose? : boolean
143
+ } = {}
144
+ ) {
145
+ const attribute_names = Object.keys(item).sort()
146
+ const sql = "INSERT INTO " + table_id + " (" + attribute_names.join(", ") + ")" + " VALUES " + "(" + attribute_names.map(attribute_name => convert_rds_input(item[attribute_name])).join(", ") + ")" + " ON CONFLICT ON CONSTRAINT " + table_id + "_pkey IGNORE;"
147
+ try {
148
+ const items = await this.exec(sql, options)
149
+ return items
150
+ } catch (error : any) {
151
+ console.log(error.stack)
152
+ return
153
+ }
154
+ }
155
+
156
+ async batch_create(
157
+ table_id : string,
158
+ items : Record<string, any>[],
159
+ options : {
160
+ is_verbose? : boolean
161
+ } = {}
162
+ ) {
163
+ if (items.length === 0) {
164
+ return
165
+ }
166
+ const attribute_names = Array.from(new Set(items.map(item => Object.keys(item)).flat())).sort()
167
+ const sql = "INSERT INTO " + table_id + " (" + attribute_names.join(", ") + ")" + " VALUES " + items.map(item => "(" + attribute_names.map(attribute_name => convert_rds_input(item[attribute_name])).join(", ") + ")").join(", ") + " ON CONFLICT ON CONSTRAINT " + table_id + "_pkey IGNORE;"
168
+ try {
169
+ const items = await this.exec(sql, options)
170
+ return items
171
+ } catch (error : any) {
172
+ console.log(error.stack)
173
+ return
174
+ }
89
175
  }
90
176
 
91
177
  async put(
@@ -95,15 +181,11 @@ export class UtilsRDS {
95
181
  is_verbose? : boolean
96
182
  } = {}
97
183
  ) {
98
- const is_verbose = Boolean(options.is_verbose)
99
184
  const attribute_names = Object.keys(item).sort()
100
185
  const sql = "INSERT INTO " + table_id + " (" + attribute_names.join(", ") + ")" + " VALUES " + "(" + attribute_names.map(attribute_name => convert_rds_input(item[attribute_name])).join(", ") + ")" + " ON CONFLICT ON CONSTRAINT " + table_id + "_pkey DO UPDATE SET " + attribute_names.map(attribute_name => attribute_name + " = " + "EXCLUDED." + attribute_name).join(", ") + ";"
101
- if (is_verbose) {
102
- console.log(sql)
103
- }
104
186
  try {
105
- const items = await this.exec(sql)
106
- return items.map(item => Object.fromEntries(attribute_names.map((attribute_name, i) => [attribute_name, convert_rds_output(item[i])])))
187
+ const items = await this.exec(sql, options)
188
+ return items
107
189
  } catch (error : any) {
108
190
  console.log(error.stack)
109
191
  return
@@ -117,15 +199,14 @@ export class UtilsRDS {
117
199
  is_verbose? : boolean
118
200
  } = {}
119
201
  ) {
120
- const is_verbose = Boolean(options.is_verbose)
202
+ if (items.length === 0) {
203
+ return
204
+ }
121
205
  const attribute_names = Array.from(new Set(items.map(item => Object.keys(item)).flat())).sort()
122
206
  const sql = "INSERT INTO " + table_id + " (" + attribute_names.join(", ") + ")" + " VALUES " + items.map(item => "(" + attribute_names.map(attribute_name => convert_rds_input(item[attribute_name])).join(", ") + ")").join(", ") + " ON CONFLICT ON CONSTRAINT " + table_id + "_pkey DO UPDATE SET " + attribute_names.map(attribute_name => attribute_name + " = " + "EXCLUDED." + attribute_name).join(", ") + ";"
123
- if (is_verbose) {
124
- console.log(sql)
125
- }
126
207
  try {
127
- const items = await this.exec(sql)
128
- return items.map(item => Object.fromEntries(attribute_names.map((attribute_name, i) => [attribute_name, convert_rds_output(item[i])])))
208
+ const items = await this.exec(sql, options)
209
+ return items
129
210
  } catch (error : any) {
130
211
  console.log(error.stack)
131
212
  return
@@ -139,17 +220,8 @@ export class UtilsRDS {
139
220
  is_verbose? : boolean
140
221
  } = {}
141
222
  ) {
142
- const is_verbose = Boolean(options.is_verbose)
143
223
  const sql = "DELETE FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, value]) => key_name + " = " + convert_rds_input(value)).join(" AND ")
144
- if (is_verbose) {
145
- console.log(sql)
146
- }
147
- const items = await this.exec(sql)
148
- const item = items[0]
149
- if (item === undefined) {
150
- return undefined
151
- }
152
- return undefined
224
+ return await this.exec(sql, options)
153
225
  }
154
226
 
155
227
  async batch_delete(
@@ -159,12 +231,8 @@ export class UtilsRDS {
159
231
  is_verbose? : boolean
160
232
  } = {}
161
233
  ) {
162
- const is_verbose = Boolean(options.is_verbose)
163
234
  const sql = "DELETE FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, values]) => key_name + " IN " + "(" + values.map(value => convert_rds_input(value)).join(", ") + ")").join(" AND ")
164
- if (is_verbose) {
165
- console.log(sql)
166
- }
167
- await this.exec(sql)
235
+ return await this.exec(sql, options)
168
236
  }
169
237
 
170
238
 
package/src/f.ts CHANGED
@@ -24,8 +24,8 @@ console.log(config)
24
24
 
25
25
  const utils = new TriangleUtils(config)
26
26
 
27
- const federal_censuses = await utils.dynamodb.scan("federal_censuses")
28
- federal_censuses.sort((a, b) => a.federal_census_id - b.federal_census_id)
27
+ // const federal_censuses = await utils.dynamodb.scan("federal_censuses")
28
+ // federal_censuses.sort((a, b) => a.federal_census_id - b.federal_census_id)
29
29
 
30
30
  // console.log(JSON.stringify(response, undefined, 4))
31
31
 
@@ -33,8 +33,53 @@ federal_censuses.sort((a, b) => a.federal_census_id - b.federal_census_id)
33
33
  // federal_census_id : federal_censuses.map(federal_census => federal_census.federal_census_id)
34
34
  // }, { is_verbose : true })
35
35
 
36
- await utils.rds.batch_put("federal_censuses", federal_censuses)
36
+ // await utils.rds.batch_put("federal_censuses", federal_censuses)
37
37
 
38
- const rds_federal_censuses = await utils.rds.batch_get("federal_censuses", { federal_census_id : federal_censuses.map(federal_census => federal_census.federal_census_id) }, ["federal_census_id", "is_decennial"], { is_verbose : true })
38
+ // const rds_federal_censuses = await utils.rds.batch_get("federal_censuses", { federal_census_id : federal_censuses.map(federal_census => federal_census.federal_census_id) }, ["federal_census_id", "is_decennial"], { is_verbose : true })
39
39
 
40
- console.log(rds_federal_censuses)
40
+ // console.log(rds_federal_censuses)
41
+
42
+ // const voters = await utils.dynamodb.scan("voters", { compile : false, max_num_items : 100 })
43
+ // for (const voter of voters) {
44
+ // const new_lal_file_voter = {
45
+ // lal_file_voter_id : voter.state_id + "-2026-08-15" + "|" + voter.voter_id,
46
+ // lal_file_id : voter.state_id + "-2026-08-15",
47
+ // lal_voter_id : voter.voter_id,
48
+ // voter_id : voter.state_voter_id,
49
+ // state_id : voter.state_id,
50
+ // address_residence_latitude : voter.address_residence_latitude,
51
+ // address_residence_longitude : voter.address_residence_longitude,
52
+ // address_residence_coordinates : "ST_GeomFromText('POINT(" + voter.address_residence_latitude + " " + voter.address_residence_longitude + ")', 4326)"
53
+ // }
54
+ // console.log(new_lal_file_voter)
55
+ // await utils.rds.put("lal_file_voters", new_lal_file_voter)
56
+ // }
57
+ // const federal_census_county_id = "2026-04021"
58
+ // const federal_census_county = await utils.rds.get("federal_census_counties", { federal_census_county_id : federal_census_county_id }, ["*"])
59
+ // console.log(federal_census_county)
60
+ // console.log(await utils.rds.exec("SELECT * FROM federal_censuses"))
61
+ // const federal_census_county = await utils.dynamodb.get("federal_census_counties", { federal_census_county_id : federal_census_county_id })
62
+ // const s3_id = config.s3_triangle + "/federal_census/2026/counties/" + federal_census_county_id + ".json"
63
+ // console.log(s3_id)
64
+ // const feature = await utils.s3.get(s3_id)
65
+ // // console.log(feature)
66
+ // if (feature !== undefined && federal_census_county !== undefined) {
67
+ // const geometry = "(" + JSON.parse(feature).geometry.coordinates[0].map((polygon : any) => "(" + polygon.map((vertex : any) => vertex[1] + " " + vertex[0]).join(", ") + ")").join(", ") + ")"
68
+ // const ewkt = "'MULTIPOLYGON(" + geometry + ")'"
69
+
70
+ // // const response = await utils.rds.exec("SELECT ST_AsText(address_residence_coordinates) FROM lal_file_voters", { is_verbose : true })
71
+ // // const response = await utils.rds.exec("SELECT lal_file_voter_id FROM lal_file_voters WHERE ST_Contains(ST_GEOMFROMTEXT(" + ewkt + ", 4326), lal_file_voters.address_residence_coordinates)", { is_verbose : true })
72
+ // const response = await utils.rds.put("federal_census_counties", {
73
+ // federal_census_county_id : federal_census_county.federal_census_county_id,
74
+ // federal_census_id : federal_census_county.federal_census_id,
75
+ // county_id : federal_census_county.county_id,
76
+ // state_id : federal_census_county.state_id,
77
+ // county_type_id : federal_census_county.county_type_id,
78
+ // function_type_id : federal_census_county.function_type_id,
79
+ // name : federal_census_county.name,
80
+ // geometry : {
81
+ // expression : "ST_GEOMFROMTEXT(" + ewkt + ", 4326)"
82
+ // }
83
+ // }, { is_verbose : true })
84
+ // console.log(response)
85
+ // }
package/src/index.ts CHANGED
@@ -34,7 +34,7 @@ export class TriangleUtils extends UtilsMisc {
34
34
  constructor(config : ApplicationConfig & GlobalConfig) {
35
35
  super(config)
36
36
  this.dynamodb = new UtilsDynamoDB(config.region)
37
- this.rds = new UtilsRDS(config.region, config.rds_cluster_arn, config.rds_secret_arn)
37
+ this.rds = new UtilsRDS(config.region, config.rds_cluster_arn, config.rds_secret_arn, config.rds_host, config.rds_username, config.rds_password)
38
38
  this.s3 = new UtilsS3(config.region)
39
39
  this.s3vectors = new UtilsS3Vectors(config.region)
40
40
  this.bedrock = new UtilsBedrock(config.region)
package/src/types/GlobalConfig.ts CHANGED
@@ -10,6 +10,9 @@ export class GlobalConfig {
10
10
  readonly s3_triage : string
11
11
  readonly rds_cluster_arn : string
12
12
  readonly rds_secret_arn : string
13
+ readonly rds_host : string
14
+ readonly rds_username : string
15
+ readonly rds_password : string
13
16
  readonly super_federal_gov_api_keys : string
14
17
  readonly federal_gov_api_keys : string
15
18
  readonly federal_census_api_key : string
@@ -35,6 +38,9 @@ export class GlobalConfig {
35
38
  this.s3_triage = global_config.s3_triage
36
39
  this.rds_cluster_arn = global_config.rds_cluster_arn
37
40
  this.rds_secret_arn = global_config.rds_secret_arn
41
+ this.rds_host = global_config.rds_host
42
+ this.rds_username = global_config.rds_username
43
+ this.rds_password = global_config.rds_password
38
44
  this.super_federal_gov_api_keys = global_config.super_federal_gov_api_keys
39
45
  this.federal_gov_api_keys = global_config.federal_gov_api_keys
40
46
  this.federal_census_api_key = global_config.federal_census_api_key
@@ -59,6 +65,9 @@ export class GlobalConfig {
59
65
  typeof global_config.s3_triage === "string" &&
60
66
  typeof global_config.rds_cluster_arn === "string" &&
61
67
  typeof global_config.rds_secret_arn === "string" &&
68
+ typeof global_config.rds_host === "string" &&
69
+ typeof global_config.rds_username === "string" &&
70
+ typeof global_config.rds_password === "string" &&
62
71
  typeof global_config.super_federal_gov_api_keys === "string" &&
63
72
  typeof global_config.federal_gov_api_keys === "string" &&
64
73
  typeof global_config.federal_census_api_key === "string" &&