triangle-utils 1.4.122 → 1.4.123

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (11) hide show
  1. package/dist/src/UtilsRDS.d.ts +28 -22
  2. package/dist/src/UtilsRDS.js +95 -67
  3. package/dist/src/f.js +48 -5
  4. package/dist/src/index.js +1 -1
  5. package/dist/src/types/GlobalConfig.d.ts +3 -0
  6. package/dist/src/types/GlobalConfig.js +9 -0
  7. package/package.json +3 -1
  8. package/src/UtilsRDS.ts +133 -71
  9. package/src/f.ts +50 -5
  10. package/src/index.ts +1 -1
  11. package/src/types/GlobalConfig.ts +9 -0
package/dist/src/UtilsRDS.d.ts CHANGED
@@ -1,34 +1,40 @@
1
- import { Field } from "@aws-sdk/client-rds-data";
1
+ import { Client } from "pg";
2
2
  export declare class UtilsRDS {
3
- private readonly rds;
3
+ private readonly postgres;
4
+ private readonly rds_data;
4
5
  private readonly rds_cluster_arn;
5
6
  private readonly rds_secret_arn;
6
- constructor(region: string, rds_cluster_arn: string, rds_secret_arn: string);
7
- exec(sql: string): Promise<Field[][]>;
8
- get(table_id: string, primary_key: Record<string, any>, attribute_names: string[], options?: {
9
- is_verbose?: boolean;
10
- }): Promise<{
11
- [k: string]: string | number | boolean | undefined;
12
- } | undefined>;
13
- batch_get(table_id: string, primary_key: Record<string, any[]>, attribute_names: string[], options?: {
14
- is_verbose?: boolean;
15
- }): Promise<{
16
- [k: string]: string | number | boolean | undefined;
17
- }[]>;
7
+ constructor(region: string, rds_cluster_arn: string, rds_secret_arn: string, rds_host: string, rds_username: string, rds_password: string);
8
+ connect(): Promise<Client>;
9
+ disconnect(): Promise<void>;
10
+ query(sql: string, options?: {
11
+ is_verbose?: boolean;
12
+ }): Promise<any[]>;
13
+ exec(sql: string, options?: {
14
+ is_verbose?: boolean;
15
+ }): Promise<any[]>;
16
+ get(table_id: string, primary_key: Record<string, any>, attribute_names?: string[], options?: {
17
+ is_verbose?: boolean;
18
+ }): Promise<Record<string, any> | undefined>;
19
+ batch_get(table_id: string, primary_key: Record<string, any[]>, attribute_names?: string[], options?: {
20
+ is_verbose?: boolean;
21
+ }): Promise<any[]>;
22
+ create(table_id: string, item: Record<string, any>, options?: {
23
+ is_verbose?: boolean;
24
+ }): Promise<any[] | undefined>;
25
+ batch_create(table_id: string, items: Record<string, any>[], options?: {
26
+ is_verbose?: boolean;
27
+ }): Promise<any[] | undefined>;
18
28
  put(table_id: string, item: Record<string, any>, options?: {
19
29
  is_verbose?: boolean;
20
- }): Promise<{
21
- [k: string]: string | number | boolean | undefined;
22
- }[] | undefined>;
30
+ }): Promise<any[] | undefined>;
23
31
  batch_put(table_id: string, items: Record<string, any>[], options?: {
24
32
  is_verbose?: boolean;
25
- }): Promise<{
26
- [k: string]: string | number | boolean | undefined;
27
- }[] | undefined>;
33
+ }): Promise<any[] | undefined>;
28
34
  delete(table_id: string, primary_key: Record<string, any>, options?: {
29
35
  is_verbose?: boolean;
30
- }): Promise<undefined>;
36
+ }): Promise<any[]>;
31
37
  batch_delete(table_id: string, primary_key: Record<string, any[]>, options?: {
32
38
  is_verbose?: boolean;
33
- }): Promise<void>;
39
+ }): Promise<any[]>;
34
40
  }
package/dist/src/UtilsRDS.js CHANGED
@@ -1,22 +1,5 @@
1
1
  import { RDSData } from "@aws-sdk/client-rds-data";
2
- function convert_rds_output(rds_output) {
3
- if (rds_output.isNull) {
4
- return undefined;
5
- }
6
- else if (rds_output.stringValue !== undefined) {
7
- return rds_output.stringValue;
8
- }
9
- else if (rds_output.longValue !== undefined) {
10
- return rds_output.longValue;
11
- }
12
- else if (rds_output.doubleValue !== undefined) {
13
- return rds_output.doubleValue;
14
- }
15
- else if (rds_output.booleanValue !== undefined) {
16
- return rds_output.booleanValue;
17
- }
18
- return undefined;
19
- }
2
+ import { Client } from "pg";
20
3
  function convert_rds_input(input) {
21
4
  if (input === undefined) {
22
5
  return "NULL";
@@ -30,58 +13,120 @@ function convert_rds_input(input) {
30
13
  else if (typeof input === "boolean") {
31
14
  return input.toString().toUpperCase();
32
15
  }
16
+ else if (typeof input === "object") {
17
+ if (input.expression !== undefined) {
18
+ return input.expression;
19
+ }
20
+ }
33
21
  return "NULL";
34
22
  }
35
23
  export class UtilsRDS {
36
- rds;
24
+ postgres;
25
+ rds_data;
37
26
  rds_cluster_arn;
38
27
  rds_secret_arn;
39
- constructor(region, rds_cluster_arn, rds_secret_arn) {
40
- this.rds = new RDSData({ region: region });
28
+ constructor(region, rds_cluster_arn, rds_secret_arn, rds_host, rds_username, rds_password) {
29
+ this.postgres = new Client({
30
+ host: rds_host,
31
+ user: rds_username,
32
+ password: rds_password,
33
+ database: "postgres",
34
+ port: 5432,
35
+ ssl: {
36
+ rejectUnauthorized: false
37
+ }
38
+ });
39
+ this.rds_data = new RDSData({ region: region });
41
40
  this.rds_cluster_arn = rds_cluster_arn,
42
41
  this.rds_secret_arn = rds_secret_arn;
43
42
  }
44
- async exec(sql) {
45
- const response = await this.rds.executeStatement({
46
- resourceArn: this.rds_cluster_arn,
47
- secretArn: this.rds_secret_arn,
48
- database: "postgres",
49
- sql: sql
50
- });
51
- return response.records || [];
43
+ async connect() {
44
+ return await this.postgres.connect();
52
45
  }
53
- async get(table_id, primary_key, attribute_names, options = {}) {
46
+ async disconnect() {
47
+ return await this.postgres.end();
48
+ }
49
+ async query(sql, options = {}) {
54
50
  const is_verbose = Boolean(options.is_verbose);
55
- const sql = "SELECT " + attribute_names.join(", ") + " FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, value]) => key_name + " = " + convert_rds_input(value)).join(" AND ");
56
51
  if (is_verbose) {
57
52
  console.log(sql);
58
53
  }
59
- const items = await this.exec(sql);
60
- const item = items[0];
61
- if (item === undefined) {
62
- return undefined;
54
+ const response_connect = await this.connect();
55
+ if (is_verbose) {
56
+ console.log(response_connect);
57
+ }
58
+ const response = await this.postgres.query(sql);
59
+ const response_disconnect = await this.disconnect();
60
+ if (is_verbose) {
61
+ console.log(response_disconnect);
63
62
  }
64
- return Object.fromEntries(attribute_names.map((attribute_name, i) => [attribute_name, convert_rds_output(item[i])]));
63
+ return response.rows;
65
64
  }
66
- async batch_get(table_id, primary_key, attribute_names, options = {}) {
65
+ async exec(sql, options = {}) {
67
66
  const is_verbose = Boolean(options.is_verbose);
68
- const sql = "SELECT " + attribute_names.join(", ") + " FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, values]) => key_name + " IN " + "(" + values.map(value => convert_rds_input(value)).join(", ") + ")").join(" AND ");
69
67
  if (is_verbose) {
70
68
  console.log(sql);
71
69
  }
72
- const items = await this.exec(sql);
73
- return items.map(item => Object.fromEntries(attribute_names.map((attribute_name, i) => [attribute_name, convert_rds_output(item[i])])));
70
+ const items = sql.length <= 60000 ?
71
+ await this.rds_data.executeStatement({
72
+ resourceArn: this.rds_cluster_arn,
73
+ secretArn: this.rds_secret_arn,
74
+ database: "postgres",
75
+ sql: sql,
76
+ formatRecordsAs: "JSON"
77
+ })
78
+ .then(response => JSON.parse(response.formattedRecords || "[]"))
79
+ :
80
+ await this.query(sql, options);
81
+ if (!Array.isArray(items)) {
82
+ if (is_verbose) {
83
+ console.log("Bad items:", items);
84
+ }
85
+ return [];
86
+ }
87
+ return items;
88
+ }
89
+ async get(table_id, primary_key, attribute_names, options = {}) {
90
+ const sql = "SELECT " + (attribute_names === undefined ? "*" : attribute_names.join(", ")) + " FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, value]) => key_name + " = " + convert_rds_input(value)).join(" AND ");
91
+ const items = await this.exec(sql, options);
92
+ const item = items[0];
93
+ return item;
94
+ }
95
+ async batch_get(table_id, primary_key, attribute_names, options = {}) {
96
+ const sql = "SELECT " + (attribute_names === undefined ? "*" : attribute_names.join(", ")) + " FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, values]) => key_name + " IN " + "(" + values.map(value => convert_rds_input(value)).join(", ") + ")").join(" AND ");
97
+ const items = await this.exec(sql, options);
98
+ return items;
99
+ }
100
+ async create(table_id, item, options = {}) {
101
+ const attribute_names = Object.keys(item).sort();
102
+ const sql = "INSERT INTO " + table_id + " (" + attribute_names.join(", ") + ")" + " VALUES " + "(" + attribute_names.map(attribute_name => convert_rds_input(item[attribute_name])).join(", ") + ")" + " ON CONFLICT ON CONSTRAINT " + table_id + "_pkey IGNORE;";
103
+ try {
104
+ const items = await this.exec(sql, options);
105
+ return items;
106
+ }
107
+ catch (error) {
108
+ console.log(error.stack);
109
+ return;
110
+ }
111
+ }
112
+ async batch_create(table_id, items, options = {}) {
113
+ const attribute_names = Array.from(new Set(items.map(item => Object.keys(item)).flat())).sort();
114
+ const sql = "INSERT INTO " + table_id + " (" + attribute_names.join(", ") + ")" + " VALUES " + items.map(item => "(" + attribute_names.map(attribute_name => convert_rds_input(item[attribute_name])).join(", ") + ")").join(", ") + " ON CONFLICT ON CONSTRAINT " + table_id + "_pkey IGNORE;";
115
+ try {
116
+ const items = await this.exec(sql, options);
117
+ return items;
118
+ }
119
+ catch (error) {
120
+ console.log(error.stack);
121
+ return;
122
+ }
74
123
  }
75
124
  async put(table_id, item, options = {}) {
76
- const is_verbose = Boolean(options.is_verbose);
77
125
  const attribute_names = Object.keys(item).sort();
78
126
  const sql = "INSERT INTO " + table_id + " (" + attribute_names.join(", ") + ")" + " VALUES " + "(" + attribute_names.map(attribute_name => convert_rds_input(item[attribute_name])).join(", ") + ")" + " ON CONFLICT ON CONSTRAINT " + table_id + "_pkey DO UPDATE SET " + attribute_names.map(attribute_name => attribute_name + " = " + "EXCLUDED." + attribute_name).join(", ") + ";";
79
- if (is_verbose) {
80
- console.log(sql);
81
- }
82
127
  try {
83
- const items = await this.exec(sql);
84
- return items.map(item => Object.fromEntries(attribute_names.map((attribute_name, i) => [attribute_name, convert_rds_output(item[i])])));
128
+ const items = await this.exec(sql, options);
129
+ return items;
85
130
  }
86
131
  catch (error) {
87
132
  console.log(error.stack);
@@ -89,15 +134,11 @@ export class UtilsRDS {
89
134
  }
90
135
  }
91
136
  async batch_put(table_id, items, options = {}) {
92
- const is_verbose = Boolean(options.is_verbose);
93
137
  const attribute_names = Array.from(new Set(items.map(item => Object.keys(item)).flat())).sort();
94
138
  const sql = "INSERT INTO " + table_id + " (" + attribute_names.join(", ") + ")" + " VALUES " + items.map(item => "(" + attribute_names.map(attribute_name => convert_rds_input(item[attribute_name])).join(", ") + ")").join(", ") + " ON CONFLICT ON CONSTRAINT " + table_id + "_pkey DO UPDATE SET " + attribute_names.map(attribute_name => attribute_name + " = " + "EXCLUDED." + attribute_name).join(", ") + ";";
95
- if (is_verbose) {
96
- console.log(sql);
97
- }
98
139
  try {
99
- const items = await this.exec(sql);
100
- return items.map(item => Object.fromEntries(attribute_names.map((attribute_name, i) => [attribute_name, convert_rds_output(item[i])])));
140
+ const items = await this.exec(sql, options);
141
+ return items;
101
142
  }
102
143
  catch (error) {
103
144
  console.log(error.stack);
@@ -105,24 +146,11 @@ export class UtilsRDS {
105
146
  }
106
147
  }
107
148
  async delete(table_id, primary_key, options = {}) {
108
- const is_verbose = Boolean(options.is_verbose);
109
149
  const sql = "DELETE FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, value]) => key_name + " = " + convert_rds_input(value)).join(" AND ");
110
- if (is_verbose) {
111
- console.log(sql);
112
- }
113
- const items = await this.exec(sql);
114
- const item = items[0];
115
- if (item === undefined) {
116
- return undefined;
117
- }
118
- return undefined;
150
+ return await this.exec(sql, options);
119
151
  }
120
152
  async batch_delete(table_id, primary_key, options = {}) {
121
- const is_verbose = Boolean(options.is_verbose);
122
153
  const sql = "DELETE FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, values]) => key_name + " IN " + "(" + values.map(value => convert_rds_input(value)).join(", ") + ")").join(" AND ");
123
- if (is_verbose) {
124
- console.log(sql);
125
- }
126
- await this.exec(sql);
154
+ return await this.exec(sql, options);
127
155
  }
128
156
  }
package/dist/src/f.js CHANGED
@@ -15,12 +15,55 @@ const config = {
15
15
  };
16
16
  console.log(config);
17
17
  const utils = new TriangleUtils(config);
18
- const federal_censuses = await utils.dynamodb.scan("federal_censuses");
19
- federal_censuses.sort((a, b) => a.federal_census_id - b.federal_census_id);
18
+ // const federal_censuses = await utils.dynamodb.scan("federal_censuses")
19
+ // federal_censuses.sort((a, b) => a.federal_census_id - b.federal_census_id)
20
20
  // console.log(JSON.stringify(response, undefined, 4))
21
21
  // await utils.rds.batch_delete("federal_censuses", {
22
22
  // federal_census_id : federal_censuses.map(federal_census => federal_census.federal_census_id)
23
23
  // }, { is_verbose : true })
24
- await utils.rds.batch_put("federal_censuses", federal_censuses);
25
- const rds_federal_censuses = await utils.rds.batch_get("federal_censuses", { federal_census_id: federal_censuses.map(federal_census => federal_census.federal_census_id) }, ["federal_census_id", "is_decennial"], { is_verbose: true });
26
- console.log(rds_federal_censuses);
24
+ // await utils.rds.batch_put("federal_censuses", federal_censuses)
25
+ // const rds_federal_censuses = await utils.rds.batch_get("federal_censuses", { federal_census_id : federal_censuses.map(federal_census => federal_census.federal_census_id) }, ["federal_census_id", "is_decennial"], { is_verbose : true })
26
+ // console.log(rds_federal_censuses)
27
+ // const voters = await utils.dynamodb.scan("voters", { compile : false, max_num_items : 100 })
28
+ // for (const voter of voters) {
29
+ // const new_lal_file_voter = {
30
+ // lal_file_voter_id : voter.state_id + "-2026-08-15" + "|" + voter.voter_id,
31
+ // lal_file_id : voter.state_id + "-2026-08-15",
32
+ // lal_voter_id : voter.voter_id,
33
+ // voter_id : voter.state_voter_id,
34
+ // state_id : voter.state_id,
35
+ // address_residence_latitude : voter.address_residence_latitude,
36
+ // address_residence_longitude : voter.address_residence_longitude,
37
+ // address_residence_coordinates : "ST_GeomFromText('POINT(" + voter.address_residence_latitude + " " + voter.address_residence_longitude + ")', 4326)"
38
+ // }
39
+ // console.log(new_lal_file_voter)
40
+ // await utils.rds.put("lal_file_voters", new_lal_file_voter)
41
+ // }
42
+ // const federal_census_county_id = "2026-04021"
43
+ // const federal_census_county = await utils.rds.get("federal_census_counties", { federal_census_county_id : federal_census_county_id }, ["*"])
44
+ // console.log(federal_census_county)
45
+ // console.log(await utils.rds.exec("SELECT * FROM federal_censuses"))
46
+ // const federal_census_county = await utils.dynamodb.get("federal_census_counties", { federal_census_county_id : federal_census_county_id })
47
+ // const s3_id = config.s3_triangle + "/federal_census/2026/counties/" + federal_census_county_id + ".json"
48
+ // console.log(s3_id)
49
+ // const feature = await utils.s3.get(s3_id)
50
+ // // console.log(feature)
51
+ // if (feature !== undefined && federal_census_county !== undefined) {
52
+ // const geometry = "(" + JSON.parse(feature).geometry.coordinates[0].map((polygon : any) => "(" + polygon.map((vertex : any) => vertex[1] + " " + vertex[0]).join(", ") + ")").join(", ") + ")"
53
+ // const ewkt = "'MULTIPOLYGON(" + geometry + ")'"
54
+ // // const response = await utils.rds.exec("SELECT ST_AsText(address_residence_coordinates) FROM lal_file_voters", { is_verbose : true })
55
+ // // const response = await utils.rds.exec("SELECT lal_file_voter_id FROM lal_file_voters WHERE ST_Contains(ST_GEOMFROMTEXT(" + ewkt + ", 4326), lal_file_voters.address_residence_coordinates)", { is_verbose : true })
56
+ // const response = await utils.rds.put("federal_census_counties", {
57
+ // federal_census_county_id : federal_census_county.federal_census_county_id,
58
+ // federal_census_id : federal_census_county.federal_census_id,
59
+ // county_id : federal_census_county.county_id,
60
+ // state_id : federal_census_county.state_id,
61
+ // county_type_id : federal_census_county.county_type_id,
62
+ // function_type_id : federal_census_county.function_type_id,
63
+ // name : federal_census_county.name,
64
+ // geometry : {
65
+ // expression : "ST_GEOMFROMTEXT(" + ewkt + ", 4326)"
66
+ // }
67
+ // }, { is_verbose : true })
68
+ // console.log(response)
69
+ // }
package/dist/src/index.js CHANGED
@@ -27,7 +27,7 @@ export class TriangleUtils extends UtilsMisc {
27
27
  constructor(config) {
28
28
  super(config);
29
29
  this.dynamodb = new UtilsDynamoDB(config.region);
30
- this.rds = new UtilsRDS(config.region, config.rds_cluster_arn, config.rds_secret_arn);
30
+ this.rds = new UtilsRDS(config.region, config.rds_cluster_arn, config.rds_secret_arn, config.rds_host, config.rds_username, config.rds_password);
31
31
  this.s3 = new UtilsS3(config.region);
32
32
  this.s3vectors = new UtilsS3Vectors(config.region);
33
33
  this.bedrock = new UtilsBedrock(config.region);
package/dist/src/types/GlobalConfig.d.ts CHANGED
@@ -10,6 +10,9 @@ export declare class GlobalConfig {
10
10
  readonly s3_triage: string;
11
11
  readonly rds_cluster_arn: string;
12
12
  readonly rds_secret_arn: string;
13
+ readonly rds_host: string;
14
+ readonly rds_username: string;
15
+ readonly rds_password: string;
13
16
  readonly super_federal_gov_api_keys: string;
14
17
  readonly federal_gov_api_keys: string;
15
18
  readonly federal_census_api_key: string;
package/dist/src/types/GlobalConfig.js CHANGED
@@ -10,6 +10,9 @@ export class GlobalConfig {
10
10
  s3_triage;
11
11
  rds_cluster_arn;
12
12
  rds_secret_arn;
13
+ rds_host;
14
+ rds_username;
15
+ rds_password;
13
16
  super_federal_gov_api_keys;
14
17
  federal_gov_api_keys;
15
18
  federal_census_api_key;
@@ -33,6 +36,9 @@ export class GlobalConfig {
33
36
  this.s3_triage = global_config.s3_triage;
34
37
  this.rds_cluster_arn = global_config.rds_cluster_arn;
35
38
  this.rds_secret_arn = global_config.rds_secret_arn;
39
+ this.rds_host = global_config.rds_host;
40
+ this.rds_username = global_config.rds_username;
41
+ this.rds_password = global_config.rds_password;
36
42
  this.super_federal_gov_api_keys = global_config.super_federal_gov_api_keys;
37
43
  this.federal_gov_api_keys = global_config.federal_gov_api_keys;
38
44
  this.federal_census_api_key = global_config.federal_census_api_key;
@@ -55,6 +61,9 @@ export class GlobalConfig {
55
61
  typeof global_config.s3_triage === "string" &&
56
62
  typeof global_config.rds_cluster_arn === "string" &&
57
63
  typeof global_config.rds_secret_arn === "string" &&
64
+ typeof global_config.rds_host === "string" &&
65
+ typeof global_config.rds_username === "string" &&
66
+ typeof global_config.rds_password === "string" &&
58
67
  typeof global_config.super_federal_gov_api_keys === "string" &&
59
68
  typeof global_config.federal_gov_api_keys === "string" &&
60
69
  typeof global_config.federal_census_api_key === "string" &&
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "triangle-utils",
3
- "version": "1.4.122",
3
+ "version": "1.4.123",
4
4
  "main": "dist/src/index.js",
5
5
  "types": "dist/src/index.d.ts",
6
6
  "directories": {
@@ -31,9 +31,11 @@
31
31
  "@types/jsdom": "^28.0.1",
32
32
  "@types/node": "^25.2.3",
33
33
  "@types/nodemailer": "^7.0.9",
34
+ "@types/pg": "^8.21.0",
34
35
  "@xdevplatform/xdk": "^0.5.0",
35
36
  "googleapis": "^170.0.0",
36
37
  "jsdom": "^29.0.1",
38
+ "pg": "^8.23.0",
37
39
  "scrapingbee": "^1.8.2"
38
40
  },
39
41
  "devDependencies": {
package/src/UtilsRDS.ts CHANGED
@@ -1,21 +1,7 @@
1
1
  import { Field, RDSData } from "@aws-sdk/client-rds-data"
2
+ import { Client } from "pg"
2
3
 
3
- function convert_rds_output(rds_output : Field) : string | boolean | number | undefined {
4
- if (rds_output.isNull) {
5
- return undefined
6
- } else if (rds_output.stringValue !== undefined) {
7
- return rds_output.stringValue
8
- } else if (rds_output.longValue !== undefined) {
9
- return rds_output.longValue
10
- } else if (rds_output.doubleValue !== undefined) {
11
- return rds_output.doubleValue
12
- } else if (rds_output.booleanValue !== undefined) {
13
- return rds_output.booleanValue
14
- }
15
- return undefined
16
- }
17
-
18
- function convert_rds_input(input : string | boolean | number | undefined) : string {
4
+ function convert_rds_input(input : string | boolean | number | Record<string, any> | undefined) : string {
19
5
  if (input === undefined) {
20
6
  return "NULL"
21
7
  } else if (typeof input === "string") {
@@ -24,68 +10,165 @@ function convert_rds_input(input : string | boolean | number | undefined) : stri
24
10
  return input.toString()
25
11
  } else if (typeof input === "boolean") {
26
12
  return input.toString().toUpperCase()
13
+ } else if (typeof input === "object") {
14
+ if (input.expression !== undefined) {
15
+ return input.expression
16
+ }
27
17
  }
28
18
  return "NULL"
29
19
  }
30
20
 
31
21
  export class UtilsRDS {
32
22
 
33
- private readonly rds : RDSData
23
+ private readonly postgres : Client
24
+ private readonly rds_data : RDSData
34
25
  private readonly rds_cluster_arn : string
35
26
  private readonly rds_secret_arn : string
36
27
 
37
- constructor(region : string, rds_cluster_arn : string, rds_secret_arn : string) {
38
- this.rds = new RDSData({ region : region })
28
+ constructor(
29
+ region : string,
30
+ rds_cluster_arn : string,
31
+ rds_secret_arn : string,
32
+ rds_host : string,
33
+ rds_username : string,
34
+ rds_password : string
35
+ ) {
36
+ this.postgres = new Client({
37
+ host : rds_host,
38
+ user : rds_username,
39
+ password : rds_password,
40
+ database : "postgres",
41
+ port : 5432,
42
+ ssl : {
43
+ rejectUnauthorized: false
44
+ }
45
+ })
46
+ this.rds_data = new RDSData({ region : region })
39
47
  this.rds_cluster_arn = rds_cluster_arn,
40
48
  this.rds_secret_arn = rds_secret_arn
41
49
  }
42
50
 
43
- async exec(sql : string) {
44
- const response = await this.rds.executeStatement({
45
- resourceArn : this.rds_cluster_arn,
46
- secretArn : this.rds_secret_arn,
47
- database : "postgres",
48
- sql : sql
49
- })
50
- return response.records || []
51
+ async connect() {
52
+ return await this.postgres.connect()
51
53
  }
52
54
 
53
- async get(
54
- table_id : string,
55
- primary_key : Record<string, any>,
56
- attribute_names : string[],
55
+ async disconnect() {
56
+ return await this.postgres.end()
57
+ }
58
+
59
+ async query(
60
+ sql : string,
57
61
  options : {
58
62
  is_verbose? : boolean
59
63
  } = {}
60
64
  ) {
61
65
  const is_verbose = Boolean(options.is_verbose)
62
- const sql = "SELECT " + attribute_names.join(", ") + " FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, value]) => key_name + " = " + convert_rds_input(value)).join(" AND ")
63
66
  if (is_verbose) {
64
67
  console.log(sql)
65
68
  }
66
- const items = await this.exec(sql)
67
- const item = items[0]
68
- if (item === undefined) {
69
- return undefined
69
+ const response_connect = await this.connect()
70
+ if (is_verbose) {
71
+ console.log(response_connect)
72
+ }
73
+ const response = await this.postgres.query(sql)
74
+ const response_disconnect = await this.disconnect()
75
+ if (is_verbose) {
76
+ console.log(response_disconnect)
70
77
  }
71
- return Object.fromEntries(attribute_names.map((attribute_name, i) => [attribute_name, convert_rds_output(item[i])]))
78
+ return response.rows
72
79
  }
73
80
 
74
- async batch_get(
75
- table_id : string,
76
- primary_key : Record<string, any[]>,
77
- attribute_names : string[],
81
+ async exec(
82
+ sql : string,
78
83
  options : {
79
84
  is_verbose? : boolean
80
85
  } = {}
81
86
  ) {
82
87
  const is_verbose = Boolean(options.is_verbose)
83
- const sql = "SELECT " + attribute_names.join(", ") + " FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, values]) => key_name + " IN " + "(" + values.map(value => convert_rds_input(value)).join(", ") + ")").join(" AND ")
84
88
  if (is_verbose) {
85
89
  console.log(sql)
86
90
  }
87
- const items = await this.exec(sql)
88
- return items.map(item => Object.fromEntries(attribute_names.map((attribute_name, i) => [attribute_name, convert_rds_output(item[i])])))
91
+ const items = sql.length <= 60000 ?
92
+ await this.rds_data.executeStatement({
93
+ resourceArn : this.rds_cluster_arn,
94
+ secretArn : this.rds_secret_arn,
95
+ database : "postgres",
96
+ sql : sql,
97
+ formatRecordsAs : "JSON"
98
+ })
99
+ .then(response => JSON.parse(response.formattedRecords || "[]"))
100
+ :
101
+ await this.query(sql, options)
102
+ if (!Array.isArray(items)) {
103
+ if (is_verbose) {
104
+ console.log("Bad items:", items)
105
+ }
106
+ return []
107
+ }
108
+ return items
109
+ }
110
+
111
+ async get(
112
+ table_id : string,
113
+ primary_key : Record<string, any>,
114
+ attribute_names? : string[],
115
+ options : {
116
+ is_verbose? : boolean
117
+ } = {}
118
+ ) {
119
+ const sql = "SELECT " + (attribute_names === undefined ? "*" : attribute_names.join(", ")) + " FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, value]) => key_name + " = " + convert_rds_input(value)).join(" AND ")
120
+ const items = await this.exec(sql, options)
121
+ const item : Record<string, any> | undefined = items[0]
122
+ return item
123
+ }
124
+
125
+ async batch_get(
126
+ table_id : string,
127
+ primary_key : Record<string, any[]>,
128
+ attribute_names? : string[],
129
+ options : {
130
+ is_verbose? : boolean
131
+ } = {}
132
+ ) {
133
+ const sql = "SELECT " + (attribute_names === undefined ? "*" : attribute_names.join(", ")) + " FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, values]) => key_name + " IN " + "(" + values.map(value => convert_rds_input(value)).join(", ") + ")").join(" AND ")
134
+ const items = await this.exec(sql, options)
135
+ return items
136
+ }
137
+
138
+ async create(
139
+ table_id : string,
140
+ item : Record<string, any>,
141
+ options : {
142
+ is_verbose? : boolean
143
+ } = {}
144
+ ) {
145
+ const attribute_names = Object.keys(item).sort()
146
+ const sql = "INSERT INTO " + table_id + " (" + attribute_names.join(", ") + ")" + " VALUES " + "(" + attribute_names.map(attribute_name => convert_rds_input(item[attribute_name])).join(", ") + ")" + " ON CONFLICT ON CONSTRAINT " + table_id + "_pkey IGNORE;"
147
+ try {
148
+ const items = await this.exec(sql, options)
149
+ return items
150
+ } catch (error : any) {
151
+ console.log(error.stack)
152
+ return
153
+ }
154
+ }
155
+
156
+ async batch_create(
157
+ table_id : string,
158
+ items : Record<string, any>[],
159
+ options : {
160
+ is_verbose? : boolean
161
+ } = {}
162
+ ) {
163
+ const attribute_names = Array.from(new Set(items.map(item => Object.keys(item)).flat())).sort()
164
+ const sql = "INSERT INTO " + table_id + " (" + attribute_names.join(", ") + ")" + " VALUES " + items.map(item => "(" + attribute_names.map(attribute_name => convert_rds_input(item[attribute_name])).join(", ") + ")").join(", ") + " ON CONFLICT ON CONSTRAINT " + table_id + "_pkey IGNORE;"
165
+ try {
166
+ const items = await this.exec(sql, options)
167
+ return items
168
+ } catch (error : any) {
169
+ console.log(error.stack)
170
+ return
171
+ }
89
172
  }
90
173
 
91
174
  async put(
@@ -95,15 +178,11 @@ export class UtilsRDS {
95
178
  is_verbose? : boolean
96
179
  } = {}
97
180
  ) {
98
- const is_verbose = Boolean(options.is_verbose)
99
181
  const attribute_names = Object.keys(item).sort()
100
182
  const sql = "INSERT INTO " + table_id + " (" + attribute_names.join(", ") + ")" + " VALUES " + "(" + attribute_names.map(attribute_name => convert_rds_input(item[attribute_name])).join(", ") + ")" + " ON CONFLICT ON CONSTRAINT " + table_id + "_pkey DO UPDATE SET " + attribute_names.map(attribute_name => attribute_name + " = " + "EXCLUDED." + attribute_name).join(", ") + ";"
101
- if (is_verbose) {
102
- console.log(sql)
103
- }
104
183
  try {
105
- const items = await this.exec(sql)
106
- return items.map(item => Object.fromEntries(attribute_names.map((attribute_name, i) => [attribute_name, convert_rds_output(item[i])])))
184
+ const items = await this.exec(sql, options)
185
+ return items
107
186
  } catch (error : any) {
108
187
  console.log(error.stack)
109
188
  return
@@ -117,15 +196,11 @@ export class UtilsRDS {
117
196
  is_verbose? : boolean
118
197
  } = {}
119
198
  ) {
120
- const is_verbose = Boolean(options.is_verbose)
121
199
  const attribute_names = Array.from(new Set(items.map(item => Object.keys(item)).flat())).sort()
122
200
  const sql = "INSERT INTO " + table_id + " (" + attribute_names.join(", ") + ")" + " VALUES " + items.map(item => "(" + attribute_names.map(attribute_name => convert_rds_input(item[attribute_name])).join(", ") + ")").join(", ") + " ON CONFLICT ON CONSTRAINT " + table_id + "_pkey DO UPDATE SET " + attribute_names.map(attribute_name => attribute_name + " = " + "EXCLUDED." + attribute_name).join(", ") + ";"
123
- if (is_verbose) {
124
- console.log(sql)
125
- }
126
201
  try {
127
- const items = await this.exec(sql)
128
- return items.map(item => Object.fromEntries(attribute_names.map((attribute_name, i) => [attribute_name, convert_rds_output(item[i])])))
202
+ const items = await this.exec(sql, options)
203
+ return items
129
204
  } catch (error : any) {
130
205
  console.log(error.stack)
131
206
  return
@@ -139,17 +214,8 @@ export class UtilsRDS {
139
214
  is_verbose? : boolean
140
215
  } = {}
141
216
  ) {
142
- const is_verbose = Boolean(options.is_verbose)
143
217
  const sql = "DELETE FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, value]) => key_name + " = " + convert_rds_input(value)).join(" AND ")
144
- if (is_verbose) {
145
- console.log(sql)
146
- }
147
- const items = await this.exec(sql)
148
- const item = items[0]
149
- if (item === undefined) {
150
- return undefined
151
- }
152
- return undefined
218
+ return await this.exec(sql, options)
153
219
  }
154
220
 
155
221
  async batch_delete(
@@ -159,12 +225,8 @@ export class UtilsRDS {
159
225
  is_verbose? : boolean
160
226
  } = {}
161
227
  ) {
162
- const is_verbose = Boolean(options.is_verbose)
163
228
  const sql = "DELETE FROM " + table_id + " WHERE " + Object.entries(primary_key).map(([key_name, values]) => key_name + " IN " + "(" + values.map(value => convert_rds_input(value)).join(", ") + ")").join(" AND ")
164
- if (is_verbose) {
165
- console.log(sql)
166
- }
167
- await this.exec(sql)
229
+ return await this.exec(sql, options)
168
230
  }
169
231
 
170
232
 
package/src/f.ts CHANGED
@@ -24,8 +24,8 @@ console.log(config)
24
24
 
25
25
  const utils = new TriangleUtils(config)
26
26
 
27
- const federal_censuses = await utils.dynamodb.scan("federal_censuses")
28
- federal_censuses.sort((a, b) => a.federal_census_id - b.federal_census_id)
27
+ // const federal_censuses = await utils.dynamodb.scan("federal_censuses")
28
+ // federal_censuses.sort((a, b) => a.federal_census_id - b.federal_census_id)
29
29
 
30
30
  // console.log(JSON.stringify(response, undefined, 4))
31
31
 
@@ -33,8 +33,53 @@ federal_censuses.sort((a, b) => a.federal_census_id - b.federal_census_id)
33
33
  // federal_census_id : federal_censuses.map(federal_census => federal_census.federal_census_id)
34
34
  // }, { is_verbose : true })
35
35
 
36
- await utils.rds.batch_put("federal_censuses", federal_censuses)
36
+ // await utils.rds.batch_put("federal_censuses", federal_censuses)
37
37
 
38
- const rds_federal_censuses = await utils.rds.batch_get("federal_censuses", { federal_census_id : federal_censuses.map(federal_census => federal_census.federal_census_id) }, ["federal_census_id", "is_decennial"], { is_verbose : true })
38
+ // const rds_federal_censuses = await utils.rds.batch_get("federal_censuses", { federal_census_id : federal_censuses.map(federal_census => federal_census.federal_census_id) }, ["federal_census_id", "is_decennial"], { is_verbose : true })
39
39
 
40
- console.log(rds_federal_censuses)
40
+ // console.log(rds_federal_censuses)
41
+
42
+ // const voters = await utils.dynamodb.scan("voters", { compile : false, max_num_items : 100 })
43
+ // for (const voter of voters) {
44
+ // const new_lal_file_voter = {
45
+ // lal_file_voter_id : voter.state_id + "-2026-08-15" + "|" + voter.voter_id,
46
+ // lal_file_id : voter.state_id + "-2026-08-15",
47
+ // lal_voter_id : voter.voter_id,
48
+ // voter_id : voter.state_voter_id,
49
+ // state_id : voter.state_id,
50
+ // address_residence_latitude : voter.address_residence_latitude,
51
+ // address_residence_longitude : voter.address_residence_longitude,
52
+ // address_residence_coordinates : "ST_GeomFromText('POINT(" + voter.address_residence_latitude + " " + voter.address_residence_longitude + ")', 4326)"
53
+ // }
54
+ // console.log(new_lal_file_voter)
55
+ // await utils.rds.put("lal_file_voters", new_lal_file_voter)
56
+ // }
57
+ // const federal_census_county_id = "2026-04021"
58
+ // const federal_census_county = await utils.rds.get("federal_census_counties", { federal_census_county_id : federal_census_county_id }, ["*"])
59
+ // console.log(federal_census_county)
60
+ // console.log(await utils.rds.exec("SELECT * FROM federal_censuses"))
61
+ // const federal_census_county = await utils.dynamodb.get("federal_census_counties", { federal_census_county_id : federal_census_county_id })
62
+ // const s3_id = config.s3_triangle + "/federal_census/2026/counties/" + federal_census_county_id + ".json"
63
+ // console.log(s3_id)
64
+ // const feature = await utils.s3.get(s3_id)
65
+ // // console.log(feature)
66
+ // if (feature !== undefined && federal_census_county !== undefined) {
67
+ // const geometry = "(" + JSON.parse(feature).geometry.coordinates[0].map((polygon : any) => "(" + polygon.map((vertex : any) => vertex[1] + " " + vertex[0]).join(", ") + ")").join(", ") + ")"
68
+ // const ewkt = "'MULTIPOLYGON(" + geometry + ")'"
69
+
70
+ // // const response = await utils.rds.exec("SELECT ST_AsText(address_residence_coordinates) FROM lal_file_voters", { is_verbose : true })
71
+ // // const response = await utils.rds.exec("SELECT lal_file_voter_id FROM lal_file_voters WHERE ST_Contains(ST_GEOMFROMTEXT(" + ewkt + ", 4326), lal_file_voters.address_residence_coordinates)", { is_verbose : true })
72
+ // const response = await utils.rds.put("federal_census_counties", {
73
+ // federal_census_county_id : federal_census_county.federal_census_county_id,
74
+ // federal_census_id : federal_census_county.federal_census_id,
75
+ // county_id : federal_census_county.county_id,
76
+ // state_id : federal_census_county.state_id,
77
+ // county_type_id : federal_census_county.county_type_id,
78
+ // function_type_id : federal_census_county.function_type_id,
79
+ // name : federal_census_county.name,
80
+ // geometry : {
81
+ // expression : "ST_GEOMFROMTEXT(" + ewkt + ", 4326)"
82
+ // }
83
+ // }, { is_verbose : true })
84
+ // console.log(response)
85
+ // }
package/src/index.ts CHANGED
@@ -34,7 +34,7 @@ export class TriangleUtils extends UtilsMisc {
34
34
  constructor(config : ApplicationConfig & GlobalConfig) {
35
35
  super(config)
36
36
  this.dynamodb = new UtilsDynamoDB(config.region)
37
- this.rds = new UtilsRDS(config.region, config.rds_cluster_arn, config.rds_secret_arn)
37
+ this.rds = new UtilsRDS(config.region, config.rds_cluster_arn, config.rds_secret_arn, config.rds_host, config.rds_username, config.rds_password)
38
38
  this.s3 = new UtilsS3(config.region)
39
39
  this.s3vectors = new UtilsS3Vectors(config.region)
40
40
  this.bedrock = new UtilsBedrock(config.region)
package/src/types/GlobalConfig.ts CHANGED
@@ -10,6 +10,9 @@ export class GlobalConfig {
10
10
  readonly s3_triage : string
11
11
  readonly rds_cluster_arn : string
12
12
  readonly rds_secret_arn : string
13
+ readonly rds_host : string
14
+ readonly rds_username : string
15
+ readonly rds_password : string
13
16
  readonly super_federal_gov_api_keys : string
14
17
  readonly federal_gov_api_keys : string
15
18
  readonly federal_census_api_key : string
@@ -35,6 +38,9 @@ export class GlobalConfig {
35
38
  this.s3_triage = global_config.s3_triage
36
39
  this.rds_cluster_arn = global_config.rds_cluster_arn
37
40
  this.rds_secret_arn = global_config.rds_secret_arn
41
+ this.rds_host = global_config.rds_host
42
+ this.rds_username = global_config.rds_username
43
+ this.rds_password = global_config.rds_password
38
44
  this.super_federal_gov_api_keys = global_config.super_federal_gov_api_keys
39
45
  this.federal_gov_api_keys = global_config.federal_gov_api_keys
40
46
  this.federal_census_api_key = global_config.federal_census_api_key
@@ -59,6 +65,9 @@ export class GlobalConfig {
59
65
  typeof global_config.s3_triage === "string" &&
60
66
  typeof global_config.rds_cluster_arn === "string" &&
61
67
  typeof global_config.rds_secret_arn === "string" &&
68
+ typeof global_config.rds_host === "string" &&
69
+ typeof global_config.rds_username === "string" &&
70
+ typeof global_config.rds_password === "string" &&
62
71
  typeof global_config.super_federal_gov_api_keys === "string" &&
63
72
  typeof global_config.federal_gov_api_keys === "string" &&
64
73
  typeof global_config.federal_census_api_key === "string" &&