triangle-utils 1.4.162 → 1.4.164

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. package/dist/src/UtilsMisc.d.ts +8 -2
  2. package/dist/src/UtilsMisc.js +42 -10
  3. package/dist/src/index.d.ts +0 -1
  4. package/dist/src/index.js +0 -1
  5. package/package.json +2 -1
  6. package/src/UtilsMisc.ts +44 -10
  7. package/src/index.ts +0 -1
package/dist/src/UtilsMisc.d.ts CHANGED
@@ -1,4 +1,5 @@
1
1
  import { GlobalConfig } from "./types/GlobalConfig.js";
2
+ import { Readable } from "stream";
2
3
  interface Encryption {
3
4
  iv: string;
4
5
  ciphertext: string;
@@ -7,11 +8,11 @@ export declare class UtilsMisc {
7
8
  readonly config: GlobalConfig;
8
9
  constructor(config: GlobalConfig);
9
10
  safe_run<T>(f: () => Promise<T>): Promise<T | undefined>;
10
- iterate<T, U>(inputs: T[], f: (input: T, iterator_id: number, index: number) => Promise<U>, options?: {
11
+ iterate<T, U>(items: T[], f: (item: T, iterator_id: number, index: number) => Promise<U>, options?: {
11
12
  num_iterators?: number;
12
13
  verbosity?: number | boolean;
13
14
  }): Promise<(U | undefined)[]>;
14
- batch_iterate<T, U>(inputs: T[], f: (inputs: T[], iterator_id: number, index: number) => Promise<U>, options: {
15
+ batch_iterate<T, U>(items: T[], f: (items: T[], iterator_id: number, index: number) => Promise<U>, options: {
15
16
  batch_num_items?: number;
16
17
  num_iterators?: number;
17
18
  verbosity?: number | boolean;
@@ -25,5 +26,10 @@ export declare class UtilsMisc {
25
26
  static encrypt(secret_key: string, text: string): Encryption;
26
27
  static decrypt(secret_key: string, encryption: Encryption): string;
27
28
  static get_secret_hash(username: string, cognito_client_id: string, cognito_client_secret: string): string;
29
+ static batch_iterate_csv<T>(stream: Readable, f: (items: any[]) => Promise<void>, options: {
30
+ delimiter?: string;
31
+ batch_num_items: number;
32
+ max_num_items?: number;
33
+ }): Promise<void>;
28
34
  }
29
35
  export {};
package/dist/src/UtilsMisc.js CHANGED
@@ -1,4 +1,5 @@
1
1
  import * as crypto from "crypto";
2
+ import csv_parser from "csv-parser";
2
3
  export class UtilsMisc {
3
4
  config;
4
5
  constructor(config) {
@@ -21,21 +22,21 @@ export class UtilsMisc {
21
22
  }
22
23
  return undefined;
23
24
  }
24
- async iterate(inputs, f, options = {}) {
25
+ async iterate(items, f, options = {}) {
25
26
  const num_iterators = options.num_iterators || 1;
26
27
  const verbosity = typeof options.verbosity === "boolean" ? (options.verbosity === true ? 1 : 0) : (options.verbosity || 0);
27
28
  let index = 0;
28
29
  let iterators = [];
29
- const outputs = new Array(inputs.length).fill(undefined);
30
+ const outputs = new Array(items.length).fill(undefined);
30
31
  for (let iterator_id = 0; iterator_id < num_iterators; iterator_id++) {
31
32
  iterators.push((async () => {
32
- while (index < inputs.length) {
33
+ while (index < items.length) {
33
34
  const local_index = index;
34
35
  index += 1;
35
36
  if (verbosity !== undefined && (local_index % verbosity === 0)) {
36
- console.log(iterator_id + ":" + local_index + "/" + inputs.length);
37
+ console.log(iterator_id + ":" + local_index + "/" + items.length);
37
38
  }
38
- const output = await this.safe_run(() => f(inputs[local_index], iterator_id, local_index));
39
+ const output = await this.safe_run(() => f(items[local_index], iterator_id, local_index));
39
40
  outputs[local_index] = output;
40
41
  }
41
42
  })());
@@ -43,22 +44,22 @@ export class UtilsMisc {
43
44
  await Promise.all(iterators);
44
45
  return outputs;
45
46
  }
46
- async batch_iterate(inputs, f, options) {
47
+ async batch_iterate(items, f, options) {
47
48
  const batch_num_items = options.batch_num_items || 1;
48
49
  const num_iterators = options.num_iterators || 1;
49
50
  const verbosity = typeof options.verbosity === "boolean" ? (options.verbosity === true ? 1 : 0) : (options.verbosity || 0);
50
51
  let index = 0;
51
52
  let iterators = [];
52
- const outputs = new Array(inputs.length).fill(undefined);
53
+ const outputs = new Array(items.length).fill(undefined);
53
54
  for (let iterator_id = 0; iterator_id < num_iterators; iterator_id++) {
54
55
  iterators.push((async () => {
55
- while (index < inputs.length) {
56
+ while (index < items.length) {
56
57
  const local_index = index;
57
58
  index += batch_num_items;
58
59
  if (verbosity !== undefined && (local_index % verbosity === 0)) {
59
- console.log(iterator_id + ":" + local_index + "/" + inputs.length);
60
+ console.log(iterator_id + ":" + local_index + "/" + items.length);
60
61
  }
61
- const output = await this.safe_run(() => f(inputs.slice(local_index, local_index + batch_num_items), iterator_id, local_index));
62
+ const output = await this.safe_run(() => f(items.slice(local_index, local_index + batch_num_items), iterator_id, local_index));
62
63
  outputs[local_index] = output;
63
64
  }
64
65
  })());
@@ -114,4 +115,35 @@ export class UtilsMisc {
114
115
  .digest("base64");
115
116
  return secret_hash;
116
117
  }
118
+ static async batch_iterate_csv(stream, f, options) {
119
+ const delimiter = options.delimiter || ",";
120
+ const max_num_items = options.max_num_items;
121
+ const batch_num_items = options.batch_num_items;
122
+ let items = [];
123
+ let i = 0;
124
+ const csv_stream = stream.pipe(csv_parser({ separator: delimiter }));
125
+ await new Promise(resolve => {
126
+ csv_stream
127
+ .on("data", async (item) => {
128
+ i += 1;
129
+ if (max_num_items !== undefined && i > max_num_items) {
130
+ csv_stream.end();
131
+ return;
132
+ }
133
+ items.push(item);
134
+ if (i % batch_num_items === 0) {
135
+ csv_stream.pause();
136
+ console.log(i);
137
+ await f(await Promise.all(items).then(items => items.flat()));
138
+ items = [];
139
+ csv_stream.resume();
140
+ }
141
+ })
142
+ .on("end", async () => {
143
+ await f(await Promise.all(items).then(items => items.flat()));
144
+ items = [];
145
+ resolve();
146
+ });
147
+ });
148
+ }
117
149
  }
package/dist/src/index.d.ts CHANGED
@@ -32,7 +32,6 @@ export declare class TriangleUtils extends UtilsMisc {
32
32
  }
33
33
  export * from "./types/ApplicationConfig.js";
34
34
  export * from "./types/GlobalConfig.js";
35
- export * from "./UtilsMisc.js";
36
35
  export * from "./UtilsAnthropic.js";
37
36
  export * from "./UtilsDynamoDB.js";
38
37
  export * from "./UtilsRDS.js";
package/dist/src/index.js CHANGED
@@ -45,7 +45,6 @@ export class TriangleUtils extends UtilsMisc {
45
45
  }
46
46
  export * from "./types/ApplicationConfig.js";
47
47
  export * from "./types/GlobalConfig.js";
48
- export * from "./UtilsMisc.js";
49
48
  export * from "./UtilsAnthropic.js";
50
49
  export * from "./UtilsDynamoDB.js";
51
50
  export * from "./UtilsRDS.js";
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "triangle-utils",
3
- "version": "1.4.162",
3
+ "version": "1.4.164",
4
4
  "main": "dist/src/index.js",
5
5
  "types": "dist/src/index.d.ts",
6
6
  "directories": {
@@ -34,6 +34,7 @@
34
34
  "@types/nodemailer": "^7.0.9",
35
35
  "@types/pg": "^8.21.0",
36
36
  "@xdevplatform/xdk": "^0.5.0",
37
+ "csv-parser": "^3.2.1",
37
38
  "googleapis": "^170.0.0",
38
39
  "jsdom": "^29.0.1",
39
40
  "pg": "^8.23.0",
package/src/UtilsMisc.ts CHANGED
@@ -1,6 +1,8 @@
1
1
  import * as crypto from "crypto"
2
2
 
3
3
  import { GlobalConfig } from "./types/GlobalConfig"
4
+ import csv_parser from "csv-parser"
5
+ import { Readable } from "stream"
4
6
 
5
7
  interface Encryption {
6
8
  iv : string,
@@ -31,21 +33,21 @@ export class UtilsMisc {
31
33
  return undefined
32
34
  }
33
35
 
34
- async iterate<T, U>(inputs : T[], f : (input : T, iterator_id : number, index : number) => Promise<U>, options : { num_iterators? : number, verbosity? : number | boolean } = {}) {
36
+ async iterate<T, U>(items : T[], f : (item : T, iterator_id : number, index : number) => Promise<U>, options : { num_iterators? : number, verbosity? : number | boolean } = {}) {
35
37
  const num_iterators = options.num_iterators || 1
36
38
  const verbosity : number = typeof options.verbosity === "boolean" ? (options.verbosity === true ? 1 : 0) : (options.verbosity || 0)
37
39
  let index = 0
38
40
  let iterators = []
39
- const outputs : (U | undefined)[] = new Array(inputs.length).fill(undefined)
41
+ const outputs : (U | undefined)[] = new Array(items.length).fill(undefined)
40
42
  for (let iterator_id = 0; iterator_id < num_iterators; iterator_id++) {
41
43
  iterators.push((async () => {
42
- while (index < inputs.length) {
44
+ while (index < items.length) {
43
45
  const local_index = index
44
46
  index += 1
45
47
  if (verbosity !== undefined && (local_index % verbosity === 0)) {
46
- console.log(iterator_id + ":" + local_index + "/" + inputs.length)
48
+ console.log(iterator_id + ":" + local_index + "/" + items.length)
47
49
  }
48
- const output = await this.safe_run(() => f(inputs[local_index], iterator_id, local_index))
50
+ const output = await this.safe_run(() => f(items[local_index], iterator_id, local_index))
49
51
  outputs[local_index] = output
50
52
  }
51
53
  })())
@@ -54,23 +56,23 @@ export class UtilsMisc {
54
56
  return outputs
55
57
  }
56
58
 
57
- async batch_iterate<T, U>(inputs : T[], f : (inputs : T[], iterator_id : number, index : number) => Promise<U>,
59
+ async batch_iterate<T, U>(items : T[], f : (items : T[], iterator_id : number, index : number) => Promise<U>,
58
60
  options : { batch_num_items? : number, num_iterators? : number, verbosity? : number | boolean }) {
59
61
  const batch_num_items = options.batch_num_items || 1
60
62
  const num_iterators = options.num_iterators || 1
61
63
  const verbosity : number = typeof options.verbosity === "boolean" ? (options.verbosity === true ? 1 : 0) : (options.verbosity || 0)
62
64
  let index = 0
63
65
  let iterators = []
64
- const outputs : (U | undefined)[] = new Array(inputs.length).fill(undefined)
66
+ const outputs : (U | undefined)[] = new Array(items.length).fill(undefined)
65
67
  for (let iterator_id = 0; iterator_id < num_iterators; iterator_id++) {
66
68
  iterators.push((async () => {
67
- while (index < inputs.length) {
69
+ while (index < items.length) {
68
70
  const local_index = index
69
71
  index += batch_num_items
70
72
  if (verbosity !== undefined && (local_index % verbosity === 0)) {
71
- console.log(iterator_id + ":" + local_index + "/" + inputs.length)
73
+ console.log(iterator_id + ":" + local_index + "/" + items.length)
72
74
  }
73
- const output = await this.safe_run(() => f(inputs.slice(local_index, local_index + batch_num_items), iterator_id, local_index))
75
+ const output = await this.safe_run(() => f(items.slice(local_index, local_index + batch_num_items), iterator_id, local_index))
74
76
  outputs[local_index] = output
75
77
  }
76
78
  })())
@@ -140,4 +142,36 @@ export class UtilsMisc {
140
142
  return secret_hash
141
143
  }
142
144
 
145
+ static async batch_iterate_csv<T>(stream : Readable, f : (items : any[]) => Promise<void>, options : { delimiter? : string, batch_num_items : number, max_num_items? : number }) {
146
+ const delimiter = options.delimiter || ","
147
+ const max_num_items = options.max_num_items
148
+ const batch_num_items = options.batch_num_items
149
+ let items : Promise<T[]>[] = []
150
+ let i = 0
151
+ const csv_stream = stream.pipe(csv_parser({ separator: delimiter }))
152
+ await new Promise<void>(resolve => {
153
+ csv_stream
154
+ .on("data", async (item) => {
155
+ i += 1
156
+ if (max_num_items !== undefined && i > max_num_items) {
157
+ csv_stream.end()
158
+ return
159
+ }
160
+ items.push(item)
161
+ if (i % batch_num_items === 0) {
162
+ csv_stream.pause()
163
+ console.log(i)
164
+ await f(await Promise.all(items).then(items => items.flat()))
165
+ items = []
166
+ csv_stream.resume()
167
+ }
168
+ })
169
+ .on("end", async () => {
170
+ await f(await Promise.all(items).then(items => items.flat()))
171
+ items = []
172
+ resolve()
173
+ })
174
+ })
175
+ }
176
+
143
177
  }
package/src/index.ts CHANGED
@@ -53,7 +53,6 @@ export class TriangleUtils extends UtilsMisc {
53
53
 
54
54
  export * from "./types/ApplicationConfig"
55
55
  export * from "./types/GlobalConfig"
56
- export * from "./UtilsMisc"
57
56
  export * from "./UtilsAnthropic"
58
57
  export * from "./UtilsDynamoDB"
59
58
  export * from "./UtilsRDS"