triangle-utils 1.4.113 → 1.4.114
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/UtilsMisc.d.ts +1 -15
- package/dist/src/UtilsMisc.js +4 -64
- package/package.json +1 -1
- package/src/UtilsMisc.ts +5 -81
package/dist/src/UtilsMisc.d.ts
CHANGED
|
@@ -21,23 +21,9 @@ export declare class UtilsMisc {
|
|
|
21
21
|
static get_current_milliseconds(): number;
|
|
22
22
|
static get_random_string(length: number, encoding?: "binary" | "hex" | "base64"): string;
|
|
23
23
|
static sha256(input: string): Promise<string>;
|
|
24
|
+
static mini_sha256(input: string): Promise<string>;
|
|
24
25
|
static encrypt(secret_key: string, text: string): Encryption;
|
|
25
26
|
static decrypt(secret_key: string, encryption: Encryption): string;
|
|
26
27
|
static get_secret_hash(username: string, cognito_client_id: string, cognito_client_secret: string): string;
|
|
27
|
-
static get_chunk_indices(text_length: number, max_length?: number, overlap?: number): [number, number][];
|
|
28
|
-
static chunkify(text: string, max_length?: number, overlap?: number): {
|
|
29
|
-
chunk_index: [number, number];
|
|
30
|
-
chunk_text: string;
|
|
31
|
-
}[];
|
|
32
|
-
static chunkify_lined(text: string, max_length?: number, overlap?: number): {
|
|
33
|
-
chunk_index: [number, number];
|
|
34
|
-
chunk_text: string;
|
|
35
|
-
}[];
|
|
36
|
-
static chunkify_pages(pages: string[], max_length?: number, overlap?: number): {
|
|
37
|
-
chunk_index: [number, number];
|
|
38
|
-
chunk_text: string;
|
|
39
|
-
}[];
|
|
40
|
-
static instantiate<T>(constructor: new (x: any) => T, x: any): T | undefined;
|
|
41
|
-
static instantiator<T>(constructor: new (x: any) => T): (x: any) => T | undefined;
|
|
42
28
|
}
|
|
43
29
|
export {};
|
package/dist/src/UtilsMisc.js
CHANGED
|
@@ -86,6 +86,10 @@ export class UtilsMisc {
|
|
|
86
86
|
const hash = hash_array.map(b => b.toString(16).padStart(2, "0")).join("");
|
|
87
87
|
return hash;
|
|
88
88
|
}
|
|
89
|
+
static async mini_sha256(input) {
|
|
90
|
+
return await this.sha256(input)
|
|
91
|
+
.then(hash => hash.substring(0, 32));
|
|
92
|
+
}
|
|
89
93
|
static encrypt(secret_key, text) {
|
|
90
94
|
const iv = crypto.randomBytes(16);
|
|
91
95
|
const cipher = crypto.createCipheriv("aes-256-cbc", Buffer.from(secret_key, "hex"), iv);
|
|
@@ -110,68 +114,4 @@ export class UtilsMisc {
|
|
|
110
114
|
.digest("base64");
|
|
111
115
|
return secret_hash;
|
|
112
116
|
}
|
|
113
|
-
static get_chunk_indices(text_length, max_length = 2500, overlap = 50) {
|
|
114
|
-
const num_chunks = Math.ceil((text_length + overlap) / max_length);
|
|
115
|
-
const chunk_length = Math.ceil((text_length - overlap) / num_chunks + overlap);
|
|
116
|
-
const chunk_indices = [];
|
|
117
|
-
for (let i = 0; i < num_chunks; i++) {
|
|
118
|
-
chunk_indices.push([
|
|
119
|
-
(chunk_length - overlap) * i,
|
|
120
|
-
(chunk_length - overlap) * i + chunk_length
|
|
121
|
-
]);
|
|
122
|
-
}
|
|
123
|
-
return chunk_indices;
|
|
124
|
-
}
|
|
125
|
-
static chunkify(text, max_length = 2500, overlap = 50) {
|
|
126
|
-
const chunk_indices = this.get_chunk_indices(text.length, max_length, overlap);
|
|
127
|
-
const chunks = chunk_indices.map(chunk_index => ({
|
|
128
|
-
chunk_index: chunk_index,
|
|
129
|
-
chunk_text: text.substring(...chunk_index)
|
|
130
|
-
}));
|
|
131
|
-
return chunks;
|
|
132
|
-
}
|
|
133
|
-
static chunkify_lined(text, max_length = 2500, overlap = 50) {
|
|
134
|
-
const chunk_indices = this.get_chunk_indices(text.length, max_length, overlap);
|
|
135
|
-
const lined_chunk_indices = [];
|
|
136
|
-
for (const chunk_index of chunk_indices) {
|
|
137
|
-
const lined_chunk_index = [
|
|
138
|
-
text.lastIndexOf("\n", chunk_index[0]) + 1,
|
|
139
|
-
text.indexOf("\n", chunk_index[1]) + 1
|
|
140
|
-
];
|
|
141
|
-
if (lined_chunk_index[1] < lined_chunk_index[0]) {
|
|
142
|
-
lined_chunk_index[1] += text.length;
|
|
143
|
-
}
|
|
144
|
-
lined_chunk_indices.push(lined_chunk_index);
|
|
145
|
-
}
|
|
146
|
-
return lined_chunk_indices.map(chunk_index => ({
|
|
147
|
-
chunk_index: chunk_index,
|
|
148
|
-
chunk_text: text.substring(...chunk_index)
|
|
149
|
-
}));
|
|
150
|
-
}
|
|
151
|
-
static chunkify_pages(pages, max_length = 2500, overlap = 50) {
|
|
152
|
-
const chunks = [];
|
|
153
|
-
for (const page of pages) {
|
|
154
|
-
const page_chunks = UtilsMisc.chunkify(page, max_length, overlap);
|
|
155
|
-
const start_index = (chunks.length > 0 ? (chunks[chunks.length - 1].chunk_index[1]) : 0);
|
|
156
|
-
for (const page_chunk of page_chunks) {
|
|
157
|
-
chunks.push({
|
|
158
|
-
chunk_index: [page_chunk.chunk_index[0] + start_index, page_chunk.chunk_index[1] + start_index],
|
|
159
|
-
chunk_text: page_chunk.chunk_text + "\n\n"
|
|
160
|
-
});
|
|
161
|
-
}
|
|
162
|
-
}
|
|
163
|
-
return chunks;
|
|
164
|
-
}
|
|
165
|
-
static instantiate(constructor, x) {
|
|
166
|
-
try {
|
|
167
|
-
const instance = new constructor(x);
|
|
168
|
-
return instance;
|
|
169
|
-
}
|
|
170
|
-
catch (error) {
|
|
171
|
-
return undefined;
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
static instantiator(constructor) {
|
|
175
|
-
return (x) => UtilsMisc.instantiate(constructor, x);
|
|
176
|
-
}
|
|
177
117
|
}
|
package/package.json
CHANGED
package/src/UtilsMisc.ts
CHANGED
|
@@ -108,6 +108,11 @@ export class UtilsMisc {
|
|
|
108
108
|
return hash
|
|
109
109
|
}
|
|
110
110
|
|
|
111
|
+
static async mini_sha256(input : string) : Promise<string> {
|
|
112
|
+
return await this.sha256(input)
|
|
113
|
+
.then(hash => hash.substring(0, 32))
|
|
114
|
+
}
|
|
115
|
+
|
|
111
116
|
static encrypt(secret_key : string, text : string) : Encryption {
|
|
112
117
|
const iv = crypto.randomBytes(16)
|
|
113
118
|
const cipher = crypto.createCipheriv("aes-256-cbc", Buffer.from(secret_key, "hex"), iv)
|
|
@@ -135,85 +140,4 @@ export class UtilsMisc {
|
|
|
135
140
|
return secret_hash
|
|
136
141
|
}
|
|
137
142
|
|
|
138
|
-
static get_chunk_indices(text_length : number, max_length = 2500, overlap = 50) : [number, number][] {
|
|
139
|
-
const num_chunks = Math.ceil((text_length + overlap) / max_length)
|
|
140
|
-
const chunk_length = Math.ceil((text_length - overlap) / num_chunks + overlap)
|
|
141
|
-
const chunk_indices : [number, number][] = []
|
|
142
|
-
for (let i = 0; i < num_chunks; i++) {
|
|
143
|
-
chunk_indices.push([
|
|
144
|
-
(chunk_length - overlap) * i,
|
|
145
|
-
(chunk_length - overlap) * i + chunk_length
|
|
146
|
-
])
|
|
147
|
-
}
|
|
148
|
-
return chunk_indices
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
static chunkify(text : string, max_length = 2500, overlap = 50) : {
|
|
152
|
-
chunk_index : [number, number],
|
|
153
|
-
chunk_text : string
|
|
154
|
-
}[] {
|
|
155
|
-
const chunk_indices = this.get_chunk_indices(text.length, max_length, overlap)
|
|
156
|
-
const chunks = chunk_indices.map(chunk_index => ({
|
|
157
|
-
chunk_index : chunk_index,
|
|
158
|
-
chunk_text : text.substring(...chunk_index)
|
|
159
|
-
}))
|
|
160
|
-
return chunks
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
static chunkify_lined(text : string, max_length = 2500, overlap = 50) : {
|
|
164
|
-
chunk_index : [number, number],
|
|
165
|
-
chunk_text : string
|
|
166
|
-
}[] {
|
|
167
|
-
const chunk_indices : [number, number][] = this.get_chunk_indices(text.length, max_length, overlap)
|
|
168
|
-
const lined_chunk_indices : [number, number][] = []
|
|
169
|
-
for (const chunk_index of chunk_indices) {
|
|
170
|
-
const lined_chunk_index : [number, number] = [
|
|
171
|
-
text.lastIndexOf("\n", chunk_index[0]) + 1,
|
|
172
|
-
text.indexOf("\n", chunk_index[1]) + 1
|
|
173
|
-
]
|
|
174
|
-
if (lined_chunk_index[1] < lined_chunk_index[0]) {
|
|
175
|
-
lined_chunk_index[1] += text.length
|
|
176
|
-
}
|
|
177
|
-
lined_chunk_indices.push(lined_chunk_index)
|
|
178
|
-
}
|
|
179
|
-
return lined_chunk_indices.map(chunk_index => ({
|
|
180
|
-
chunk_index : chunk_index,
|
|
181
|
-
chunk_text : text.substring(...chunk_index)
|
|
182
|
-
}))
|
|
183
|
-
}
|
|
184
|
-
|
|
185
|
-
static chunkify_pages(pages : string[], max_length = 2500, overlap = 50) : {
|
|
186
|
-
chunk_index : [number, number],
|
|
187
|
-
chunk_text : string
|
|
188
|
-
}[] {
|
|
189
|
-
const chunks : {
|
|
190
|
-
chunk_index : [number, number],
|
|
191
|
-
chunk_text : string
|
|
192
|
-
}[] = []
|
|
193
|
-
for (const page of pages) {
|
|
194
|
-
const page_chunks = UtilsMisc.chunkify(page, max_length, overlap)
|
|
195
|
-
const start_index : number = (chunks.length > 0 ? (chunks[chunks.length - 1].chunk_index[1]) : 0)
|
|
196
|
-
for (const page_chunk of page_chunks) {
|
|
197
|
-
chunks.push({
|
|
198
|
-
chunk_index : [page_chunk.chunk_index[0] + start_index, page_chunk.chunk_index[1] + start_index],
|
|
199
|
-
chunk_text : page_chunk.chunk_text + "\n\n"
|
|
200
|
-
})
|
|
201
|
-
}
|
|
202
|
-
}
|
|
203
|
-
return chunks
|
|
204
|
-
}
|
|
205
|
-
|
|
206
|
-
static instantiate<T>(constructor : new (x : any) => T, x : any) : T | undefined {
|
|
207
|
-
try {
|
|
208
|
-
const instance = new constructor(x)
|
|
209
|
-
return instance
|
|
210
|
-
} catch (error) {
|
|
211
|
-
return undefined
|
|
212
|
-
}
|
|
213
|
-
}
|
|
214
|
-
|
|
215
|
-
static instantiator<T>(constructor : new (x : any) => T) : (x : any) => T | undefined {
|
|
216
|
-
return (x : any) => UtilsMisc.instantiate(constructor, x)
|
|
217
|
-
}
|
|
218
|
-
|
|
219
143
|
}
|