triangle-utils 1.4.15 → 1.4.17

package/dist/src/UtilsDynamoDB.d.ts

@@ -2,7 +2,7 @@ export declare class UtilsDynamoDB {
     private readonly dynamodb;
     constructor(region: string);
     get_table_key_names(table_name: string): Promise<string[] | undefined>;
-    scan(table: string, options?: {
+    scan(table_index_name: string, options?: {
         filters?: Record<string, any>;
         undefined_attribute_names?: string[];
         defined_attribute_names?: string[];

package/dist/src/UtilsDynamoDB.js

@@ -98,7 +98,9 @@ export class UtilsDynamoDB {
             .filter(table_key_name => table_key_name !== undefined);
         return table_key_names;
     }
-    async scan(table, options = {}) {
+    async scan(table_index_name, options = {}) {
+        const table_name = table_index_name.split(":")[0];
+        const index_name = table_index_name.split(":")[1];
         const filters = options.filters !== undefined ? options.filters : {};
         const undefined_attribute_names = options.undefined_attribute_names !== undefined ? options.undefined_attribute_names : [];
         const defined_attribute_names = options.defined_attribute_names !== undefined ? options.defined_attribute_names : [];
@@ -118,7 +120,8 @@ export class UtilsDynamoDB {
             ].join(" AND ");
             const projection_expression = attribute_names.map(attribute_name => "#" + attribute_name).join(", ");
             const request = {
-                TableName: table,
+                TableName: table_name,
+                IndexName: index_name,
                 ExpressionAttributeNames: Object.keys(expression_attribute_names).length > 0 ? expression_attribute_names : undefined,
                 ExpressionAttributeValues: Object.keys(expression_attribute_values).length > 0 ? expression_attribute_values : undefined,
                 FilterExpression: filter_expression.length > 0 ? filter_expression : undefined,

package/dist/src/UtilsMisc.d.ts

@@ -28,6 +28,10 @@ export declare class UtilsMisc {
         chunk_index: [number, number];
         chunk_text: string;
     }[];
+    static chunkify_pages(pages: string[], max_length?: number, overlap?: number): {
+        chunk_index: [number, number];
+        chunk_text: string;
+    }[];
     static instantiate<T>(constructor: new (x: any) => T, x: any): T | undefined;
     static instantiator<T>(constructor: new (x: any) => T): (x: any) => T | undefined;
 }

package/dist/src/UtilsMisc.js

@@ -153,6 +153,20 @@ export class UtilsMisc {
             chunk_text: text.substring(...chunk_index)
         }));
     }
+    static chunkify_pages(pages, max_length = 2500, overlap = 50) {
+        const chunks = [];
+        for (const page of pages) {
+            const page_chunks = UtilsMisc.chunkify(page, max_length, overlap);
+            const start_index = (chunks.length > 0 ? (chunks[chunks.length - 1].chunk_index[1]) : 0);
+            for (const page_chunk of page_chunks) {
+                chunks.push({
+                    chunk_index: [page_chunk.chunk_index[0] + start_index, page_chunk.chunk_index[1] + start_index],
+                    chunk_text: page_chunk.chunk_text + "\n\n"
+                });
+            }
+        }
+        return chunks;
+    }
     static instantiate(constructor, x) {
         try {
             const instance = new constructor(x);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "triangle-utils",
-    "version": "1.4.15",
+    "version": "1.4.17",
     "main": "dist/src/index.js",
     "types": "dist/src/index.d.ts",
     "directories": {

package/src/UtilsDynamoDB.ts

@@ -106,7 +106,7 @@ export class UtilsDynamoDB {
     }
 
     async scan(
-        table : string,
+        table_index_name : string,
         options : {
             filters? : Record<string, any>,
             undefined_attribute_names? : string[],
@@ -115,6 +115,8 @@ export class UtilsDynamoDB {
             concurrency? : number
         } = {}
     ) : Promise<Record<string, any>[]> {
+        const table_name : string = table_index_name.split(":")[0]
+        const index_name : string | undefined = table_index_name.split(":")[1]
         const filters = options.filters !== undefined ? options.filters : {}
         const undefined_attribute_names : string[] = options.undefined_attribute_names !== undefined ? options.undefined_attribute_names : []
         const defined_attribute_names : string[] = options.defined_attribute_names !== undefined ? options.defined_attribute_names : []
@@ -138,7 +140,8 @@ export class UtilsDynamoDB {
             ].join(" AND ")
             const projection_expression = attribute_names.map(attribute_name => "#" + attribute_name).join(", ")
             const request : ScanCommandInput = {
-                TableName : table,
+                TableName : table_name,
+                IndexName : index_name,
                 ExpressionAttributeNames : Object.keys(expression_attribute_names).length > 0 ? expression_attribute_names : undefined,
                 ExpressionAttributeValues : Object.keys(expression_attribute_values).length > 0 ? expression_attribute_values : undefined,
                 FilterExpression : filter_expression.length > 0 ? filter_expression : undefined,