triangle-utils 1.4.14 → 1.4.16

package/dist/src/UtilsMisc.d.ts

@@ -28,6 +28,10 @@ export declare class UtilsMisc {
         chunk_index: [number, number];
         chunk_text: string;
     }[];
+    static chunkify_pages(pages: string[], max_length?: number, overlap?: number): {
+        chunk_index: [number, number];
+        chunk_text: string;
+    }[];
     static instantiate<T>(constructor: new (x: any) => T, x: any): T | undefined;
     static instantiator<T>(constructor: new (x: any) => T): (x: any) => T | undefined;
 }

package/dist/src/UtilsMisc.js

@@ -153,6 +153,20 @@ export class UtilsMisc {
             chunk_text: text.substring(...chunk_index)
         }));
     }
+    static chunkify_pages(pages, max_length = 2500, overlap = 50) {
+        const chunks = [];
+        for (const page of pages) {
+            const page_chunks = UtilsMisc.chunkify(page, max_length, overlap);
+            const start_index = (chunks.length > 0 ? (chunks[chunks.length - 1].chunk_index[1]) : 0);
+            for (const page_chunk of page_chunks) {
+                chunks.push({
+                    chunk_index: [page_chunk.chunk_index[0] + start_index, page_chunk.chunk_index[1] + start_index],
+                    chunk_text: page_chunk.chunk_text + "\n\n"
+                });
+            }
+        }
+        return chunks;
+    }
     static instantiate(constructor, x) {
         try {
             const instance = new constructor(x);

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "triangle-utils",
-    "version": "1.4.14",
+    "version": "1.4.16",
     "main": "dist/src/index.js",
     "types": "dist/src/index.d.ts",
     "directories": {

package/src/UtilsMisc.ts

@@ -185,6 +185,27 @@ export class UtilsMisc {
         }))
     }
 
+    static chunkify_pages(pages : string[], max_length = 2500, overlap = 50) : {
+        chunk_index : [number, number],
+        chunk_text : string
+    }[] {
+        const chunks : {
+            chunk_index : [number, number],
+            chunk_text : string
+        }[] = []
+        for (const page of pages) {
+            const page_chunks = UtilsMisc.chunkify(page, max_length, overlap)
+            const start_index : number = (chunks.length > 0 ? (chunks[chunks.length - 1].chunk_index[1]) : 0)
+            for (const page_chunk of page_chunks) {
+                chunks.push({
+                    chunk_index : [page_chunk.chunk_index[0] + start_index, page_chunk.chunk_index[1] + start_index],
+                    chunk_text : page_chunk.chunk_text + "\n\n"
+                })
+            }
+        }
+        return chunks
+    }
+
     static instantiate<T>(constructor : new (x : any) => T, x : any) : T | undefined {
         try {
             const instance = new constructor(x)