triangle-utils 1.0.31 → 1.0.33

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "triangle-utils",
-    "version": "1.0.31",
+    "version": "1.0.33",
     "main": "Utils.js",
     "author": "",
     "license": "ISC",

package/test.js

@@ -1,6 +1,8 @@
-import Utils from "./index.js";
+import Utils from "./Utils.js";
 
 
-const utils = new Utils({})
+const utils = new Utils({ region : "us-east-2"})
 
-console.log(utils)
+const items = await utils.s3vectors.batch_get("doersnetwork", "candidate-document-chunks", ["yeeerp", "text|2025-12-10T21:34:37.429Z|0|1761", "text|2025-12-10T21:37:00.719Z|0|2413"])
+
+console.log(items)

package/utils/Utils_Misc.js

@@ -137,4 +137,17 @@ export default class Utils_Misc {
         }
         return election_id
     }
+
+    get_chunk_indices(text_length, max_length = 2500, overlap = 50) {
+        const num_chunks = Math.ceil((text_length + overlap) / max_length)
+        const chunk_length = Math.ceil((text_length - overlap) / num_chunks + overlap)
+        const chunk_indices = []
+        for (let i = 0; i < num_chunks; i++) {
+            chunk_indices.push([
+                (chunk_length - overlap) * i,
+                (chunk_length - overlap) * i + chunk_length
+            ])
+        }
+        return chunk_indices
+    }
 }

package/utils/Utils_S3Vectors.js

@@ -6,35 +6,97 @@ export default class Utils_S3Vectors {
         this.s3vectors = new S3Vectors({ region : config.region })
     }
 
-    async get_vector(vector_bucket, vector_index, vector_key) {
+    async scan(vector_bucket, vector_index) {
+        const vectors = []
+        let next_token = undefined
+        while (true) {
+            const response = (next_token === undefined) ? (await this.s3vectors.listVectors({
+                vectorBucketName : vector_bucket,
+                indexName : vector_index,
+                maxResults : 1000
+            })) : (await this.s3vectors.listVectors({
+                vectorBucketName : vector_bucket,
+                indexName : vector_index,
+                maxResults : 1000,
+                nextToken : next_token
+            }))
+            vectors.push(...response.vectors)
+            next_token = response.nextToken
+            if (next_token === undefined) {
+                return vectors
+            }
+        }
+    }
+
+    async get(vector_bucket, vector_index, vector_key) {
         const vector = await this.s3vectors.getVectors({
             vectorBucketName : vector_bucket,
             indexName : vector_index,
             keys : [vector_key],
             returnData : true,
             returnMetadata : true
-        }).then(response => response.vectors[0])
+        })
+            .then(response => response.vectors[0])
         if (vector === undefined) {
             return undefined
         }
-        return vector
+        return {
+            key : vector.key,
+            data : vector.data.float32,
+            metadata : vector.metadata
+        }
     }
 
-    async create_vector(vector_bucket, vector_index, vector_key, vector, metadata) {
+    async batch_get(vector_bucket, vector_index, vector_keys) {
+        const vectors_by_key = await this.s3vectors.getVectors({
+            vectorBucketName : vector_bucket,
+            indexName : vector_index,
+            keys : vector_keys,
+            returnData : true,
+            returnMetadata : true
+        })
+            .then(response => response.vectors)
+            .then(vectors => Object.fromEntries(vectors.map(vector => [
+                vector.key,
+                {
+                    key : vector.key,
+                    data : vector.data.float32,
+                    metadata : vector.metadata
+                }
+            ])))
+        const vectors = vector_keys.map(vector_key => vectors_by_key[vector_key])
+        return vectors
+    }
+
+    async create(vector_bucket, vector_index, vector) {
         await this.s3vectors.putVectors({
             vectorBucketName : vector_bucket,
             indexName : vector_index,
             vectors : [{
-                key : vector_key,
+                key : vector.key,
                 data : {
-                    float32 : vector
+                    float32 : vector.data
                 },
-                metadata : metadata
+                metadata : vector.metadata
             }]
         })
     }
 
-    async delete_vector(vector_bucket, vector_index, vector_key) {
+    async batch_create(vector_bucket, vector_index, vectors) {
+        await this.s3vectors.putVectors({
+            vectorBucketName : vector_bucket,
+            indexName : vector_index,
+            vectors : vectors.map(vector => ({
+                key : vector.key,
+                data : {
+                    float32 : vector.data
+                },
+                metadata : vector.metadata
+            }))
+        })
+    }
+
+    async delete(vector_bucket, vector_index, vector_key) {
         await this.s3vectors.deleteVectors({
             vectorBucketName : vector_bucket,
             indexName : vector_index,
@@ -42,12 +104,20 @@ export default class Utils_S3Vectors {
         })
     }
 
-    async query_vectors(vector_bucket, vector_index, vector, num_vectors, filters) {
+    async batch_delete(vector_bucket, vector_index, vector_keys) {
+        await this.s3vectors.deleteVectors({
+            vectorBucketName : vector_bucket,
+            indexName : vector_index,
+            keys : vector_keys
+        })
+    }
+
+    async query(vector_bucket, vector_index, data, num_vectors, filters) {
         const response = await this.s3vectors.queryVectors({
             vectorBucketName : vector_bucket,
             indexName : vector_index,
             queryVector : {
-                float32 : vector
+                float32 : data
             },
             topK : num_vectors, 
             returnDistance : true,
@@ -57,34 +127,4 @@ export default class Utils_S3Vectors {
         return response.vectors
     }
 
-    async delete_vector_index(vector_bucket, vector_index) {
-        const response = await this.s3vectors.deleteIndex({
-            vectorBucketName : vector_bucket,
-            indexName : vector_index
-        })
-    }
-
-    async scan_vectors(vector_bucket, vector_index) {
-        // console.log(vector_bucket, vector_index, vector, num_vectors)
-        const vectors = []
-        let next_token = undefined
-        while (true) {
-            const response = (next_token === undefined) ? (await this.s3vectors.listVectors({
-                vectorBucketName : vector_bucket,
-                indexName : vector_index,
-                maxResults : 1000
-            })) : (await this.s3vectors.listVectors({
-                vectorBucketName : vector_bucket,
-                indexName : vector_index,
-                maxResults : 1000,
-                nextToken : next_token
-            }))
-            vectors.push(...response.vectors)
-            next_token = response.nextToken
-            if (next_token === undefined) {
-                return vectors
-            }
-        }
-    }
-
 }