triangle-utils 1.0.27 → 1.0.29

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "triangle-utils",
-    "version": "1.0.27",
+    "version": "1.0.29",
     "main": "index.js",
     "author": "",
     "license": "ISC",
@@ -10,6 +10,8 @@
         "@aws-sdk/client-bedrock-runtime": "^3.953.0",
         "@aws-sdk/client-dynamodb": "^3.953.0",
         "@aws-sdk/client-s3": "^3.953.0",
+        "@aws-sdk/client-s3vectors": "^3.953.0",
+        "@aws-sdk/s3-request-presigner": "^3.953.0",
         "nodemailer": "^7.0.11",
         "scrapingbee": "^1.7.6"
     }

package/utils/Utils_Bee.js

@@ -19,7 +19,7 @@ export default class Utils_Bee {
                     console.log("Status", response.status)
                     return undefined
                 }
-                const text = text_decoder.decode(response.data)
+                const text = this.text_decoder.decode(response.data)
                 return text
             } catch (e) {
                 console.log(e.stack)

package/utils/Utils_S3.js

@@ -1,4 +1,5 @@
-import { S3, NoSuchKey } from "@aws-sdk/client-s3"
+import { S3, NoSuchKey, GetObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3"
+import { getSignedUrl } from "@aws-sdk/s3-request-presigner"
 
 function parse_s3_filename(s3_filename) {
     if (s3_filename.substring(0, 5) !== "s3://") {
@@ -71,4 +72,22 @@ export default class Utils_S3 {
         return await this.s3.deleteObject(s3_key)
     }
 
+    async generate_download_url(s3_filename) {
+        const exists = await this.file_exists(s3_filename)
+        if (!exists) {
+            return undefined
+        }
+        const s3_key = parse_s3_filename(s3_filename)
+        const command = new GetObjectCommand(s3_key)
+        const url = await getSignedUrl(s3, command)
+        return url
+    }
+
+    async generate_upload_url(s3_filename) {
+        const s3_key = parse_s3_filename(s3_filename)
+        const command = new PutObjectCommand(s3_key)
+        const url = await getSignedUrl(s3, command)
+        return url
+    }
+
 }

package/utils/Utils_S3Vectors.js

@@ -0,0 +1,90 @@
+import { S3Vectors } from "@aws-sdk/client-s3vectors"
+
+export default class Utils_S3Vectors {
+
+    constructor(config) {
+        this.s3vectors = new S3Vectors({ region : config.region })
+    }
+
+    async get_vector(vector_bucket, vector_index, vector_key) {
+        const vector = await this.s3vectors.getVectors({
+            vectorBucketName : vector_bucket,
+            indexName : vector_index,
+            keys : [vector_key],
+            returnData : true,
+            returnMetadata : true
+        }).then(response => response.vectors[0])
+        if (vector === undefined) {
+            return undefined
+        }
+        return vector
+    }
+
+    async create_vector(vector_bucket, vector_index, vector_key, vector, metadata) {
+        await this.s3vectors.putVectors({
+            vectorBucketName : vector_bucket,
+            indexName : vector_index,
+            vectors : [{
+                key : vector_key,
+                data : {
+                    float32 : vector
+                },
+                metadata : metadata
+            }]
+        })
+    }
+
+    async delete_vector(vector_bucket, vector_index, vector_key) {
+        await this.s3vectors.deleteVectors({
+            vectorBucketName : vector_bucket,
+            indexName : vector_index,
+            keys : [vector_key]
+        })
+    }
+
+    async query_vectors(vector_bucket, vector_index, vector, num_vectors, filters) {
+        const response = await this.s3vectors.queryVectors({
+            vectorBucketName : vector_bucket,
+            indexName : vector_index,
+            queryVector : {
+                float32 : vector
+            },
+            topK : num_vectors, 
+            returnDistance : true,
+            returnMetadata : true,
+            filter : filters
+        })
+        return response.vectors
+    }
+
+    async delete_vector_index(vector_bucket, vector_index) {
+        const response = await this.s3vectors.deleteIndex({
+            vectorBucketName : vector_bucket,
+            indexName : vector_index
+        })
+    }
+
+    async scan_vectors(vector_bucket, vector_index) {
+        // console.log(vector_bucket, vector_index, vector, num_vectors)
+        const vectors = []
+        let next_token = undefined
+        while (true) {
+            const response = (next_token === undefined) ? (await this.s3vectors.listVectors({
+                vectorBucketName : vector_bucket,
+                indexName : vector_index,
+                maxResults : 1000
+            })) : (await this.s3vectors.listVectors({
+                vectorBucketName : vector_bucket,
+                indexName : vector_index,
+                maxResults : 1000,
+                nextToken : next_token
+            }))
+            vectors.push(...response.vectors)
+            next_token = response.nextToken
+            if (next_token === undefined) {
+                return vectors
+            }
+        }
+    }
+
+}