triangle-utils 1.2.12 → 1.3.1

package/dist/src/Utils_S3.js

@@ -0,0 +1,96 @@
+import { S3, NoSuchKey, GetObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3";
+import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
+function parse_s3_filename(s3_filename) {
+    if (s3_filename.substring(0, 5) !== "s3://") {
+        return undefined;
+    }
+    const bucket_filename = s3_filename.substring(5);
+    const bucket = bucket_filename.split("/")[0];
+    const filename = bucket_filename.split("/").slice(1).join("/");
+    return { Bucket: bucket, Key: filename };
+}
+export class Utils_S3 {
+    s3;
+    constructor(config) {
+        this.s3 = new S3({ region: config.region });
+    }
+    async file_exists(s3_filename) {
+        const s3_key = parse_s3_filename(s3_filename);
+        if (s3_key === undefined) {
+            return undefined;
+        }
+        try {
+            const head = await this.s3.headObject(s3_key);
+            return true;
+        }
+        catch (error) {
+            return false;
+        }
+    }
+    async get_file(s3_filename) {
+        const s3_key = parse_s3_filename(s3_filename);
+        if (s3_key === undefined) {
+            return undefined;
+        }
+        try {
+            const response = await this.s3.getObject(s3_key);
+            const body = response.Body;
+            if (body === undefined) {
+                return undefined;
+            }
+            const text = body.transformToString("utf-8");
+            return text;
+        }
+        catch (error) {
+            if (error instanceof NoSuchKey) {
+                return undefined;
+            }
+            throw error;
+        }
+    }
+    async create_file(s3_filename, content, options = {}) {
+        const content_type = options.content_type !== undefined ? options.content_type : undefined;
+        const exists = await this.file_exists(s3_filename);
+        if (exists) {
+            return undefined;
+        }
+        const s3_key = parse_s3_filename(s3_filename);
+        if (s3_key === undefined) {
+            return undefined;
+        }
+        return await this.s3.putObject({
+            ...s3_key,
+            Body: content,
+            ContentType: content_type
+        });
+    }
+    async delete_file(s3_filename) {
+        const s3_key = parse_s3_filename(s3_filename);
+        if (s3_key === undefined) {
+            return undefined;
+        }
+        await this.s3.deleteObject(s3_key);
+    }
+    async generate_download_url(s3_filename) {
+        const exists = await this.file_exists(s3_filename);
+        if (!exists) {
+            return undefined;
+        }
+        const s3_key = parse_s3_filename(s3_filename);
+        if (s3_key === undefined) {
+            return undefined;
+        }
+        const command = new GetObjectCommand(s3_key);
+        const url = await getSignedUrl(this.s3, command);
+        return url;
+    }
+    async generate_upload_url(s3_filename) {
+        const s3_key = parse_s3_filename(s3_filename);
+        if (s3_key === undefined) {
+            return undefined;
+        }
+        const command = new PutObjectCommand(s3_key);
+        const url = await getSignedUrl(this.s3, command);
+        return url;
+    }
+}

package/dist/src/Utils_S3Vectors.js

@@ -0,0 +1,131 @@
+import { S3Vectors } from "@aws-sdk/client-s3vectors";
+export class Utils_S3Vectors {
+    s3vectors;
+    constructor(config) {
+        this.s3vectors = new S3Vectors({ region: config.region });
+    }
+    async scan(vector_bucket, vector_index) {
+        const vector_keys = [];
+        let next_token = undefined;
+        while (true) {
+            const request = {
+                vectorBucketName: vector_bucket,
+                indexName: vector_index,
+                maxResults: 1000,
+                nextToken: next_token
+            };
+            const response = await this.s3vectors.listVectors(request);
+            if (response === undefined || response.vectors === undefined) {
+                return [];
+            }
+            vector_keys.push(...response.vectors.map(vector => vector.key));
+            next_token = response.nextToken;
+            if (next_token === undefined) {
+                return vector_keys.filter(vector_key => vector_key !== undefined);
+            }
+        }
+    }
+    async get(vector_bucket, vector_index, vector_key) {
+        const response = await this.s3vectors.getVectors({
+            vectorBucketName: vector_bucket,
+            indexName: vector_index,
+            keys: [vector_key],
+            returnData: true,
+            returnMetadata: true
+        });
+        const vector = response.vectors?.[0];
+        if (vector === undefined || vector.key === undefined || vector.data === undefined || vector.data.float32 === undefined || vector.metadata === undefined) {
+            return undefined;
+        }
+        return {
+            key: vector.key,
+            data: vector.data.float32,
+            metadata: vector.metadata
+        };
+    }
+    async batch_get(vector_bucket, vector_index, vector_keys) {
+        const vectors = [];
+        for (let i = 0; i < vector_keys.length; i += 100) {
+            const new_vectors = await this.s3vectors.getVectors({
+                vectorBucketName: vector_bucket,
+                indexName: vector_index,
+                keys: vector_keys.slice(i, i + 100),
+                returnData: true,
+                returnMetadata: true
+            })
+                .then(response => response.vectors);
+            if (new_vectors === undefined) {
+                return [];
+            }
+            const vectors_by_key = Object.fromEntries(new_vectors
+                .map(vector => [
+                vector.key,
+                vector.key !== undefined && vector.data !== undefined && vector.data.float32 !== undefined && vector.metadata !== undefined ? {
+                    key: vector.key,
+                    data: vector.data.float32,
+                    metadata: vector.metadata
+                } : undefined
+            ]));
+            vectors.push(...vector_keys.map(vector_key => vectors_by_key[vector_key]));
+        }
+        return vectors;
+    }
+    async create(vector_bucket, vector_index, vector) {
+        await this.s3vectors.putVectors({
+            vectorBucketName: vector_bucket,
+            indexName: vector_index,
+            vectors: [{
+                    key: vector.key,
+                    data: {
+                        float32: vector.data
+                    },
+                    metadata: vector.metadata
+                }]
+        });
+    }
+    async batch_create(vector_bucket, vector_index, vectors) {
+        for (let i = 0; i < vectors.length; i += 100) {
+            await this.s3vectors.putVectors({
+                vectorBucketName: vector_bucket,
+                indexName: vector_index,
+                vectors: vectors.slice(i, i + 100).map(vector => ({
+                    key: vector.key,
+                    data: {
+                        float32: vector.data
+                    },
+                    metadata: vector.metadata
+                }))
+            });
+        }
+    }
+    async delete(vector_bucket, vector_index, vector_key) {
+        await this.s3vectors.deleteVectors({
+            vectorBucketName: vector_bucket,
+            indexName: vector_index,
+            keys: [vector_key]
+        });
+    }
+    async batch_delete(vector_bucket, vector_index, vector_keys) {
+        for (let i = 0; i < vector_keys.length; i += 100) {
+            await this.s3vectors.deleteVectors({
+                vectorBucketName: vector_bucket,
+                indexName: vector_index,
+                keys: vector_keys.slice(i, i + 100)
+            });
+        }
+    }
+    async query(vector_bucket, vector_index, data, num_vectors, filters = {}) {
+        const response = await this.s3vectors.queryVectors({
+            vectorBucketName: vector_bucket,
+            indexName: vector_index,
+            queryVector: {
+                float32: data
+            },
+            topK: num_vectors,
+            returnDistance: true,
+            returnMetadata: true,
+            filter: filters
+        });
+        return response.vectors;
+    }
+}

package/dist/src/Utils_Youtube.js

@@ -0,0 +1,35 @@
+import { youtube_v3 } from "googleapis";
+export class Utils_Youtube {
+    config;
+    youtube;
+    constructor(config) {
+        this.config = config;
+        this.youtube = new youtube_v3.Youtube({
+            auth: config.youtube_api_key
+        });
+    }
+    async batch_get_videos(video_ids, attributes = ["snippet", "contentDetails"]) {
+        for (let i = 0; i < 3; i++) {
+            try {
+                const response = await this.youtube.videos.list({
+                    part: attributes,
+                    id: video_ids
+                });
+                if (response.status !== 200) {
+                    console.log("Status", response.status);
+                    return undefined;
+                }
+                const items = response.data.items;
+                return items;
+            }
+            catch (error) {
+                if (error instanceof Error) {
+                    console.log(error.stack);
+                }
+                console.log("Failed to get from Scraping Bee.");
+            }
+        }
+        console.log("Failed to get from Scraping Bee, quitting.");
+        return undefined;
+    }
+}