triangle-utils 1.4.87 → 1.4.89

package/dist/src/UtilsS3.js

@@ -1,7 +1,7 @@
 import { S3, NoSuchKey, GetObjectCommand, PutObjectCommand } from "@aws-sdk/client-s3";
 import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
 import { Readable } from "stream";
-function get_s3_key(s3_id) {
+function get_s3_input(s3_id) {
     const [bucket, path] = (s3_id.match(/^s3\:\/\/([^\/]+)\/([^$]+)$/) || []).slice(1, 3);
     if (bucket === undefined || path === undefined) {
         return undefined;
@@ -14,12 +14,12 @@ export class UtilsS3 {
         this.s3 = new S3({ region: region });
     }
     async get_headers(s3_id) {
-        const s3_key = get_s3_key(s3_id);
-        if (s3_key === undefined) {
+        const s3_input = get_s3_input(s3_id);
+        if (s3_input === undefined) {
             return undefined;
         }
         try {
-            const headers = await this.s3.headObject(s3_key);
+            const headers = await this.s3.headObject(s3_input);
             return {
                 size: headers.ContentLength,
                 ...headers
@@ -30,12 +30,12 @@ export class UtilsS3 {
         }
     }
     async get(s3_id, encoding = "utf-8") {
-        const s3_key = get_s3_key(s3_id);
-        if (s3_key === undefined) {
+        const s3_input = get_s3_input(s3_id);
+        if (s3_input === undefined) {
             return undefined;
         }
         try {
-            const response = await this.s3.getObject(s3_key);
+            const response = await this.s3.getObject(s3_input);
             const body = response.Body;
             if (body === undefined) {
                 return undefined;
@@ -51,12 +51,12 @@ export class UtilsS3 {
         }
     }
     async get_buffer(s3_id) {
-        const s3_key = get_s3_key(s3_id);
-        if (s3_key === undefined) {
+        const s3_input = get_s3_input(s3_id);
+        if (s3_input === undefined) {
             return undefined;
         }
         try {
-            const response = await this.s3.getObject(s3_key);
+            const response = await this.s3.getObject(s3_input);
             const body = response.Body;
             if (body === undefined) {
                 return undefined;
@@ -72,12 +72,12 @@ export class UtilsS3 {
         }
     }
     async get_stream(s3_id) {
-        const s3_key = get_s3_key(s3_id);
-        if (s3_key === undefined) {
+        const s3_input = get_s3_input(s3_id);
+        if (s3_input === undefined) {
             return undefined;
         }
         try {
-            const response = await this.s3.getObject(s3_key);
+            const response = await this.s3.getObject(s3_input);
             const body = response.Body;
             if (body === undefined) {
                 return undefined;
@@ -96,28 +96,34 @@ export class UtilsS3 {
     }
     async query_prefix(s3_id_prefix, options = {}) {
         const compile = options.compile !== undefined ? options.compile : false;
-        const s3_key = get_s3_key(s3_id_prefix);
-        if (s3_key === undefined) {
+        const s3_input = get_s3_input(s3_id_prefix);
+        if (s3_input === undefined) {
             return [];
         }
         const s3_ids = [];
         let last_token = undefined;
         while (true) {
-            const response = await this.s3.listObjectsV2({
-                Bucket: s3_key.Bucket,
+            const request = {
+                Bucket: s3_input.Bucket,
                 Delimiter: "/",
-                Prefix: s3_key.Key,
+                Prefix: s3_input.Key,
                 ContinuationToken: last_token
-            });
-            if (response.Contents === undefined) {
-                return [];
-            }
-            const new_s3_ids = response.Contents
+            };
+            const response = await this.s3.listObjectsV2(request);
+            const new_s3_ids = (response.Contents || [])
                 .map(content => content.Key)
                 .filter(key => key !== undefined)
                 .filter(key => key.substring(key.length - 1) !== "/")
-                .map(key => "s3://" + s3_key.Bucket + "/" + key);
+                .map(key => "s3://" + s3_input.Bucket + "/" + key);
             s3_ids.push(...new_s3_ids);
+            const sub_s3_id_prefixes = (response.CommonPrefixes || [])
+                .map(prefix => prefix.Prefix)
+                .filter(prefix => prefix !== undefined)
+                .map(prefix => "s3://" + s3_input.Bucket + "/" + prefix);
+            for (const sub_s3_id_prefix of sub_s3_id_prefixes) {
+                const new_sub_s3_ids = await this.query_prefix(sub_s3_id_prefix);
+                s3_ids.push(...new_sub_s3_ids);
+            }
             if (response.NextContinuationToken === undefined || !compile) {
                 return s3_ids;
             }
@@ -130,42 +136,42 @@ export class UtilsS3 {
         if (headers !== undefined) {
             return undefined;
         }
-        const s3_key = get_s3_key(s3_id);
-        if (s3_key === undefined) {
+        const s3_input = get_s3_input(s3_id);
+        if (s3_input === undefined) {
             return undefined;
         }
         return await this.s3.putObject({
-            ...s3_key,
+            ...s3_input,
             Body: content,
             ContentType: content_type
         });
     }
     async delete(s3_id) {
-        const s3_key = get_s3_key(s3_id);
-        if (s3_key === undefined) {
+        const s3_input = get_s3_input(s3_id);
+        if (s3_input === undefined) {
             return undefined;
         }
-        await this.s3.deleteObject(s3_key);
+        await this.s3.deleteObject(s3_input);
     }
     async get_download_url(s3_id) {
         const headers = await this.get_headers(s3_id);
         if (headers === undefined) {
             return undefined;
         }
-        const s3_key = get_s3_key(s3_id);
-        if (s3_key === undefined) {
+        const s3_input = get_s3_input(s3_id);
+        if (s3_input === undefined) {
             return undefined;
         }
-        const command = new GetObjectCommand(s3_key);
+        const command = new GetObjectCommand(s3_input);
         const url = await getSignedUrl(this.s3, command);
         return url;
     }
     async get_upload_url(s3_id) {
-        const s3_key = get_s3_key(s3_id);
-        if (s3_key === undefined) {
+        const s3_input = get_s3_input(s3_id);
+        if (s3_input === undefined) {
             return undefined;
         }
-        const command = new PutObjectCommand(s3_key);
+        const command = new PutObjectCommand(s3_input);
         const url = await getSignedUrl(this.s3, command);
         return url;
     }

package/dist/src/UtilsS3Vectors.d.ts

@@ -1,17 +1,22 @@
-export interface Vector {
-    key: string;
-    data: number[];
-    metadata: any;
+export declare class S3Vector {
+    readonly s3_vector_id: string;
+    readonly coordinates: number[];
+    readonly headers: Record<string, any>;
+    constructor(s3_vector: S3Vector);
+    static is(s3_vector: any): s3_vector is S3Vector;
 }
 export declare class UtilsS3Vectors {
     private readonly s3vectors;
     constructor(region: string);
-    scan(vector_bucket: string, vector_index: string): Promise<string[]>;
-    get(vector_bucket: string, vector_index: string, vector_key: string): Promise<Vector | undefined>;
-    batch_get(vector_bucket: string, vector_index: string, vector_keys: string[]): Promise<(Vector | undefined)[]>;
-    create(vector_bucket: string, vector_index: string, vector: Vector): Promise<void>;
-    batch_create(vector_bucket: string, vector_index: string, vectors: Vector[]): Promise<void>;
-    delete(vector_bucket: string, vector_index: string, vector_key: string): Promise<void>;
-    batch_delete(vector_bucket: string, vector_index: string, vector_keys: string[]): Promise<void>;
-    query(vector_bucket: string, vector_index: string, data: number[], num_vectors: number, filters?: Record<string, any> | undefined): Promise<import("@aws-sdk/client-s3vectors").QueryOutputVector[] | undefined>;
+    scan(s3_vector_id_prefix: string): Promise<string[]>;
+    get(s3_vector_id: string): Promise<S3Vector | undefined>;
+    batch_get(s3_vector_ids: string[]): Promise<(S3Vector | undefined)[]>;
+    put(s3_vector: S3Vector): Promise<void>;
+    batch_put(s3_vectors: S3Vector[]): Promise<void>;
+    delete(s3_vector_id: string): Promise<void>;
+    batch_delete(s3_vector_ids: string[]): Promise<void>;
+    query(s3_vector_id_prefix: string, coordinates: number[], num_vectors: number, filters?: Record<string, any> | undefined): Promise<{
+        s3_vector_id: string;
+        distance: number;
+    }[]>;
 }

package/dist/src/UtilsS3Vectors.js

@@ -1,16 +1,47 @@
 import { S3Vectors } from "@aws-sdk/client-s3vectors";
+export class S3Vector {
+    s3_vector_id;
+    coordinates;
+    headers;
+    constructor(s3_vector) {
+        if (!S3Vector.is(s3_vector)) {
+            throw Error("Invalid input.");
+        }
+        this.s3_vector_id = s3_vector.s3_vector_id;
+        this.coordinates = s3_vector.coordinates;
+        this.headers = s3_vector.headers;
+    }
+    static is(s3_vector) {
+        return (s3_vector !== undefined &&
+            typeof s3_vector.s3_vector_id === "string" &&
+            Array.isArray(s3_vector.coordinates) && s3_vector.coordinates.every((coordinate) => typeof coordinate === "number") &&
+            typeof s3_vector.headers === "object" && Object.keys(s3_vector.headers).every((header_id) => typeof header_id === "string"));
+    }
+}
+function get_s3vectors_input(s3_vector_id) {
+    const [s3_vector_bucket_name, s3_vector_index_name, s3_vector_key] = (s3_vector_id.match(/^s3vectors:\/\/([^\/]+)\/([^\/]+)\/([^$]*)$/) || []).slice(1, 4);
+    if (s3_vector_bucket_name === undefined || s3_vector_index_name === undefined || s3_vector_key === undefined) {
+        throw Error("Bad s3_vector_id: " + s3_vector_id);
+    }
+    return {
+        s3_vector_bucket_name: s3_vector_bucket_name,
+        s3_vector_index_name: s3_vector_index_name,
+        s3_vector_key: s3_vector_key
+    };
+}
 export class UtilsS3Vectors {
     s3vectors;
     constructor(region) {
         this.s3vectors = new S3Vectors({ region: region });
     }
-    async scan(vector_bucket, vector_index) {
-        const vector_keys = [];
+    async scan(s3_vector_id_prefix) {
+        const { s3_vector_bucket_name, s3_vector_index_name } = get_s3vectors_input(s3_vector_id_prefix);
+        const s3_vector_ids = [];
         let next_token = undefined;
         while (true) {
             const request = {
-                vectorBucketName: vector_bucket,
-                indexName: vector_index,
+                vectorBucketName: s3_vector_bucket_name,
+                indexName: s3_vector_index_name,
                 maxResults: 1000,
                 nextToken: next_token
             };
@@ -18,38 +49,54 @@ export class UtilsS3Vectors {
             if (response === undefined || response.vectors === undefined) {
                 return [];
             }
-            vector_keys.push(...response.vectors.map(vector => vector.key));
+            s3_vector_ids.push(...response.vectors
+                .map(vector => vector.key)
+                .filter(s3_vector_key => s3_vector_key !== undefined)
+                .map(s3_vector_key => "s3vectors://" + s3_vector_bucket_name + "/" + s3_vector_index_name + "/" + s3_vector_key));
             next_token = response.nextToken;
             if (next_token === undefined) {
-                return vector_keys.filter(vector_key => vector_key !== undefined);
+                return s3_vector_ids;
             }
         }
     }
-    async get(vector_bucket, vector_index, vector_key) {
+    async get(s3_vector_id) {
+        const { s3_vector_bucket_name, s3_vector_index_name, s3_vector_key } = get_s3vectors_input(s3_vector_id);
         const response = await this.s3vectors.getVectors({
-            vectorBucketName: vector_bucket,
-            indexName: vector_index,
-            keys: [vector_key],
+            vectorBucketName: s3_vector_bucket_name,
+            indexName: s3_vector_index_name,
+            keys: [s3_vector_key],
             returnData: true,
             returnMetadata: true
         });
         const vector = response.vectors?.[0];
-        if (vector === undefined || vector.key === undefined || vector.data === undefined || vector.data.float32 === undefined || vector.metadata === undefined) {
+        if (vector === undefined) {
             return undefined;
         }
-        return {
-            key: vector.key,
-            data: vector.data.float32,
-            metadata: vector.metadata
+        const coordinates = vector.data?.float32;
+        const headers = vector.metadata;
+        const s3_vector = {
+            s3_vector_id: s3_vector_id,
+            coordinates: coordinates,
+            headers: headers
         };
+        if (!S3Vector.is(s3_vector)) {
+            console.log("Bad S3Vector:", s3_vector);
+            return undefined;
+        }
+        return s3_vector;
     }
-    async batch_get(vector_bucket, vector_index, vector_keys) {
-        const vectors = [];
-        for (let i = 0; i < vector_keys.length; i += 100) {
+    async batch_get(s3_vector_ids) {
+        const s3vectors_inputs = s3_vector_ids.map(get_s3vectors_input);
+        const { s3_vector_bucket_name, s3_vector_index_name } = s3vectors_inputs[0];
+        if (!s3vectors_inputs.every(s3vectors_input => s3vectors_input.s3_vector_bucket_name === s3_vector_bucket_name && s3vectors_input.s3_vector_index_name === s3_vector_index_name)) {
+            return [];
+        }
+        const s3_vectors = [];
+        for (let i = 0; i < s3vectors_inputs.length; i += 100) {
             const new_vectors = await this.s3vectors.getVectors({
-                vectorBucketName: vector_bucket,
-                indexName: vector_index,
-                keys: vector_keys.slice(i, i + 100),
+                vectorBucketName: s3_vector_bucket_name,
+                indexName: s3_vector_index_name,
+                keys: s3vectors_inputs.slice(i, i + 100).map(s3vectors_input => s3vectors_input.s3_vector_key),
                 returnData: true,
                 returnMetadata: true
             })
@@ -57,75 +104,118 @@ export class UtilsS3Vectors {
             if (new_vectors === undefined) {
                 return [];
             }
-            const vectors_by_key = Object.fromEntries(new_vectors
-                .map(vector => [
-                vector.key,
-                vector.key !== undefined && vector.data !== undefined && vector.data.float32 !== undefined && vector.metadata !== undefined ? {
-                    key: vector.key,
-                    data: vector.data.float32,
-                    metadata: vector.metadata
-                } : undefined
-            ]));
-            vectors.push(...vector_keys.map(vector_key => vectors_by_key[vector_key]));
+            const new_s3_vectors = [];
+            for (const new_vector of new_vectors) {
+                const s3_vector_key = new_vector.key;
+                const coordinates = new_vector.data?.float32;
+                const headers = new_vector.metadata;
+                const new_s3_vector = {
+                    s3_vector_id: "s3vectors://" + s3_vector_bucket_name + "/" + s3_vector_index_name + "/" + s3_vector_key,
+                    coordinates: coordinates,
+                    headers: headers
+                };
+                if (!S3Vector.is(new_s3_vector)) {
+                    console.log("Bad S3Vector:", new_s3_vector);
+                    continue;
+                }
+                new_s3_vectors.push(new_s3_vector);
+            }
+            const new_s3_vectors_by_s3_vector_id = Object.fromEntries(new_s3_vectors.map(new_s3_vector => [new_s3_vector.s3_vector_id, new_s3_vector]));
+            s3_vectors.push(...s3_vector_ids.slice(i, i + 100).map(s3_vector_id => new_s3_vectors_by_s3_vector_id[s3_vector_id]));
         }
-        return vectors;
+        return s3_vectors;
     }
-    async create(vector_bucket, vector_index, vector) {
+    async put(s3_vector) {
+        const { s3_vector_bucket_name, s3_vector_index_name, s3_vector_key } = get_s3vectors_input(s3_vector.s3_vector_id);
         await this.s3vectors.putVectors({
-            vectorBucketName: vector_bucket,
-            indexName: vector_index,
+            vectorBucketName: s3_vector_bucket_name,
+            indexName: s3_vector_index_name,
             vectors: [{
-                    key: vector.key,
+                    key: s3_vector_key,
                     data: {
-                        float32: vector.data
+                        float32: s3_vector.coordinates
                     },
-                    metadata: vector.metadata
+                    metadata: s3_vector.headers
                 }]
         });
     }
-    async batch_create(vector_bucket, vector_index, vectors) {
-        for (let i = 0; i < vectors.length; i += 100) {
+    async batch_put(s3_vectors) {
+        const s3vectors_inputs = s3_vectors.map(s3_vector => get_s3vectors_input(s3_vector.s3_vector_id));
+        const { s3_vector_bucket_name, s3_vector_index_name } = s3vectors_inputs[0];
+        if (!s3vectors_inputs.every(s3vectors_input => s3vectors_input.s3_vector_bucket_name === s3_vector_bucket_name && s3vectors_input.s3_vector_index_name === s3_vector_index_name)) {
+            return undefined;
+        }
+        for (let i = 0; i < s3_vectors.length; i += 100) {
             await this.s3vectors.putVectors({
-                vectorBucketName: vector_bucket,
-                indexName: vector_index,
-                vectors: vectors.slice(i, i + 100).map(vector => ({
-                    key: vector.key,
+                vectorBucketName: s3_vector_bucket_name,
+                indexName: s3_vector_index_name,
+                vectors: s3_vectors.slice(i, i + 100).map(s3_vector => ({
+                    key: get_s3vectors_input(s3_vector.s3_vector_id).s3_vector_key,
                     data: {
-                        float32: vector.data
+                        float32: s3_vector.coordinates
                     },
-                    metadata: vector.metadata
+                    metadata: s3_vector.headers
                 }))
             });
         }
     }
-    async delete(vector_bucket, vector_index, vector_key) {
+    async delete(s3_vector_id) {
+        const { s3_vector_bucket_name, s3_vector_index_name, s3_vector_key } = get_s3vectors_input(s3_vector_id);
         await this.s3vectors.deleteVectors({
-            vectorBucketName: vector_bucket,
-            indexName: vector_index,
-            keys: [vector_key]
+            vectorBucketName: s3_vector_bucket_name,
+            indexName: s3_vector_index_name,
+            keys: [s3_vector_key]
         });
     }
-    async batch_delete(vector_bucket, vector_index, vector_keys) {
-        for (let i = 0; i < vector_keys.length; i += 100) {
+    async batch_delete(s3_vector_ids) {
+        const s3vectors_inputs = s3_vector_ids.map(get_s3vectors_input);
+        const { s3_vector_bucket_name, s3_vector_index_name } = s3vectors_inputs[0];
+        if (!s3vectors_inputs.every(s3vectors_input => s3vectors_input.s3_vector_bucket_name === s3_vector_bucket_name && s3vectors_input.s3_vector_index_name === s3_vector_index_name)) {
+            return undefined;
+        }
+        for (let i = 0; i < s3vectors_inputs.length; i += 100) {
             await this.s3vectors.deleteVectors({
-                vectorBucketName: vector_bucket,
-                indexName: vector_index,
-                keys: vector_keys.slice(i, i + 100)
+                vectorBucketName: s3_vector_bucket_name,
+                indexName: s3_vector_index_name,
+                keys: s3vectors_inputs.slice(i, i + 100).map(s3vectors_input => s3vectors_input.s3_vector_key)
             });
         }
     }
-    async query(vector_bucket, vector_index, data, num_vectors, filters = undefined) {
-        const response = await this.s3vectors.queryVectors({
-            vectorBucketName: vector_bucket,
-            indexName: vector_index,
+    async query(s3_vector_id_prefix, coordinates, num_vectors, filters = undefined) {
+        const { s3_vector_bucket_name, s3_vector_index_name } = get_s3vectors_input(s3_vector_id_prefix);
+        const new_vectors = await this.s3vectors.queryVectors({
+            vectorBucketName: s3_vector_bucket_name,
+            indexName: s3_vector_index_name,
             queryVector: {
-                float32: data
+                float32: coordinates
             },
             topK: num_vectors,
             returnDistance: true,
             returnMetadata: true,
             filter: filters
-        });
-        return response.vectors;
+        })
+            .then(response => response.vectors);
+        if (new_vectors === undefined) {
+            return [];
+        }
+        const results = [];
+        for (const new_vector of new_vectors) {
+            const s3_vector_key = new_vector.key;
+            if (s3_vector_key === undefined) {
+                console.log("Bad s3_vector_key:", s3_vector_key);
+                continue;
+            }
+            const s3_vector_id = s3_vector_id_prefix + "/" + s3_vector_key;
+            const distance = new_vector.distance;
+            if (distance === undefined) {
+                console.log("Bad distance:", distance);
+                continue;
+            }
+            results.push({
+                s3_vector_id: s3_vector_id,
+                distance: distance
+            });
+        }
+        return results;
     }
 }

package/dist/src/f.js

@@ -1,7 +1,7 @@
 import { SecretsManager } from "@aws-sdk/client-secrets-manager";
 import { TriangleUtils } from "./index.js";
 console.log("Starting triangle-utils!");
-const secret_name = "elections_config";
+const secret_name = "global_config";
 const secrets_manager = new SecretsManager({ region: "us-east-1" });
 const secret = await secrets_manager.getSecretValue({
     SecretId: secret_name
@@ -15,6 +15,9 @@ const config = {
 };
 console.log(config);
 const utils = new TriangleUtils(config);
+const raw_s3_id_prefix = config.s3_scout + "/raw_scout_documents/57523929653b604e/";
+const raw_s3_ids = await utils.s3.query_prefix(raw_s3_id_prefix, { compile: true });
+console.log(raw_s3_ids);
 // const text = await utils.xai.grok_simple_query("grok-4.3", "Find the X username corresponding to the candidate of the committee \"Darializa for Congress\"", {
 //     print_usage : true,
 //     json_format : {
@@ -218,10 +221,14 @@ const utils = new TriangleUtils(config);
 // console.log(results)
 // const f = await utils.dynamodb.query_range("federal_election_donations:donor_address_state_id.federal_election_committee_id-receipt_date-federal_election_donation_id", { donor_address_state_id : "NH", federal_election_committee_id : "C00694323" }, { receipt_date : ["2021", "2023"] }, { compile : false })
 // console.log(f)
-console.log("YERP");
-const response = await utils.bedrock.claude_converse("anthropic.claude-opus-4-8", [
-    { role: "user", text: "Hi" }
-], {
-    print_usage: true
-});
-console.log(response);
+// console.log("YERP")
+// const response = await utils.bedrock.claude_converse(
+//     "anthropic.claude-opus-4-8",
+//     [
+//         { role : "user", text : "Hi" }
+//     ],
+//     {
+//         print_usage : true
+//     }
+// )
+// console.log(response)

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "triangle-utils",
-    "version": "1.4.87",
+    "version": "1.4.89",
     "main": "dist/src/index.js",
     "types": "dist/src/index.d.ts",
     "directories": {

package/src/UtilsS3.ts

@@ -1,13 +1,13 @@
-import { S3, NoSuchKey, GetObjectCommand, PutObjectCommand, ListObjectsV2CommandOutput } from "@aws-sdk/client-s3"
+import { S3, NoSuchKey, GetObjectCommand, PutObjectCommand, ListObjectsV2CommandOutput, ListObjectsV2CommandInput } from "@aws-sdk/client-s3"
 import { getSignedUrl } from "@aws-sdk/s3-request-presigner"
 import { Readable } from "stream"
 
-interface S3Key {
+interface S3Input {
     Bucket : string,
     Key : string
 }
 
-function get_s3_key(s3_id : string) : S3Key | undefined {
+function get_s3_input(s3_id : string) : S3Input | undefined {
     const [bucket, path] = (s3_id.match(/^s3\:\/\/([^\/]+)\/([^$]+)$/) || []).slice(1, 3)
     if (bucket === undefined || path === undefined) {
         return undefined
@@ -24,12 +24,12 @@ export class UtilsS3 {
     }
 
     async get_headers(s3_id : string) {
-        const s3_key = get_s3_key(s3_id)
-        if (s3_key === undefined) {
+        const s3_input = get_s3_input(s3_id)
+        if (s3_input === undefined) {
             return undefined
         }
         try {
-            const headers = await this.s3.headObject(s3_key)
+            const headers = await this.s3.headObject(s3_input)
             return {
                 size : headers.ContentLength,
                 ...headers
@@ -40,12 +40,12 @@ export class UtilsS3 {
     }
 
     async get(s3_id : string, encoding : string = "utf-8") : Promise<string | undefined> {
-        const s3_key = get_s3_key(s3_id)
-        if (s3_key === undefined) {
+        const s3_input = get_s3_input(s3_id)
+        if (s3_input === undefined) {
             return undefined
         }
         try {
-            const response = await this.s3.getObject(s3_key)
+            const response = await this.s3.getObject(s3_input)
             const body = response.Body
             if (body === undefined) {
                 return undefined
@@ -61,12 +61,12 @@ export class UtilsS3 {
     }
 
     async get_buffer(s3_id : string) : Promise<Buffer | undefined> {
-        const s3_key = get_s3_key(s3_id)
-        if (s3_key === undefined) {
+        const s3_input = get_s3_input(s3_id)
+        if (s3_input === undefined) {
             return undefined
         }
         try {
-            const response = await this.s3.getObject(s3_key)
+            const response = await this.s3.getObject(s3_input)
             const body = response.Body
             if (body === undefined) {
                 return undefined
@@ -82,12 +82,12 @@ export class UtilsS3 {
     }
 
     async get_stream(s3_id : string) : Promise<Readable | undefined> {
-        const s3_key = get_s3_key(s3_id)
-        if (s3_key === undefined) {
+        const s3_input = get_s3_input(s3_id)
+        if (s3_input === undefined) {
             return undefined
         }
         try {
-            const response = await this.s3.getObject(s3_key)
+            const response = await this.s3.getObject(s3_input)
             const body = response.Body
             if (body === undefined) {
                 return undefined
@@ -106,28 +106,34 @@ export class UtilsS3 {
 
     async query_prefix(s3_id_prefix : string, options : { compile? : boolean } = {}) {
         const compile = options.compile !== undefined ? options.compile : false
-        const s3_key = get_s3_key(s3_id_prefix)
-        if (s3_key === undefined) {
+        const s3_input = get_s3_input(s3_id_prefix)
+        if (s3_input === undefined) {
             return []
         }
-        const s3_ids = []
+        const s3_ids : string[] = []
         let last_token : string | undefined = undefined
         while (true) {
-            const response : ListObjectsV2CommandOutput = await this.s3.listObjectsV2({
-                Bucket : s3_key.Bucket,
+            const request : ListObjectsV2CommandInput = {
+                Bucket : s3_input.Bucket,
                 Delimiter : "/",
-                Prefix : s3_key.Key,
+                Prefix : s3_input.Key,
                 ContinuationToken : last_token
-            })
-            if (response.Contents === undefined) {
-                return []
             }
-            const new_s3_ids = response.Contents
+            const response : ListObjectsV2CommandOutput = await this.s3.listObjectsV2(request)
+            const new_s3_ids = (response.Contents || [])
                 .map(content => content.Key)
                 .filter(key => key !== undefined)
                 .filter(key => key.substring(key.length - 1) !== "/")
-                .map(key => "s3://" + s3_key.Bucket + "/" + key)
+                .map(key => "s3://" + s3_input.Bucket + "/" + key)
             s3_ids.push(...new_s3_ids)
+            const sub_s3_id_prefixes = (response.CommonPrefixes || [])
+                .map(prefix => prefix.Prefix)
+                .filter(prefix => prefix !== undefined)
+                .map(prefix => "s3://" + s3_input.Bucket + "/" + prefix)
+            for (const sub_s3_id_prefix of sub_s3_id_prefixes) {
+                const new_sub_s3_ids = await this.query_prefix(sub_s3_id_prefix)
+                s3_ids.push(...new_sub_s3_ids)
+            }
             if (response.NextContinuationToken === undefined || !compile) {
                 return s3_ids
             }
@@ -144,23 +150,23 @@ export class UtilsS3 {
         if (headers !== undefined) {
             return undefined
         }
-        const s3_key = get_s3_key(s3_id)
-        if (s3_key === undefined) {
+        const s3_input = get_s3_input(s3_id)
+        if (s3_input === undefined) {
             return undefined
         }
         return await this.s3.putObject({
-            ...s3_key,
+            ...s3_input,
             Body : content,
             ContentType : content_type
         })
     }
 
     async delete(s3_id : string) : Promise<void>{
-        const s3_key = get_s3_key(s3_id)
-        if (s3_key === undefined) {
+        const s3_input = get_s3_input(s3_id)
+        if (s3_input === undefined) {
             return undefined
         }
-        await this.s3.deleteObject(s3_key)
+        await this.s3.deleteObject(s3_input)
     }
 
     async get_download_url(s3_id : string) : Promise<string | undefined> {
@@ -168,21 +174,21 @@ export class UtilsS3 {
         if (headers === undefined) {
             return undefined
         }
-        const s3_key = get_s3_key(s3_id)
-        if (s3_key === undefined) {
+        const s3_input = get_s3_input(s3_id)
+        if (s3_input === undefined) {
             return undefined
         }
-        const command = new GetObjectCommand(s3_key)
+        const command = new GetObjectCommand(s3_input)
         const url = await getSignedUrl(this.s3, command)
         return url
     }
 
     async get_upload_url(s3_id : string) : Promise<string | undefined> {
-        const s3_key = get_s3_key(s3_id)
-        if (s3_key === undefined) {
+        const s3_input = get_s3_input(s3_id)
+        if (s3_input === undefined) {
             return undefined
         }
-        const command = new PutObjectCommand(s3_key)
+        const command = new PutObjectCommand(s3_input)
         const url = await getSignedUrl(this.s3, command)
         return url
     }

package/src/UtilsS3Vectors.ts

@@ -1,9 +1,45 @@
 import { ListVectorsCommandInput, S3Vectors } from "@aws-sdk/client-s3vectors"
 
-export interface Vector {
-    key : string,
-    data : number[],
-    metadata : any
+export class S3Vector {
+    readonly s3_vector_id : string
+    readonly coordinates : number[]
+    readonly headers : Record<string, any>
+
+    constructor(s3_vector : S3Vector) {
+        if (!S3Vector.is(s3_vector)) {
+            throw Error("Invalid input.")
+        }
+        this.s3_vector_id = s3_vector.s3_vector_id
+        this.coordinates = s3_vector.coordinates
+        this.headers = s3_vector.headers
+    }
+
+    static is(s3_vector : any) : s3_vector is S3Vector {
+        return (
+            s3_vector !== undefined &&
+            typeof s3_vector.s3_vector_id === "string" &&
+            Array.isArray(s3_vector.coordinates) && s3_vector.coordinates.every((coordinate : any) => typeof coordinate === "number") &&
+            typeof s3_vector.headers === "object" && Object.keys(s3_vector.headers).every((header_id : any) => typeof header_id === "string")
+        )
+    }
+}
+
+interface S3VectorsInput {
+    s3_vector_bucket_name : string,
+    s3_vector_index_name : string,
+    s3_vector_key : string
+}
+
+function get_s3vectors_input(s3_vector_id : string) : S3VectorsInput {
+    const [s3_vector_bucket_name, s3_vector_index_name, s3_vector_key] = (s3_vector_id.match(/^s3vectors:\/\/([^\/]+)\/([^\/]+)\/([^$]*)$/) || []).slice(1, 4)
+    if (s3_vector_bucket_name === undefined || s3_vector_index_name === undefined || s3_vector_key === undefined) {
+        throw Error("Bad s3_vector_id: " + s3_vector_id)
+    }
+    return {
+        s3_vector_bucket_name : s3_vector_bucket_name,
+        s3_vector_index_name : s3_vector_index_name,
+        s3_vector_key : s3_vector_key
+    }
 }
 
 export class UtilsS3Vectors {
@@ -14,13 +50,14 @@ export class UtilsS3Vectors {
         this.s3vectors = new S3Vectors({ region : region })
     }
 
-    async scan(vector_bucket : string, vector_index : string) : Promise<string[]> {
-        const vector_keys = []
+    async scan(s3_vector_id_prefix : string) : Promise<string[]> {
+        const { s3_vector_bucket_name, s3_vector_index_name } = get_s3vectors_input(s3_vector_id_prefix)
+        const s3_vector_ids : string[] = []
         let next_token : string | undefined = undefined
         while (true) {
             const request : ListVectorsCommandInput = {
-                vectorBucketName : vector_bucket,
-                indexName : vector_index,
+                vectorBucketName : s3_vector_bucket_name,
+                indexName : s3_vector_index_name,
                 maxResults : 1000,
                 nextToken : next_token
             }
@@ -28,40 +65,58 @@ export class UtilsS3Vectors {
             if (response === undefined || response.vectors === undefined) {
                 return []
             }
-            vector_keys.push(...response.vectors.map(vector => vector.key))
+            s3_vector_ids.push(
+                ...response.vectors
+                    .map(vector => vector.key)
+                    .filter(s3_vector_key => s3_vector_key !== undefined)
+                    .map(s3_vector_key => "s3vectors://" + s3_vector_bucket_name + "/" + s3_vector_index_name + "/" + s3_vector_key)
+            )
             next_token = response.nextToken
             if (next_token === undefined) {
-                return vector_keys.filter(vector_key => vector_key !== undefined)
+                return s3_vector_ids
             }
         }
     }
 
-    async get(vector_bucket : string, vector_index : string, vector_key : string) : Promise<Vector | undefined> {
+    async get(s3_vector_id : string) : Promise<S3Vector | undefined> {
+        const { s3_vector_bucket_name, s3_vector_index_name, s3_vector_key } = get_s3vectors_input(s3_vector_id)
         const response = await this.s3vectors.getVectors({
-            vectorBucketName : vector_bucket,
-            indexName : vector_index,
-            keys : [vector_key],
+            vectorBucketName : s3_vector_bucket_name,
+            indexName : s3_vector_index_name,
+            keys : [s3_vector_key],
             returnData : true,
             returnMetadata : true
         })
         const vector = response.vectors?.[0]
-        if (vector === undefined || vector.key === undefined || vector.data === undefined || vector.data.float32 === undefined || vector.metadata === undefined) {
+        if (vector === undefined) {
             return undefined
         }
-        return {
-            key : vector.key,
-            data : vector.data.float32,
-            metadata : vector.metadata
+        const coordinates : any = vector.data?.float32
+        const headers : any = vector.metadata
+        const s3_vector : S3Vector = {
+            s3_vector_id : s3_vector_id,
+            coordinates : coordinates,
+            headers : headers
+        }
+        if (!S3Vector.is(s3_vector)) {
+            console.log("Bad S3Vector:", s3_vector)
+            return undefined
         }
+        return s3_vector
     }
 
-    async batch_get(vector_bucket : string, vector_index : string, vector_keys : string[]) {
-        const vectors = []
-        for (let i = 0; i < vector_keys.length; i += 100) {
+    async batch_get(s3_vector_ids : string[]) {
+        const s3vectors_inputs : S3VectorsInput[] = s3_vector_ids.map(get_s3vectors_input)
+        const { s3_vector_bucket_name, s3_vector_index_name } = s3vectors_inputs[0]
+        if (!s3vectors_inputs.every(s3vectors_input => s3vectors_input.s3_vector_bucket_name === s3_vector_bucket_name && s3vectors_input.s3_vector_index_name === s3_vector_index_name)) {
+            return []
+        }
+        const s3_vectors : (S3Vector | undefined)[] = []
+        for (let i = 0; i < s3vectors_inputs.length; i += 100) {
             const new_vectors = await this.s3vectors.getVectors({
-                vectorBucketName : vector_bucket,
-                indexName : vector_index,
-                keys : vector_keys.slice(i, i + 100),
+                vectorBucketName : s3_vector_bucket_name,
+                indexName : s3_vector_index_name,
+                keys : s3vectors_inputs.slice(i, i + 100).map(s3vectors_input => s3vectors_input.s3_vector_key),
                 returnData : true,
                 returnMetadata : true
             })
@@ -69,83 +124,125 @@ export class UtilsS3Vectors {
             if (new_vectors === undefined) {
                 return []
             }
-            const vectors_by_key : Record<string, Vector | undefined> = Object.fromEntries(new_vectors
-                .map(vector => [
-                    vector.key,
-                    vector.key !== undefined && vector.data !== undefined && vector.data.float32 !== undefined && vector.metadata !== undefined ? {
-                        key : vector.key,
-                        data : vector.data.float32,
-                        metadata : vector.metadata
-                    } : undefined
-                ])
-            )
-            vectors.push(...vector_keys.map(vector_key => vectors_by_key[vector_key]))
+            const new_s3_vectors : S3Vector[] = []
+            for (const new_vector of new_vectors) {
+                const s3_vector_key : any = new_vector.key
+                const coordinates : any = new_vector.data?.float32
+                const headers : any = new_vector.metadata
+                const new_s3_vector : S3Vector | undefined = {
+                    s3_vector_id : "s3vectors://" + s3_vector_bucket_name + "/" + s3_vector_index_name + "/" + s3_vector_key,
+                    coordinates : coordinates,
+                    headers : headers
+                }
+                if (!S3Vector.is(new_s3_vector)) {
+                    console.log("Bad S3Vector:", new_s3_vector)
+                    continue
+                }
+                new_s3_vectors.push(new_s3_vector)
+            }
+            const new_s3_vectors_by_s3_vector_id = Object.fromEntries(new_s3_vectors.map(new_s3_vector => [new_s3_vector.s3_vector_id, new_s3_vector]))
+            s3_vectors.push(...s3_vector_ids.slice(i, i + 100).map(s3_vector_id => new_s3_vectors_by_s3_vector_id[s3_vector_id]))
         }
-        return vectors
+        return s3_vectors
     }
 
-    async create(vector_bucket : string, vector_index : string, vector : Vector) : Promise<void> {
+    async put(s3_vector : S3Vector) : Promise<void> {
+        const { s3_vector_bucket_name, s3_vector_index_name, s3_vector_key } = get_s3vectors_input(s3_vector.s3_vector_id)
         await this.s3vectors.putVectors({
-            vectorBucketName : vector_bucket,
-            indexName : vector_index,
+            vectorBucketName : s3_vector_bucket_name,
+            indexName : s3_vector_index_name,
             vectors : [{
-                key : vector.key,
+                key : s3_vector_key,
                 data : {
-                    float32 : vector.data
+                    float32 : s3_vector.coordinates
                 },
-                metadata : vector.metadata
+                metadata : s3_vector.headers
             }]
         })
     }
 
-    async batch_create(vector_bucket : string, vector_index : string, vectors : Vector[]) : Promise<void> {
-        for (let i = 0; i < vectors.length; i += 100) {
+    async batch_put(s3_vectors : S3Vector[]) : Promise<void> {
+        const s3vectors_inputs : S3VectorsInput[] = s3_vectors.map(s3_vector => get_s3vectors_input(s3_vector.s3_vector_id))
+        const { s3_vector_bucket_name, s3_vector_index_name } = s3vectors_inputs[0]
+        if (!s3vectors_inputs.every(s3vectors_input => s3vectors_input.s3_vector_bucket_name === s3_vector_bucket_name && s3vectors_input.s3_vector_index_name === s3_vector_index_name)) {
+            return undefined
+        }
+        for (let i = 0; i < s3_vectors.length; i += 100) {
             await this.s3vectors.putVectors({
-                vectorBucketName : vector_bucket,
-                indexName : vector_index,
-                vectors : vectors.slice(i, i + 100).map(vector => ({
-                    key : vector.key,
+                vectorBucketName : s3_vector_bucket_name,
+                indexName : s3_vector_index_name,
+                vectors : s3_vectors.slice(i, i + 100).map(s3_vector => ({
+                    key : get_s3vectors_input(s3_vector.s3_vector_id).s3_vector_key,
                     data : {
-                        float32 : vector.data
+                        float32 : s3_vector.coordinates
                     },
-                    metadata : vector.metadata
+                    metadata : s3_vector.headers
                 }))
             })
             
         }
     }
 
-    async delete(vector_bucket : string, vector_index : string, vector_key : string) : Promise<void> {
+    async delete(s3_vector_id : string) : Promise<void> {
+        const { s3_vector_bucket_name, s3_vector_index_name, s3_vector_key } = get_s3vectors_input(s3_vector_id)
         await this.s3vectors.deleteVectors({
-            vectorBucketName : vector_bucket,
-            indexName : vector_index,
-            keys : [vector_key]
+            vectorBucketName : s3_vector_bucket_name,
+            indexName : s3_vector_index_name,
+            keys : [s3_vector_key]
         })
     }
 
-    async batch_delete(vector_bucket : string, vector_index : string, vector_keys : string[]) : Promise<void> {
-        for (let i = 0; i < vector_keys.length; i += 100) {
+    async batch_delete(s3_vector_ids : string[]) : Promise<void> {
+        const s3vectors_inputs : S3VectorsInput[] = s3_vector_ids.map(get_s3vectors_input)
+        const { s3_vector_bucket_name, s3_vector_index_name } = s3vectors_inputs[0]
+        if (!s3vectors_inputs.every(s3vectors_input => s3vectors_input.s3_vector_bucket_name === s3_vector_bucket_name && s3vectors_input.s3_vector_index_name === s3_vector_index_name)) {
+            return undefined
+        }
+        for (let i = 0; i < s3vectors_inputs.length; i += 100) {
             await this.s3vectors.deleteVectors({
-                vectorBucketName : vector_bucket,
-                indexName : vector_index,
-                keys : vector_keys.slice(i, i + 100)
+                vectorBucketName : s3_vector_bucket_name,
+                indexName : s3_vector_index_name,
+                keys : s3vectors_inputs.slice(i, i + 100).map(s3vectors_input => s3vectors_input.s3_vector_key)
             })
         }
     }
 
-    async query(vector_bucket : string, vector_index : string, data : number[], num_vectors : number, filters : Record<string, any> | undefined = undefined) {
-        const response = await this.s3vectors.queryVectors({
-            vectorBucketName : vector_bucket,
-            indexName : vector_index,
+    async query(s3_vector_id_prefix : string, coordinates : number[], num_vectors : number, filters : Record<string, any> | undefined = undefined) {
+        const { s3_vector_bucket_name, s3_vector_index_name } = get_s3vectors_input(s3_vector_id_prefix)
+        const new_vectors = await this.s3vectors.queryVectors({
+            vectorBucketName : s3_vector_bucket_name,
+            indexName : s3_vector_index_name,
             queryVector : {
-                float32 : data
+                float32 : coordinates
             },
-            topK : num_vectors, 
+            topK : num_vectors,
             returnDistance : true,
             returnMetadata : true,
             filter : filters
         })
-        return response.vectors
+            .then(response => response.vectors)
+        if (new_vectors === undefined) {
+            return []
+        }
+        const results : { s3_vector_id : string, distance : number }[] = []
+        for (const new_vector of new_vectors) {
+            const s3_vector_key = new_vector.key
+            if (s3_vector_key === undefined) {
+                console.log("Bad s3_vector_key:", s3_vector_key)
+                continue
+            }
+            const s3_vector_id = s3_vector_id_prefix + "/" + s3_vector_key
+            const distance = new_vector.distance
+            if (distance === undefined) {
+                console.log("Bad distance:", distance)
+                continue
+            }
+            results.push({
+                s3_vector_id : s3_vector_id,
+                distance : distance
+            })
+        }
+        return results
     }
 
 }

package/src/UtilsXAI.ts

@@ -90,7 +90,7 @@ export class UtilsXAI {
                 "Authorization" : "Bearer " + this.xai_api_key
             }
         })
-        const output = await response.json()
+        const output : any = await response.json()
         if (options.print_usage !== undefined && options.print_usage) {
             console.log(output.usage)
         }

package/src/f.ts

@@ -3,7 +3,7 @@ import { TriangleUtils } from "."
 
 console.log("Starting triangle-utils!")
 
-const secret_name = "elections_config"
+const secret_name = "global_config"
 
 const secrets_manager = new SecretsManager({ region: "us-east-1" })
 
@@ -24,6 +24,11 @@ console.log(config)
 
 const utils = new TriangleUtils(config)
 
+const raw_s3_id_prefix = config.s3_scout + "/raw_scout_documents/57523929653b604e/"
+const raw_s3_ids = await utils.s3.query_prefix(raw_s3_id_prefix, { compile : true })
+
+console.log(raw_s3_ids)
+
 // const text = await utils.xai.grok_simple_query("grok-4.3", "Find the X username corresponding to the candidate of the committee \"Darializa for Congress\"", {
 //     print_usage : true,
 //     json_format : {
@@ -253,14 +258,14 @@ const utils = new TriangleUtils(config)
 // const f = await utils.dynamodb.query_range("federal_election_donations:donor_address_state_id.federal_election_committee_id-receipt_date-federal_election_donation_id", { donor_address_state_id : "NH", federal_election_committee_id : "C00694323" }, { receipt_date : ["2021", "2023"] }, { compile : false })
 
 // console.log(f)
-console.log("YERP")
-const response = await utils.bedrock.claude_converse(
-    "anthropic.claude-opus-4-8",
-    [
-        { role : "user", text : "Hi" }
-    ],
-    {
-        print_usage : true
-    }
-)
-console.log(response)
+// console.log("YERP")
+// const response = await utils.bedrock.claude_converse(
+//     "anthropic.claude-opus-4-8",
+//     [
+//         { role : "user", text : "Hi" }
+//     ],
+//     {
+//         print_usage : true
+//     }
+// )
+// console.log(response)