triangle-utils 1.2.12 → 1.3.1

package/src/Utils_DynamoDB.ts

@@ -0,0 +1,448 @@
+import { AttributeValue, DynamoDB, QueryCommandInput, QueryCommandOutput, ScanCommandInput, ScanCommandOutput, UpdateItemCommandInput } from "@aws-sdk/client-dynamodb"
+import { Config } from "./types/Config"
+
+function convert_output(dynamoobject : AttributeValue) : string | boolean | number | any[] | Set<any> | Record<string, any> | undefined {
+    if (dynamoobject.S !== undefined) {
+        return dynamoobject.S
+    } else if (dynamoobject.N !== undefined) {
+        return Number(dynamoobject.N)
+    } else if (dynamoobject.L !== undefined) {
+        return dynamoobject.L.map((a : AttributeValue) => convert_output(a))
+    } else if (dynamoobject.SS !== undefined) {
+        return new Set(dynamoobject.SS)
+    } else if (dynamoobject.M !== undefined) {
+        return Object.fromEntries(Object.entries(dynamoobject.M).map(([key, value]) => [key, convert_output(value)]))
+    } else if (dynamoobject.BOOL !== undefined) {
+        return dynamoobject.BOOL
+    }
+    return undefined
+}
+
+function not_undefined<T>(x : T | undefined): x is T {
+    return x !== undefined
+}
+
+function is_item(x : any): x is Record<string, any> {
+    return typeof x === "object" && Object.keys(x).filter(key => typeof key !== "string").length === 0 && !Array.isArray(x)
+}
+
+function convert_input(input : string | boolean | number | any[] | Set<any> | Record<string, any>) : AttributeValue | undefined {
+    if (typeof input === "string") {
+        return { S : input }
+    } else if (typeof input === "boolean") {
+        return { BOOL : input }
+    } else if (typeof input === "number") {
+        return { N : input.toString() }
+    } else if (Array.isArray(input)) {
+        const converted_list : AttributeValue[] = input.map((a : any) => convert_input(a))
+            .filter(converted_input => converted_input !== undefined)
+        if (converted_list.length !== input.length) {
+            return undefined
+        }
+        return { L : converted_list }
+    } else if (input instanceof Set) {
+        const converted_list : string[] = Array.from(input)
+            .filter(converted_input => typeof converted_input === "string")
+        if (converted_list.length !== input.size) {
+            return undefined
+        }
+        return { SS : converted_list }
+    } else {
+        const converted_inputs : Record<string, AttributeValue> = Object.fromEntries(Object.entries(input)
+            .filter(([key, value]) => value !== undefined && value !== null && key !== "")
+            .map(([key, value]) => [key, convert_input(value)])
+            .filter(([key, value]) => not_undefined(value))
+        )
+        return {
+            M : converted_inputs
+        }
+    }
+}
+
+async function compile_pages(
+    request : ScanCommandInput|QueryCommandInput,
+    f : (request : ScanCommandInput|QueryCommandInput) => Promise<ScanCommandOutput|QueryCommandOutput>,
+    compile : boolean = true
+) : Promise<any[]> {
+    const items = []
+    let last_eval_key : Record<string, AttributeValue> | undefined = undefined
+    while (true) {
+        const request_page : ScanCommandInput|QueryCommandInput = {
+            ...request,
+            ExclusiveStartKey : last_eval_key
+        }
+        const response = await f(request_page)
+        if (response.Items === undefined) {
+            return []
+        }
+        const new_items = response.Items.map(item => convert_output({ M : item }))
+        items.push(...new_items)
+        if (response.LastEvaluatedKey === undefined || !compile) {
+            return items
+        }
+        last_eval_key = response.LastEvaluatedKey
+    }
+}
+
+export class Utils_DynamoDB {
+
+    readonly dynamodb : DynamoDB
+
+    constructor(config : Config) {
+        this.dynamodb = new DynamoDB({ region : config.region })
+    }
+
+    async scan(
+        table : string,
+        options : {
+            filters? : Record<string, any>,
+            undefined_attribute_names? : string[],
+            defined_attribute_names? : string[],
+            attribute_names? : string[],
+            concurrency? : number
+        } = {}
+    ) : Promise<Record<string, any>[]> {
+        const filters = options.filters !== undefined ? options.filters : {}
+        const undefined_attribute_names : string[] = options.undefined_attribute_names !== undefined ? options.undefined_attribute_names : []
+        const defined_attribute_names : string[] = options.defined_attribute_names !== undefined ? options.defined_attribute_names : []
+        const concurrency : number = options.concurrency !== undefined ? options.concurrency : 1
+        const attribute_names : string[] = options.attribute_names !== undefined ? options.attribute_names : []
+        const iterators = []
+        for (let i = 0; i < concurrency; i++) {
+            const expression_attribute_names = Object.fromEntries(
+                [...Object.keys(filters), ...attribute_names, ...undefined_attribute_names, ...defined_attribute_names]
+                    .map(attribute_name => ["#" + attribute_name, attribute_name])
+            )
+            const expression_attribute_values = Object.fromEntries(
+                Object.entries(filters)
+                    .map(([attribute_name, attribute_value]) => [":" + attribute_name, convert_input(attribute_value)])
+                    .filter(([attribute_name, attribute_value]) => not_undefined(attribute_value))
+            )
+            const filter_expression = [
+                ...Object.keys(filters).map(attribute_name => "#" + attribute_name + " = :" + attribute_name),
+                ...undefined_attribute_names.map(attribute_name => "attribute_not_exists(#" + attribute_name + ")"),
+                ...defined_attribute_names.map(attribute_name => "attribute_exists(#" + attribute_name + ")")
+            ].join(" AND ")
+            const projection_expression = attribute_names.map(attribute_name => "#" + attribute_name).join(", ")
+            const request : ScanCommandInput = {
+                TableName : table,
+                ExpressionAttributeNames : Object.keys(expression_attribute_names).length > 0 ? expression_attribute_names : undefined,
+                ExpressionAttributeValues : Object.keys(expression_attribute_values).length > 0 ? expression_attribute_values : undefined,
+                FilterExpression : filter_expression.length > 0 ? filter_expression : undefined,
+                ProjectionExpression : projection_expression.length > 0 ? projection_expression : undefined,
+                Segment : i,
+                TotalSegments : concurrency
+            }
+            iterators.push(compile_pages(request, (request : ScanCommandInput) => this.dynamodb.scan(request)))
+        }
+        const segments = await Promise.all(iterators)
+        const items = segments.flat().filter(is_item)
+        return items
+    }
+
+    async get(
+        table : string,
+        key : Record<string, any>,
+        consistent : boolean = false
+    ) : Promise<Record<string, any> | undefined> {
+        const converted_key = convert_input(key)?.M
+        if (converted_key === undefined) {
+            return undefined
+        }
+        const item = await this.dynamodb.getItem({
+            ConsistentRead : consistent,
+            TableName : table,
+            Key : converted_key
+        })
+            .then(response => response.Item)
+        if (item === undefined) {
+            return undefined
+        }
+        const converted_output = convert_output({ M : item })
+        if (!is_item(converted_output)) {
+            return undefined
+        }
+        return converted_output
+    }
+
+    async get_max(table : string, primary_key : Record<string, any>) : Promise<Record<string, any> | undefined> {
+        if (Object.keys(primary_key).length !== 1) {
+            return undefined
+        }
+        const key = Object.keys(primary_key)[0]
+        const value = convert_input(Object.values(primary_key)[0])
+        if (value === undefined) {
+            return undefined
+        }
+        const request : QueryCommandInput = {
+            TableName : table,
+            ExpressionAttributeNames: {
+                "#a": key
+            },
+            ExpressionAttributeValues: {
+                ":a": value
+            },
+            KeyConditionExpression: "#a = :a",
+            Limit : 1,
+            ScanIndexForward : false
+        }
+        const items = await this.dynamodb.query(request)
+            .then(response => response.Items)
+        if (items === undefined || items[0] === undefined) {
+            return undefined
+        }
+        const converted_output = convert_output({ M : items[0] })
+        if (!is_item(converted_output)) {
+            return undefined
+        }
+        return converted_output
+    }
+
+    async query(
+        table : string,
+        primary_key : Record<string, any>,
+        options : {
+            reverse? : boolean,
+            compile? : boolean
+        } = {}
+    ) : Promise<Record<string, any>[] | undefined>{
+        const reverse = options.reverse !== undefined ? options.reverse : false
+        const compile = options.compile !== undefined ? options.compile : true
+        if (Object.keys(primary_key).length !== 1) {
+            return undefined
+        }
+        const key = Object.keys(primary_key)[0]
+        const value = convert_input(Object.values(primary_key)[0])
+        if (value === undefined) {
+            return undefined
+        }
+        const request : QueryCommandInput = {
+            TableName : table,
+            ExpressionAttributeNames: {
+                "#a": key
+            },
+            ExpressionAttributeValues: {
+                ":a": value
+            },
+            KeyConditionExpression: "#a = :a",
+            ScanIndexForward : !reverse
+        }
+        return await compile_pages(request, (request) => this.dynamodb.query(request), compile)
+    }
+
+    async query_prefix(
+        table : string,
+        primary_key : Record<string, any>,
+        secondary_key_prefix : Record<string, string>,
+        options : {
+            reverse? : boolean,
+            compile? : boolean
+        } = {}
+    ) : Promise<Record<string, any>[] | undefined>{
+        const reverse = options.reverse !== undefined ? options.reverse : false
+        const compile = options.compile !== undefined ? options.compile : true
+        if (Object.keys(primary_key).length !== 1 || Object.keys(secondary_key_prefix).length !== 1) {
+            return undefined
+        }
+        const converted_primary_value = convert_input(Object.values(primary_key)[0])
+        const converted_secondary_prefix_value = convert_input(Object.values(secondary_key_prefix)[0])
+        if (converted_primary_value === undefined || converted_secondary_prefix_value === undefined) {
+            return undefined
+        }
+        const request : QueryCommandInput = {
+            TableName : table,
+            ExpressionAttributeNames: {
+                "#a": Object.keys(primary_key)[0],
+                "#b": Object.keys(secondary_key_prefix)[0]
+            },
+            ExpressionAttributeValues: {
+                ":a": converted_primary_value,
+                ":b": converted_secondary_prefix_value
+            },
+            KeyConditionExpression: "#a = :a AND begins_with(#b, :b)",
+            ScanIndexForward : !reverse
+        }
+        return await compile_pages(request, (request) => this.dynamodb.query(request), compile)
+    }
+
+    async query_range(
+        table : string,
+        primary_key : Record<string, any>,
+        secondary_key_range : Record<string, string|number[]>,
+        options : {
+            reverse? : boolean,
+            compile? : boolean
+        } = {}
+    ) : Promise<Record<string, any>[] | undefined>{
+        const reverse = options.reverse !== undefined ? options.reverse : false
+        const compile = options.compile !== undefined ? options.compile : true
+        if (Object.keys(primary_key).length !== 1 || Object.keys(secondary_key_range).length !== 1 || Object.values(secondary_key_range)[0].length !== 2) {
+            return undefined
+        }
+        const converted_primary_value = convert_input(Object.values(primary_key)[0])
+        const converted_secondary_range_start_value = convert_input(Object.values(secondary_key_range)[0][0])
+        const converted_secondary_range_end_value = convert_input(Object.values(secondary_key_range)[0][1])
+        if (converted_primary_value === undefined || converted_secondary_range_start_value === undefined || converted_secondary_range_end_value === undefined) {
+            return undefined
+        }
+        const request : QueryCommandInput = {
+            TableName : table,
+            ExpressionAttributeNames: {
+                "#a": Object.keys(primary_key)[0],
+                "#b": Object.keys(secondary_key_range)[0]
+            },
+            ExpressionAttributeValues: {
+                ":a": converted_primary_value,
+                ":b1": converted_secondary_range_start_value,
+                ":b2": converted_secondary_range_end_value
+            },
+            KeyConditionExpression: "#a = :a AND (#b BETWEEN :b1 AND :b2)",
+            ScanIndexForward : !reverse
+        }
+        return await compile_pages(request, (request) => this.dynamodb.query(request), compile)
+    }
+
+    async set(table : string, key : Record<string, any>, attributes : Record<string, any>) : Promise<void> {
+        const converted_key = convert_input(key)?.M
+
+        const request : UpdateItemCommandInput = {
+            TableName : table,
+            Key : converted_key,
+            UpdateExpression: "set " + Object.keys(attributes)
+                .filter(attribute_name => !Object.keys(key).includes(attribute_name))
+                .filter(attribute_name => attributes[attribute_name] !== undefined)
+                .map(attribute_name => "#" + attribute_name + " = :" + attribute_name
+            ).join(", "),
+            ExpressionAttributeNames: Object.fromEntries(Object.keys(attributes)
+                .filter(attribute_name => !Object.keys(key).includes(attribute_name))
+                .filter(attribute_name => attributes[attribute_name] !== undefined)
+                .map(attribute_name => ["#" + attribute_name, attribute_name]
+            )),
+            ExpressionAttributeValues: Object.fromEntries(Object.entries(attributes)
+                .filter(([attribute_name, attribute_value]) => !Object.keys(key).includes(attribute_name) && not_undefined(attribute_value))
+                .map(([attribute_name, attribute_value]) => [":" + attribute_name, convert_input(attribute_value)])
+                .filter(([attribute_name, attribute_value]) => not_undefined(attribute_value))
+            )
+        }
+        await this.dynamodb.updateItem(request)
+    }
+
+    async append(table : string, key : Record<string, any>, attributes : Record<string, any[]>) : Promise<void>{
+        const converted_key = convert_input(key)?.M
+
+        const request : UpdateItemCommandInput = {
+            TableName : table,
+            Key : converted_key,
+            UpdateExpression: "set " + Object.keys(attributes)
+                .filter(attribute_name => attributes[attribute_name] !== undefined)
+                .map(attribute_name => "#" + attribute_name + " = list_append(#" + attribute_name + ", :" + attribute_name + ")").join(", "),
+            ExpressionAttributeNames: Object.fromEntries(Object.keys(attributes)
+                .filter(attribute_name => attributes[attribute_name] !== undefined)
+                .map(attribute_name => ["#" + attribute_name, attribute_name]
+            )),
+            ExpressionAttributeValues: Object.fromEntries(Object.entries(attributes)
+                .filter(([attribute_name, attribute_value]) => not_undefined(attribute_value))
+                .map(([attribute_name, attribute_value]) => [":" + attribute_name, convert_input(attribute_value)])
+                .filter(([attribute_name, attribute_value]) => not_undefined(attribute_value))
+            )
+        }
+        this.dynamodb.updateItem(request)
+    }
+
+    async add(table : string, key : Record<string, any>, attributes : Record<string, any[]>) : Promise<void> {
+        const item = await this.get(table, key, true)
+        if (item === undefined) {
+            return
+        }
+        const new_attributes : Record<string, any[]> = {}
+        for (const [attribute, values] of Object.entries(attributes)) {
+            if (item[attribute] === undefined) {
+                continue
+            }
+            const new_values = values.filter(value => !item[attribute].includes(value))
+            if (new_values.length > 0) {
+                new_attributes[attribute] = new_values
+            }
+        }
+        if (Object.values(new_attributes).flat().length === 0) {
+            return undefined
+        }
+        return await this.append(table, key, attributes)
+    }
+
+    async remove(table : string, key : Record<string, any>, attributes : string[]) : Promise<void> {
+        const converted_key = convert_input(key)?.M
+
+        const request : UpdateItemCommandInput = {
+            TableName : table,
+            Key : converted_key,
+            UpdateExpression: "remove " + attributes
+                .map(attribute_name => "#" + attribute_name).join(", "),
+            ExpressionAttributeNames: Object.fromEntries(attributes
+                .map(attribute_name => ["#" + attribute_name, attribute_name]
+            ))
+        }
+        await this.dynamodb.updateItem(request)
+    }
+
+    async create(table : string, key : Record<string, any>, attributes : Record<string, any[]> = {}) : Promise<void> {
+        const item = await this.get(table, key, true)
+        if (item !== undefined) {
+            return
+        }
+        const converted_key = convert_input(key)?.M
+        const converted_attributes = convert_input(attributes)?.M
+        await this.dynamodb.putItem({
+            TableName : table,
+            Item : { ...converted_key, ...converted_attributes }
+        })
+    }
+
+    async delete(table : string, key : Record<string, any>) : Promise<void> {
+        const converted_key = convert_input(key)?.M
+        await this.dynamodb.deleteItem({
+            TableName: table,
+            Key: converted_key
+        })
+    }
+
+    async duplicate_attribute(table : string, attribute_name : string, new_attribute_name : string) : Promise<void>{
+        const table_metadata = await this.dynamodb.describeTable({
+            TableName : table
+        })
+        if (table_metadata.Table === undefined || table_metadata.Table.KeySchema === undefined) {
+            return
+        }
+        const table_key_names = table_metadata.Table.KeySchema.map(key => key.AttributeName)
+            .filter(table_key_name => table_key_name !== undefined)
+        const items = await this.scan(table, { attribute_names : table_key_names.concat([attribute_name, new_attribute_name]) })
+        if (items.filter(item => item[new_attribute_name] !== undefined).length > 0) {
+            console.log("Cannot rename.", new_attribute_name, "is an existing item.")
+            return
+        }
+        for (const item of items) {
+            if (item[attribute_name] === undefined) {
+                continue
+            }
+            const key = Object.fromEntries(table_key_names.map(key_name => [key_name, item[key_name]]))
+            await this.set(table, key, { [new_attribute_name] : item[attribute_name] })
+        }
+    }
+
+    async remove_attribute(table : string, attribute_name : string) {
+        const table_metadata = await this.dynamodb.describeTable({
+            TableName : table
+        })
+        if (table_metadata.Table === undefined || table_metadata.Table.KeySchema === undefined) {
+            return
+        }
+        const table_key_names = table_metadata.Table.KeySchema.map(key => key.AttributeName)
+            .filter(table_key_name => table_key_name !== undefined)
+        const items = await this.scan(table)
+            .then(items => items.filter(item => item[attribute_name] !== undefined))
+        for (const item of items) {
+            const key = Object.fromEntries(table_key_names.map(key_name => [key_name, item[key_name]]))
+            await this.remove(table, key, [attribute_name])
+        }
+    }
+}