qstd 0.2.26 → 0.2.28

package/dist/server/index.cjs

@@ -2,6 +2,8 @@
 
 var dateFns = require('date-fns');
 var fs = require('fs');
+var libDynamodb = require('@aws-sdk/lib-dynamodb');
+var clientDynamodb = require('@aws-sdk/client-dynamodb');
 
 var __defProp = Object.defineProperty;
 var __export = (target, all) => {
@@ -607,8 +609,8 @@ var readFile = async (filePath, encoding = "utf-8") => {
 var writeBufferToFile = async (filepath, buffer) => {
   if (!buffer) return;
   const chunks = [];
-  for await (const chunk2 of buffer) {
-    chunks.push(chunk2);
+  for await (const chunk3 of buffer) {
+    chunks.push(chunk3);
   }
   await fs.promises.writeFile(filepath, Buffer.concat(chunks));
 };
@@ -616,6 +618,603 @@ var writeFile = async (filepath, content) => {
   return fs.promises.writeFile(filepath, content);
 };
 
+// src/server/aws/ddb/index.ts
+var ddb_exports = {};
+__export(ddb_exports, {
+  batchDelete: () => batchDelete,
+  batchGet: () => batchGet,
+  batchWrite: () => batchWrite,
+  create: () => create2,
+  deleteTable: () => deleteTable,
+  find: () => find,
+  lsi: () => lsi,
+  lsi2: () => lsi2,
+  lsiNormalized: () => lsiNormalized,
+  lsiPhash: () => lsiPhash,
+  lsiUsername: () => lsiUsername,
+  remove: () => remove,
+  save: () => save,
+  tableExists: () => tableExists
+});
+
+// src/server/aws/ddb/fns.ts
+var validateFindProps = (props) => {
+  const isScan = "scan" in props && props.scan === true;
+  if (!props.tableName) {
+    throw new Error(`[ddb] "tableName" is required`);
+  }
+  if (props.limit && props.recursive) {
+    throw new Error(`[ddb] "limit" and "recursive" cannot be used together`);
+  }
+  if (!isScan) {
+    const queryProps = props;
+    if (!queryProps.pk) {
+      throw new Error(`[ddb] [find] "pk" is required for Query mode. Use scan: true to scan without pk.`);
+    }
+    if (queryProps.sk && "key" in queryProps.sk && queryProps.sk.key && queryProps.sk.key !== "sk" && !queryProps.indexName) {
+      throw new Error(
+        `[ddb] [find] you provided a custom sk but no indexName. If this is a mistake, change this error to a warn.`
+      );
+    }
+  }
+};
+var buildKeyConditionExpression = (pk, sk, names, values) => {
+  const pkKey = pk.key ?? "pk";
+  names["#pk"] = pkKey;
+  values[":pk"] = pk.value;
+  if (!sk) return "#pk = :pk";
+  const skName = sk.key ?? "sk";
+  names["#sk"] = skName;
+  if ("op" in sk && sk.op) {
+    switch (sk.op) {
+      case "=":
+      case ">=":
+      case ">":
+      case "<=":
+      case "<":
+        values[":sk"] = sk.value;
+        return `#pk = :pk AND #sk ${sk.op} :sk`;
+      case "begins_with":
+        values[":sk"] = sk.value;
+        return `#pk = :pk AND begins_with(#sk, :sk)`;
+      case "between": {
+        const [from, to] = sk.value;
+        values[":skFrom"] = from;
+        values[":skTo"] = to;
+        return `#pk = :pk AND #sk BETWEEN :skFrom AND :skTo`;
+      }
+      default:
+        throw new Error(`Unsupported SK op: ${JSON.stringify(sk, null, 2)}`);
+    }
+  }
+  if ("value" in sk && sk.value !== void 0) {
+    values[":sk"] = sk.value;
+    return `#pk = :pk AND #sk = :sk`;
+  }
+  if ("valueBeginsWith" in sk && sk.valueBeginsWith !== void 0) {
+    values[":sk"] = sk.valueBeginsWith;
+    return `#pk = :pk AND begins_with(#sk, :sk)`;
+  }
+  throw new Error(
+    `Invalid SK condition: expected 'op' or legacy 'value'/'valueBeginsWith'`
+  );
+};
+var buildFilterExpression = (filters, names, values) => {
+  if (!filters || filters.length === 0) return void 0;
+  const frags = [];
+  filters.forEach((f, i) => {
+    const nameToken = `#f${i}`;
+    names[nameToken] = f.key;
+    switch (f.op) {
+      case "attribute_exists":
+        frags.push(`attribute_exists(${nameToken})`);
+        break;
+      case "attribute_not_exists":
+        frags.push(`attribute_not_exists(${nameToken})`);
+        break;
+      case "between": {
+        const [lo, hi] = f.value;
+        const loPh = `:f${i}_lo`;
+        const hiPh = `:f${i}_hi`;
+        values[loPh] = lo;
+        values[hiPh] = hi;
+        frags.push(`${nameToken} BETWEEN ${loPh} AND ${hiPh}`);
+        break;
+      }
+      case "in": {
+        const arr = f.value;
+        if (!arr.length) {
+          throw new Error(
+            `'in' filter for ${f.key} requires at least one value`
+          );
+        }
+        const phs = arr.map((_, j) => `:f${i}_${j}`);
+        arr.forEach((v, j) => {
+          const ph = phs[j];
+          if (ph) values[ph] = v;
+        });
+        frags.push(`${nameToken} IN (${phs.join(", ")})`);
+        break;
+      }
+      case "contains":
+      case "begins_with": {
+        const ph = `:f${i}`;
+        values[ph] = f.value;
+        frags.push(`${f.op}(${nameToken}, ${ph})`);
+        break;
+      }
+      case "=":
+      case "<>":
+      case ">":
+      case ">=":
+      case "<":
+      case "<=": {
+        const ph = `:f${i}`;
+        values[ph] = f.value;
+        frags.push(`${nameToken} ${f.op} ${ph}`);
+        break;
+      }
+    }
+  });
+  return frags.join(" AND ");
+};
+var buildProjectionExpression = (projection, names) => {
+  const projectionNames = projection.map((key, idx) => {
+    const nameKey = `#proj${idx}`;
+    names[nameKey] = key;
+    return nameKey;
+  });
+  return projectionNames.join(", ");
+};
+var sleep3 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+var backoffDelay = (attempt) => {
+  return Math.min(1e3 * Math.pow(2, attempt), 1e4);
+};
+var chunk2 = (arr, size) => {
+  return Array.from(
+    { length: Math.ceil(arr.length / size) },
+    (_, i) => arr.slice(i * size, i * size + size)
+  );
+};
+
+// src/server/aws/ddb/domain.ts
+var create2 = (props) => {
+  const tableName = props?.tableName;
+  const credentials = props?.credentials;
+  const client = libDynamodb.DynamoDBDocumentClient.from(
+    new clientDynamodb.DynamoDBClient({
+      ...credentials && { credentials },
+      region: "us-east-1"
+    }),
+    {
+      marshallOptions: {
+        // Whether to automatically convert empty strings, blobs, and sets to `null`.
+        convertEmptyValues: false,
+        // false, by default.
+        // Whether to remove undefined values while marshalling.
+        removeUndefinedValues: false,
+        // false, by default.
+        // Whether to convert typeof object to map attribute.
+        convertClassInstanceToMap: false
+        // false, by default.
+      },
+      unmarshallOptions: {
+        // Whether to return numbers as a string instead of converting them to native JavaScript numbers.
+        wrapNumbers: false
+        // false, by default.
+      }
+    }
+  );
+  return { client, tableName };
+};
+async function find(ddb, props) {
+  try {
+    const TableName = props.tableName ?? ddb.tableName;
+    const { filters, projection, debug } = props;
+    const isScan = "scan" in props && props.scan === true;
+    validateFindProps(props);
+    const names = {};
+    const values = {};
+    const KeyConditionExpression = isScan ? void 0 : buildKeyConditionExpression(
+      props.pk,
+      props.sk,
+      names,
+      values
+    );
+    const FilterExpression = filters ? buildFilterExpression(filters, names, values) : void 0;
+    if (debug) {
+      console.log(`[debug] [ddb] [find] input:`, {
+        isScan,
+        filters,
+        FilterExpression,
+        names,
+        values
+      });
+    }
+    const ProjectionExpression = projection ? buildProjectionExpression(projection, names) : void 0;
+    const all = [];
+    let startKey = props.startKey;
+    let pageCount = 0;
+    let totalItems = 0;
+    do {
+      if (props.maxPages && pageCount >= props.maxPages) break;
+      const command = isScan ? new libDynamodb.ScanCommand({
+        TableName,
+        IndexName: props.indexName,
+        FilterExpression,
+        Limit: props.limit,
+        ProjectionExpression,
+        ExclusiveStartKey: startKey,
+        ConsistentRead: props.strong,
+        ...Object.keys(names).length > 0 && {
+          ExpressionAttributeNames: names
+        },
+        ...Object.keys(values).length > 0 && {
+          ExpressionAttributeValues: values
+        }
+      }) : new libDynamodb.QueryCommand({
+        TableName,
+        IndexName: props.indexName,
+        KeyConditionExpression,
+        FilterExpression,
+        Limit: props.limit,
+        ProjectionExpression,
+        ExclusiveStartKey: startKey,
+        ConsistentRead: props.strong,
+        ExpressionAttributeNames: names,
+        ExpressionAttributeValues: values,
+        ScanIndexForward: props.sort === "desc" ? false : true
+      });
+      const result = await ddb.client.send(command);
+      const pageItems = result.Items ?? [];
+      all.push(...pageItems);
+      totalItems += pageItems.length;
+      pageCount++;
+      startKey = result.LastEvaluatedKey;
+      if (props.recursive) {
+        if (props.maxItems && totalItems >= props.maxItems) break;
+        if (!startKey) break;
+        if (typeof props.recursive === "function") {
+          const page = {
+            lastEvaluatedKey: result.LastEvaluatedKey,
+            scannedCount: result.ScannedCount ?? 0,
+            count: result.Count ?? 0,
+            items: pageItems
+          };
+          const shouldContinue = props.recursive(page, pageCount, totalItems);
+          if (!shouldContinue) break;
+        }
+      } else {
+        break;
+      }
+    } while (true);
+    const rawResponse = {
+      lastEvaluatedKey: startKey,
+      scannedCount: totalItems,
+      count: totalItems,
+      items: all
+    };
+    if (props.raw && props.first) {
+      return {
+        count: rawResponse.count,
+        item: rawResponse.items[0],
+        scannedCount: rawResponse.scannedCount
+      };
+    }
+    if (props.raw) return rawResponse;
+    if (props.first) return rawResponse.items[0];
+    return rawResponse.items;
+  } catch (error2) {
+    const err = error2;
+    console.log(`[error] [ddb] [find] failed with ${err.message}. Input:`);
+    console.dir(props, { depth: 100 });
+    throw err;
+  }
+}
+var remove = async (ddb, props) => {
+  const TableName = props.tableName ?? ddb.tableName;
+  const command = new libDynamodb.DeleteCommand({ Key: props.key, TableName });
+  return ddb.client.send(command);
+};
+var save = async (ddb, props) => {
+  const TableName = props.tableName ?? ddb.tableName;
+  const command = new libDynamodb.PutCommand({
+    Item: props.item,
+    TableName
+  });
+  return ddb.client.send(command);
+};
+async function batchGet(ddb, props) {
+  const { keys, maxRetries = 3, strong = false } = props;
+  const TableName = props.tableName ?? ddb.tableName;
+  const chunks = chunk2(keys, 100);
+  const allItems = [];
+  const requestedCount = keys.length;
+  let totalConsumedCapacity = 0;
+  for (const chunk3 of chunks) {
+    let attempt = 0;
+    let keysToFetch = chunk3;
+    while (keysToFetch.length > 0 && attempt <= maxRetries) {
+      if (attempt > 0) {
+        const delay = backoffDelay(attempt);
+        console.log(
+          `[info] [ddb] [batchGet]: Retrying ${keysToFetch.length} keys (attempt ${attempt}/${maxRetries}) after ${delay}ms`
+        );
+        await sleep3(delay);
+      }
+      try {
+        const command = new libDynamodb.BatchGetCommand({
+          RequestItems: {
+            [TableName]: { Keys: keysToFetch, ConsistentRead: strong }
+          }
+        });
+        const result = await ddb.client.send(command);
+        const items = result.Responses?.[TableName] ?? [];
+        allItems.push(...items);
+        if (result.ConsumedCapacity) {
+          totalConsumedCapacity += result.ConsumedCapacity.reduce(
+            (sum, cap) => sum + (cap.CapacityUnits ?? 0),
+            0
+          );
+        }
+        const unprocessed = result.UnprocessedKeys?.[TableName]?.Keys;
+        if (unprocessed && unprocessed.length > 0) {
+          keysToFetch = unprocessed;
+          attempt++;
+        } else {
+          keysToFetch = [];
+        }
+      } catch (error2) {
+        console.log(
+          `[error] [ddb] [batchGet]: Error fetching chunk (attempt ${attempt}/${maxRetries}):`,
+          error2
+        );
+        if (attempt >= maxRetries) throw error2;
+        attempt++;
+      }
+    }
+    if (keysToFetch.length > 0) {
+      console.log(
+        `[error] [ddb] [batchGet]: Failed to fetch ${keysToFetch.length} keys after ${maxRetries} retries`
+      );
+    }
+  }
+  const missing = requestedCount - allItems.length;
+  return {
+    missing,
+    items: allItems,
+    count: allItems.length,
+    consumedCapacity: totalConsumedCapacity || void 0
+  };
+}
+async function batchWrite(ddb, props) {
+  const { maxRetries = 3 } = props;
+  const TableName = props.tableName ?? ddb.tableName;
+  const items = props.transform ? props.items.map(props.transform) : props.items;
+  const hasConditions = items.some((x) => x.cond);
+  const chunkSize = hasConditions ? 100 : 25;
+  const chunks = chunk2(items, chunkSize);
+  let processedCount = 0;
+  let failedCount = 0;
+  let totalConsumedCapacity = 0;
+  for (const chunk3 of chunks) {
+    let attempt = 0;
+    let itemsToWrite = chunk3;
+    while (itemsToWrite.length > 0 && attempt <= maxRetries) {
+      if (attempt > 0) {
+        const delay = backoffDelay(attempt);
+        console.log(
+          `[info] [ddb] [batchWrite]: Retrying ${itemsToWrite.length} items (attempt ${attempt}/${maxRetries}) after ${delay}ms`
+        );
+        await sleep3(delay);
+      }
+      try {
+        if (hasConditions) {
+          const transactItems = itemsToWrite.map((x) => ({
+            Put: {
+              TableName,
+              Item: x.item,
+              ...x.cond && { ConditionExpression: x.cond }
+            }
+          }));
+          const command = new libDynamodb.TransactWriteCommand({
+            TransactItems: transactItems
+          });
+          const result = await ddb.client.send(command);
+          if (result.ConsumedCapacity) {
+            totalConsumedCapacity += result.ConsumedCapacity.reduce(
+              (sum, cap) => sum + (cap.CapacityUnits ?? 0),
+              0
+            );
+          }
+          processedCount += itemsToWrite.length;
+          itemsToWrite = [];
+        } else {
+          const writeRequests = itemsToWrite.map((x) => ({
+            PutRequest: { Item: x.item }
+          }));
+          const command = new libDynamodb.BatchWriteCommand({
+            RequestItems: { [TableName]: writeRequests }
+          });
+          const result = await ddb.client.send(command);
+          if (result.ConsumedCapacity) {
+            totalConsumedCapacity += result.ConsumedCapacity.reduce(
+              (sum, cap) => sum + (cap.CapacityUnits ?? 0),
+              0
+            );
+          }
+          const unprocessed = result.UnprocessedItems?.[TableName];
+          if (unprocessed && unprocessed.length > 0) {
+            itemsToWrite = unprocessed.map((req) => ({
+              item: req.PutRequest.Item
+            }));
+            processedCount += chunk3.length - itemsToWrite.length;
+            attempt++;
+          } else {
+            processedCount += itemsToWrite.length;
+            itemsToWrite = [];
+          }
+        }
+      } catch (error2) {
+        console.log(
+          `[error] [ddb] [batchWrite]: Error writing chunk (attempt ${attempt}/${maxRetries}):`,
+          error2
+        );
+        if (attempt >= maxRetries) {
+          failedCount += itemsToWrite.length;
+          itemsToWrite = [];
+        } else {
+          attempt++;
+        }
+      }
+    }
+    if (itemsToWrite.length > 0) {
+      console.log(
+        `[error] [ddb] [batchWrite]: Failed to write ${itemsToWrite.length} items after ${maxRetries} retries`
+      );
+      failedCount += itemsToWrite.length;
+    }
+  }
+  return {
+    failed: failedCount,
+    processed: processedCount,
+    consumedCapacity: totalConsumedCapacity || void 0
+  };
+}
+async function batchDelete(ddb, props) {
+  const { maxRetries = 3 } = props;
+  const TableName = props.tableName ?? ddb.tableName;
+  const keys = props.transform ? props.keys.map(props.transform) : props.keys;
+  const hasConditions = keys.some((x) => x.cond);
+  const chunkSize = hasConditions ? 100 : 25;
+  const chunks = chunk2(keys, chunkSize);
+  let processedCount = 0;
+  let failedCount = 0;
+  let totalConsumedCapacity = 0;
+  for (const chunk3 of chunks) {
+    let attempt = 0;
+    let itemsToDelete = chunk3;
+    while (itemsToDelete.length > 0 && attempt <= maxRetries) {
+      if (attempt > 0) {
+        const delay = backoffDelay(attempt);
+        console.log(
+          `[info] [ddb] [batchDelete]: Retrying ${itemsToDelete.length} items (attempt ${attempt}/${maxRetries}) after ${delay}ms`
+        );
+        await sleep3(delay);
+      }
+      try {
+        if (hasConditions) {
+          const transactItems = itemsToDelete.map((x) => ({
+            Delete: {
+              TableName,
+              Key: x.key,
+              ...x.cond && { ConditionExpression: x.cond }
+            }
+          }));
+          const command = new libDynamodb.TransactWriteCommand({
+            TransactItems: transactItems
+          });
+          const result = await ddb.client.send(command);
+          if (result.ConsumedCapacity) {
+            totalConsumedCapacity += result.ConsumedCapacity.reduce(
+              (sum, cap) => sum + (cap.CapacityUnits ?? 0),
+              0
+            );
+          }
+          processedCount += itemsToDelete.length;
+          itemsToDelete = [];
+        } else {
+          const writeRequests = itemsToDelete.map((x) => ({
+            DeleteRequest: { Key: x.key }
+          }));
+          const command = new libDynamodb.BatchWriteCommand({
+            RequestItems: { [TableName]: writeRequests }
+          });
+          const result = await ddb.client.send(command);
+          if (result.ConsumedCapacity) {
+            totalConsumedCapacity += result.ConsumedCapacity.reduce(
+              (sum, cap) => sum + (cap.CapacityUnits ?? 0),
+              0
+            );
+          }
+          const unprocessed = result.UnprocessedItems?.[TableName];
+          if (unprocessed && unprocessed.length > 0) {
+            itemsToDelete = unprocessed.map((req) => ({
+              key: req.DeleteRequest.Key
+            }));
+            processedCount += chunk3.length - itemsToDelete.length;
+            attempt++;
+          } else {
+            processedCount += itemsToDelete.length;
+            itemsToDelete = [];
+          }
+        }
+      } catch (error2) {
+        console.log(
+          `[error] [ddb] [batchDelete]: Error deleting chunk (attempt ${attempt}/${maxRetries}):`,
+          error2
+        );
+        if (attempt >= maxRetries) {
+          failedCount += itemsToDelete.length;
+          itemsToDelete = [];
+        } else {
+          attempt++;
+        }
+      }
+    }
+    if (itemsToDelete.length > 0) {
+      console.log(
+        `[error] [ddb] [batchDelete]: Failed to delete ${itemsToDelete.length} items after ${maxRetries} retries`
+      );
+      failedCount += itemsToDelete.length;
+    }
+  }
+  return {
+    failed: failedCount,
+    processed: processedCount,
+    consumedCapacity: totalConsumedCapacity || void 0
+  };
+}
+var deleteTable = (ddb, tableName) => {
+  const TableName = tableName;
+  const command = new clientDynamodb.DeleteTableCommand({ TableName });
+  return ddb.client.send(command);
+};
+var tableExists = async (ddb, props) => {
+  try {
+    const TableName = props.tableName;
+    const command = new clientDynamodb.DescribeTableCommand({ TableName });
+    await ddb.client.send(command);
+    return true;
+  } catch (err) {
+    if (err instanceof clientDynamodb.DynamoDBServiceException) {
+      if (err.name === "ResourceNotFoundException" || err.$metadata.httpStatusCode === 400) {
+        return false;
+      }
+    }
+    throw err;
+  }
+};
+
+// src/server/aws/ddb/literals.ts
+var lsi = { name: "lsi", sk: "lsi" };
+var lsi2 = { name: "lsi2", sk: "lsi2" };
+var lsiUsername = {
+  name: "username-lsi",
+  sk: "username"
+};
+var lsiNormalized = {
+  /** use in index_name */
+  name: "normalized-lsi",
+  sk: "normalized"
+};
+var lsiPhash = {
+  /** use in index_name */
+  name: "phash-lsi",
+  sk: "phash"
+};
+
+exports.DDB = ddb_exports;
 exports.Dict = dict_exports;
 exports.File = file_exports;
 exports.Flow = flow_exports;

package/dist/server/index.d.ts

@@ -8,4 +8,5 @@ export * as Flow from "../shared/flow";
 export * as Random from "../shared/random";
 export * as Log from "../shared/log";
 export * as File from "./file";
+export * as DDB from "./aws/ddb";
 //# sourceMappingURL=index.d.ts.map

package/dist/server/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/server/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,IAAI,MAAM,gBAAgB,CAAC;AACvC,OAAO,KAAK,IAAI,MAAM,gBAAgB,CAAC;AACvC,OAAO,KAAK,GAAG,MAAM,eAAe,CAAC;AACrC,OAAO,KAAK,GAAG,MAAM,eAAe,CAAC;AACrC,OAAO,KAAK,KAAK,MAAM,iBAAiB,CAAC;AACzC,OAAO,KAAK,IAAI,MAAM,gBAAgB,CAAC;AACvC,OAAO,KAAK,IAAI,MAAM,gBAAgB,CAAC;AACvC,OAAO,KAAK,MAAM,MAAM,kBAAkB,CAAC;AAC3C,OAAO,KAAK,GAAG,MAAM,eAAe,CAAC;AAGrC,OAAO,KAAK,IAAI,MAAM,QAAQ,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/server/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,IAAI,MAAM,gBAAgB,CAAC;AACvC,OAAO,KAAK,IAAI,MAAM,gBAAgB,CAAC;AACvC,OAAO,KAAK,GAAG,MAAM,eAAe,CAAC;AACrC,OAAO,KAAK,GAAG,MAAM,eAAe,CAAC;AACrC,OAAO,KAAK,KAAK,MAAM,iBAAiB,CAAC;AACzC,OAAO,KAAK,IAAI,MAAM,gBAAgB,CAAC;AACvC,OAAO,KAAK,IAAI,MAAM,gBAAgB,CAAC;AACvC,OAAO,KAAK,MAAM,MAAM,kBAAkB,CAAC;AAC3C,OAAO,KAAK,GAAG,MAAM,eAAe,CAAC;AAGrC,OAAO,KAAK,IAAI,MAAM,QAAQ,CAAC;AAC/B,OAAO,KAAK,GAAG,MAAM,WAAW,CAAC"}