s3-querier 1.2.1 → 1.2.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "s3-querier",
-  "version": "1.2.1",
+  "version": "1.2.3",
   "description": "Query S3-compatible storage with DuckDB and SQL",
   "type": "module",
   "main": "src/s3-querier.js",

package/src/s3/s3.js

@@ -1,6 +1,6 @@
 import fsPromise from 'node:fs/promises';
 import { dirname } from 'node:path';
-import { S3Client, ListObjectsV2Command, GetObjectCommand } from '@aws-sdk/client-s3';
+import { S3Client, paginateListObjectsV2, GetObjectCommand } from '@aws-sdk/client-s3';
 
 import { logger } from '../utils/logger.js';
 import { datesInRange, hoursInRange, monthsInRange, buildPath } from '../utils/file-path-builder/file-path-builder.js';
@@ -103,8 +103,7 @@ export default class S3 {
 
     return Promise.allSettled(listPromises).then((results) => {
       const regex = regexFromPattern(file);
-      const todayPrefix = this.getTodayPrefix(file);
-      this.listingCache.delete(`${this.bucket}/${todayPrefix}`);
+      this.evictTodayFromListingCache(file);
 
       return results
         .filter((result) => result.status === 'fulfilled')
@@ -225,6 +224,19 @@ export default class S3 {
     return buildPath(`${trimmed}{dd}`, new Date());
   }
 
+  /**
+   * Removes all listing cache entries whose key falls under today's day prefix.
+   * Covers day-level, hour-level, and minute-level cache keys in one pass.
+   *
+   * @param {string} filePattern File pattern
+   */
+  evictTodayFromListingCache(filePattern) {
+    const todayPrefix = `${this.bucket}/${this.getTodayPrefix(filePattern)}`;
+    [...this.listingCache.keys()]
+      .filter((key) => key.startsWith(todayPrefix))
+      .forEach((key) => this.listingCache.delete(key));
+  }
+
   /**
    * Returns a list of files from S3 under the given prefix
    *
@@ -238,16 +250,9 @@ export default class S3 {
     }
 
     const files = [];
-    let continuationToken;
-    do {
-      const response = await this.s3.send(
-        new ListObjectsV2Command({ Bucket: this.bucket, Prefix: prefix, ContinuationToken: continuationToken }),
-      );
-      response.Contents?.forEach((content) => {
-        files.push({ file: content.Key, size: content.Size });
-      });
-      continuationToken = response.NextContinuationToken;
-    } while (continuationToken);
+    for await (const page of paginateListObjectsV2({ client: this.s3 }, { Bucket: this.bucket, Prefix: prefix })) {
+      page.Contents?.forEach((content) => files.push({ file: content.Key, size: content.Size }));
+    }
 
     this.listingCache.set(cacheKey, files);
     return files;

package/src/utils/logger.js

@@ -1,3 +1,3 @@
 import pino from 'pino';
-const logger = pino();
+const logger = pino({}, process.stderr);
 export { logger };