s3-querier 1.2.2 → 1.3.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "s3-querier",
-  "version": "1.2.2",
+  "version": "1.3.0",
   "description": "Query S3-compatible storage with DuckDB and SQL",
   "type": "module",
   "main": "src/s3-querier.js",
@@ -51,7 +51,7 @@
     "@duckdb/node-api": "^1.5.3-r.3",
     "@modelcontextprotocol/sdk": "^1.29.0",
     "avsc": "^5.7.7",
-    "date-fns": "^4.0.0",
+    "glob": "^13.0.6",
     "hyparquet": "^1.26.0",
     "lru-cache": "^11.0.0",
     "peggy": "^5.1.0",

package/src/duck-db/index.js

@@ -20,19 +20,13 @@ const formatStrategies = {
  */
 export async function query(sql, options = {}) {
   const { format } = options;
-  const queryStart = new Date();
-
   try {
     const connection = await db.connect();
     const reader = await connection.runAndReadAll(sql);
     const columnsResult = reader.getColumnsObjectJS();
 
     const formatter = formatStrategies[format] ?? formatStrategies.default;
-    const result = formatter(columnsResult);
-
-    const queryTime = new Date() - queryStart;
-    logger.info(`Query completed in : ${queryTime / 1000} seconds`);
-    return result ?? [];
+    return formatter(columnsResult) ?? [];
   } catch (error) {
     logger.error(error);
     throw error;

package/src/mcp/tools/query/query.js

@@ -2,7 +2,8 @@ import { readFileSync } from 'node:fs';
 import { z } from 'zod';
 
 import BaseTool from '../base-tool.js';
-import s3Querier, { bigintReplacer } from '../../../s3-querier.js';
+import s3Querier, { bigintReplacer, FSPurgePlugin, StatsPlugin } from '../../../s3-querier.js';
+import { logger } from '../../../utils/logger.js';
 
 const {
   S3_ACCESS_KEY_ID,
@@ -13,6 +14,9 @@ const {
   S3_BUCKETS_DIR = '/tmp/s3-querier',
 } = process.env;
 
+const purgePlugin = new FSPurgePlugin({ bucketsDir: S3_BUCKETS_DIR });
+const statsPlugin = new StatsPlugin((event) => logger.info(event));
+
 const sqlDescription = readFileSync(new URL('../../descriptions/sql-param.md', import.meta.url), 'utf8');
 const toolDescription = readFileSync(new URL('../../descriptions/tool.md', import.meta.url), 'utf8');
 
@@ -59,6 +63,7 @@ export default class QueryTool extends BaseTool {
       accessKeyId: S3_ACCESS_KEY_ID,
       secretAccessKey: S3_SECRET_ACCESS_KEY,
       format: 'jsonRecords',
+      plugins: [purgePlugin, statsPlugin],
     });
 
     return {

package/src/plugins/fs-purge/fs-purge-plugin.js

@@ -0,0 +1,21 @@
+import FSPurge from '../../utils/fs-purge/fs-purge.js';
+
+export default class FSPurgePlugin {
+  /**
+   * @param {object} options
+   * @param {string} options.bucketsDir Local directory where S3 files are cached
+   * @param {number} [options.lastAccessTTLMinutes=60] Minutes since last access before a file is eligible for purge
+   * @param {number} [options.refreshIntervalMin=60] Minimum minutes between sweeps
+   */
+  constructor({ bucketsDir, lastAccessTTLMinutes = 60, refreshIntervalMin = 60 }) {
+    this.purger = new FSPurge({ pattern: `${bucketsDir}/**`, lastAccessTTLMinutes, refreshIntervalMin });
+  }
+
+  processQuery(context) {
+    return context;
+  }
+
+  postQuery() {
+    this.purger.sweep();
+  }
+}

package/src/plugins/lifecycle.js

@@ -0,0 +1,124 @@
+import { logger } from '../utils/logger.js';
+import { mergeSettings } from '../utils/file-settings/file-settings.js';
+
+/**
+ * Passes the query through each plugin's `processQuery` hook and merges the
+ * resulting file download settings.
+ *
+ * @param {object[]} plugins - Plugin instances.
+ * @param {object} context - `{ query, endpoint, defaultBucket, bucketsDir }`.
+ * @returns {{ query: string, fileSettings: object[], settings: object[], ... }}
+ */
+export function processQuery(plugins = [], { query = '', endpoint, defaultBucket, bucketsDir }) {
+  const processedQuery = plugins.reduce((result, plugin) => plugin.processQuery(result), {
+    endpoint,
+    defaultBucket,
+    bucketsDir,
+    query,
+    settings: [],
+  });
+  const fileSettings = processedQuery.settings;
+  processedQuery.settings = mergeSettings(processedQuery.settings);
+  return { ...processedQuery, fileSettings };
+}
+
+/**
+ * Passes the raw query through each plugin's `finalizeQuery` hook, substituting
+ * exact downloaded paths in place of glob patterns.
+ *
+ * @param {object} params
+ * @param {object[]} params.plugins - Plugin instances.
+ * @param {string} params.rawQuery - SQL with original file references and date/location tokens.
+ * @param {object[]} params.fileSettings - Pre-merge per-file settings from `processQuery`.
+ * @param {string[]} params.downloadedPaths - Absolute local paths of all downloaded files.
+ * @param {string} params.bucketsDir - Root directory where files are cached locally.
+ * @returns {string} Finalized SQL ready for DuckDB execution.
+ */
+export function runFinalizers({ plugins, rawQuery, fileSettings, downloadedPaths, bucketsDir }) {
+  return plugins.reduce((query, plugin) => {
+    if (!plugin.finalizeQuery) return query;
+    return plugin.finalizeQuery(query, fileSettings, downloadedPaths, bucketsDir);
+  }, rawQuery);
+}
+
+/**
+ * Calls each plugin's `preListFiles` hook and collects the returned callbacks.
+ * Plugins that do not implement `preListFiles` contribute a `null` placeholder
+ * so that callback indices stay aligned with plugin indices.
+ *
+ * @param {object[]} plugins - Plugin instances.
+ * @param {object} context - `{ prefix, bucket }`.
+ * @returns {Array<Function|null>} One entry per plugin — a callback or `null`.
+ */
+export function runPreListFiles(plugins, context) {
+  return plugins.map((plugin) => plugin.preListFiles?.(context) ?? null);
+}
+
+/**
+ * Invokes each callback returned by `runPreListFiles`. Errors are logged and
+ * swallowed so a failing plugin never rejects the caller's query result.
+ *
+ * @param {Array<Function|null>} callbacks - Collected from `runPreListFiles`.
+ * @param {object} context - `{ prefix, bucket, files, durationMs, cacheHit }`.
+ */
+export function runPostListFiles(callbacks, context) {
+  callbacks.forEach((cb) => {
+    if (cb) Promise.resolve(cb(context)).catch((error) => logger.error(error));
+  });
+}
+
+/**
+ * Calls each plugin's `preDownloadFiles` hook and collects the returned callbacks.
+ * Plugins that do not implement `preDownloadFiles` contribute a `null` placeholder
+ * so that callback indices stay aligned with plugin indices.
+ *
+ * @param {object[]} plugins - Plugin instances.
+ * @param {object} context - `{ bucket, from, to }`.
+ * @returns {Array<Function|null>} One entry per plugin — a callback or `null`.
+ */
+export function runPreDownloadFiles(plugins, context) {
+  return plugins.map((plugin) => plugin.preDownloadFiles?.(context) ?? null);
+}
+
+/**
+ * Invokes each callback returned by `runPreDownloadFiles`. Errors are logged and
+ * swallowed so a failing plugin never rejects the caller's query result.
+ *
+ * @param {Array<Function|null>} callbacks - Collected from `runPreDownloadFiles`.
+ * @param {object} context - `{ cacheHits, cacheMisses, enqueuedHits, bytesDownloaded, durationMs, bucket }`.
+ */
+export function runPostDownloadFiles(callbacks, context) {
+  callbacks.forEach((cb) => {
+    if (cb) Promise.resolve(cb(context)).catch((error) => logger.error(error));
+  });
+}
+
+/**
+ * Calls each plugin's `preQuery` hook and collects the returned callbacks.
+ * Plugins that do not implement `preQuery` contribute a `null` placeholder
+ * so that callback indices stay aligned with plugin indices.
+ *
+ * @param {object[]} plugins - Plugin instances.
+ * @param {object} context - `{ sql, downloadedPaths, bucketsDir }`.
+ * @returns {Array<Function|null>} One entry per plugin — a callback or `null`.
+ */
+export function runPreQuery(plugins, context) {
+  return plugins.map((plugin) => plugin.preQuery?.(context) ?? null);
+}
+
+/**
+ * Invokes each callback returned by `runPreQuery` and each plugin's `postQuery`
+ * hook. Errors are logged and swallowed so a failing plugin never rejects the
+ * caller's query result.
+ *
+ * @param {object[]} plugins - Plugin instances.
+ * @param {object} context - `{ result, downloadedPaths, bucketsDir }`.
+ * @param {Array<Function|null>} callbacks - Collected from `runPreQuery`.
+ */
+export function runPostQuery(plugins, context, callbacks = []) {
+  plugins.forEach((plugin, index) => {
+    const cb = callbacks[index];
+    if (cb) Promise.resolve(cb(context)).catch((error) => logger.error(error));
+    if (plugin.postQuery) Promise.resolve(plugin.postQuery(context)).catch((error) => logger.error(error));
+  });
+}

package/src/plugins/stats/stats-plugin.js

@@ -0,0 +1,49 @@
+/**
+ * Collects listing, download, and query timing stats and delivers them via a
+ * single callback. Each invocation fires independently — the caller is
+ * responsible for aggregation across events.
+ *
+ * @example
+ * const stats = new StatsPlugin((event) => console.log(event));
+ * // { type: 'listing',  prefix, bucket, fileCount, durationMs, cacheHit }
+ * // { type: 'download', bucket, cacheHits, cacheMisses, bytesDownloaded, durationMs }
+ * // { type: 'query',    sql, durationMs, rowCount }
+ */
+export default class StatsPlugin {
+  constructor(onStats) {
+    this.onStats = onStats;
+  }
+
+  processQuery(context) {
+    return context;
+  }
+
+  preListFiles({ prefix, bucket }) {
+    return ({ files, durationMs, cacheHit }) => {
+      this.onStats({ type: 'listing', prefix, bucket, fileCount: files.length, durationMs, cacheHit });
+    };
+  }
+
+  preDownloadFiles({ bucket, from, to }) {
+    return ({ cacheHits, cacheMisses, enqueuedHits, bytesDownloaded, durationMs }) => {
+      this.onStats({
+        type: 'download',
+        bucket,
+        from,
+        to,
+        cacheHits,
+        cacheMisses,
+        enqueuedHits,
+        bytesDownloaded,
+        durationMs,
+      });
+    };
+  }
+
+  preQuery({ sql }) {
+    const start = Date.now();
+    return ({ result }) => {
+      this.onStats({ type: 'query', sql, durationMs: Date.now() - start, rowCount: result.length });
+    };
+  }
+}

package/src/s3/s3.js

@@ -1,8 +1,9 @@
 import fsPromise from 'node:fs/promises';
 import { dirname } from 'node:path';
-import { S3Client, ListObjectsV2Command, GetObjectCommand } from '@aws-sdk/client-s3';
+import { S3Client, paginateListObjectsV2, GetObjectCommand } from '@aws-sdk/client-s3';
 
 import { logger } from '../utils/logger.js';
+import { runPreListFiles, runPostListFiles, runPreDownloadFiles, runPostDownloadFiles } from '../plugins/lifecycle.js';
 import { datesInRange, hoursInRange, monthsInRange, buildPath } from '../utils/file-path-builder/file-path-builder.js';
 import { regexFromPattern } from '../utils/date-regex/date-regex.js';
 import { buildIbmIamClient } from './auth/ibm-iam-client.js';
@@ -44,16 +45,25 @@ export default class S3 {
    * @returns {PromiseSettledResult<string[]>} Promise result for each file downloaded
    */
   async downloadFiles({ from, to, filePatterns = [], staticFiles = [] }) {
-    const startListing = new Date();
-    const listPromises = filePatterns.map((pattern) => {
-      return this.getFilePathsFromPrefixes(from, to, pattern);
-    });
-    const filePaths = await Promise.allSettled(listPromises).then((fileList) => {
-      return fileList.map((list) => list.value).flat();
+    const listPromises = filePatterns.map((pattern) => this.getFilePathsFromPrefixes(from, to, pattern));
+    const filePaths = await Promise.allSettled(listPromises).then((fileList) =>
+      fileList.map((list) => list.value).flat(),
+    );
+    const stats = { start: new Date(), cacheHits: 0, cacheMisses: 0, enqueuedHits: 0, bytesDownloaded: 0 };
+    const downloadCallbacks = runPreDownloadFiles(this.plugins, { bucket: this.bucket, from, to });
+    const downloadedPaths = await this.downloadFileList([...filePaths, ...staticFiles], stats);
+    const durationMs = new Date() - stats.start;
+
+    runPostDownloadFiles(downloadCallbacks, {
+      cacheHits: stats.cacheHits,
+      cacheMisses: stats.cacheMisses,
+      enqueuedHits: stats.enqueuedHits,
+      bytesDownloaded: stats.bytesDownloaded,
+      durationMs,
+      bucket: this.bucket,
     });
 
-    logger.info(`Total listing time: ${(new Date() - startListing) / 1000}s`);
-    return this.downloadFileList([...filePaths, ...staticFiles]);
+    return downloadedPaths;
   }
 
   /**
@@ -62,30 +72,14 @@ export default class S3 {
    * @param {string[]} filePaths A list of files to download
    * @returns {PromiseSettledResult} A Promise that resolves to an array of file paths
    */
-  downloadFileList(filePaths = []) {
-    logger.info(`Starting downloads for ${filePaths.length} files`);
+  downloadFileList(filePaths = [], stats = { cacheHits: 0, cacheMisses: 0, enqueuedHits: 0, bytesDownloaded: 0 }) {
     this.preFlightCheck(filePaths);
 
-    const stats = {
-      start: new Date(),
-      cacheHits: 0,
-      cacheMisses: 0,
-      enqueuedHits: 0,
-      bytesDownloaded: 0,
-    };
     const filesPromises = this.startDownloads(stats, filePaths);
 
     return Promise.allSettled(filesPromises)
-      .then((results) => {
-        return results
-          .filter((result) => {
-            return result.value;
-          })
-          .map((result) => result.value);
-      })
-      .then(this.logStatistics(stats))
-      .then(this.resetEnqueued)
-      .then((results) => results);
+      .then((results) => results.filter((result) => result.value).map((result) => result.value))
+      .then(this.resetEnqueued);
   }
 
   /**
@@ -245,23 +239,34 @@ export default class S3 {
    */
   async listFiles(prefix) {
     const cacheKey = `${this.bucket}/${prefix}`;
+    const start = new Date();
+    const listCallbacks = runPreListFiles(this.plugins, { prefix, bucket: this.bucket });
+
     if (this.listingCache.has(cacheKey)) {
-      return this.listingCache.get(cacheKey);
+      const files = this.listingCache.get(cacheKey);
+      runPostListFiles(listCallbacks, {
+        prefix,
+        bucket: this.bucket,
+        files,
+        durationMs: new Date() - start,
+        cacheHit: true,
+      });
+      return files;
     }
 
     const files = [];
-    let continuationToken;
-    do {
-      const response = await this.s3.send(
-        new ListObjectsV2Command({ Bucket: this.bucket, Prefix: prefix, ContinuationToken: continuationToken }),
-      );
-      response.Contents?.forEach((content) => {
-        files.push({ file: content.Key, size: content.Size });
-      });
-      continuationToken = response.NextContinuationToken;
-    } while (continuationToken);
+    for await (const page of paginateListObjectsV2({ client: this.s3 }, { Bucket: this.bucket, Prefix: prefix })) {
+      page.Contents?.forEach((content) => files.push({ file: content.Key, size: content.Size }));
+    }
 
     this.listingCache.set(cacheKey, files);
+    runPostListFiles(listCallbacks, {
+      prefix,
+      bucket: this.bucket,
+      files,
+      durationMs: new Date() - start,
+      cacheHit: false,
+    });
     return files;
   }
 
@@ -283,26 +288,6 @@ export default class S3 {
     return fileDLPromises;
   }
 
-  /**
-   * Logs download statistics
-   *
-   * @param {object} stats A statistics object
-   * @returns {(PromiseSettledResult) => PromiseSettledResult}
-   */
-  logStatistics(stats) {
-    return (results) => {
-      const mbDownloaded = stats.bytesDownloaded !== 0 ? stats.bytesDownloaded / (1024 * 1024) : 0;
-      const seconds = (new Date() - stats.start) / 1000;
-      const mbPerSecond = mbDownloaded / seconds;
-
-      logger.info(`Enqueued keys: ${this.enqueuedFiles.size}`);
-      logger.info(
-        `Download completed in: ${seconds} seconds. Cache hits: ${stats.cacheHits}. Cache misses: ${stats.cacheMisses}. Enqueued hits: ${stats.enqueuedHits}. MB downloaded: ${mbDownloaded}. MB/s ${mbPerSecond}`,
-      );
-      return results;
-    };
-  }
-
   /**
    * Starts the download for all files
    *

package/src/s3-querier.js

@@ -2,10 +2,12 @@ import { LRUCache } from 'lru-cache';
 
 import S3 from './s3/s3.js';
 export { bigintReplacer } from './utils/bigint-replacer.js';
-import { mergeSettings } from './utils/file-settings/file-settings.js';
 import { query as execQuery } from './duck-db/index.js';
 import QueryParserPlugin from './plugins/query-parser/query-parser.js';
 import QueryFinalizerPlugin from './plugins/query-finalizer/query-finalizer.js';
+export { default as FSPurgePlugin } from './plugins/fs-purge/fs-purge-plugin.js';
+export { default as StatsPlugin } from './plugins/stats/stats-plugin.js';
+import { processQuery, runFinalizers, runPreQuery, runPostQuery } from './plugins/lifecycle.js';
 
 const listingCache = new LRUCache({ max: 1000 });
 
@@ -68,49 +70,13 @@ export default function s3Querier({
     });
     const downloadedPaths = results.flatMap((result) => result.value);
     const finalQuery = runFinalizers({ plugins: systemPlugins, rawQuery, fileSettings, downloadedPaths, bucketsDir });
-    return execQuery(finalQuery, { format });
-  });
-}
+    const postQueryCallbacks = runPreQuery(systemPlugins, { sql: finalQuery, downloadedPaths, bucketsDir });
 
-/**
- * Orchestrates:
- *  - Passing the query through to each plugin
- *  - Merging file download settings
- *
- * @param {Array} plugins
- * @param {object} context
- * @returns
- */
-function processQuery(plugins = [], { query = '', endpoint, defaultBucket, bucketsDir }) {
-  const processedQuery = plugins.reduce((result, plugin) => plugin.processQuery(result), {
-    endpoint,
-    defaultBucket,
-    bucketsDir,
-    query,
-    settings: [],
+    return execQuery(finalQuery, { format }).then((result) => {
+      runPostQuery(systemPlugins, { result, downloadedPaths, bucketsDir }, postQueryCallbacks);
+      return result;
+    });
   });
-  const fileSettings = processedQuery.settings;
-  processedQuery.settings = mergeSettings(processedQuery.settings);
-  return { ...processedQuery, fileSettings };
-}
-
-/**
- * Passes the raw query through each plugin's `finalizeQuery` lifecycle method,
- * substituting exact downloaded paths in place of glob patterns.
- *
- * @param {object} params
- * @param {object[]} params.plugins - Plugin instances to run finalizers on.
- * @param {string} params.rawQuery - SQL with original file references and date/location tokens.
- * @param {object[]} params.fileSettings - Pre-merge per-file settings from processQuery.
- * @param {string[]} params.downloadedPaths - Absolute local paths of all downloaded files.
- * @param {string} params.bucketsDir - Root directory where files are cached locally.
- * @returns {string} Finalized SQL ready for DuckDB execution.
- */
-function runFinalizers({ plugins, rawQuery, fileSettings, downloadedPaths, bucketsDir }) {
-  return plugins.reduce((query, plugin) => {
-    if (!plugin.finalizeQuery) return query;
-    return plugin.finalizeQuery(query, fileSettings, downloadedPaths, bucketsDir);
-  }, rawQuery);
 }
 
 /**

package/src/utils/date-regex/date-regex.js

@@ -2,27 +2,27 @@ const DIGITS_4 = '\\d{4}';
 const DIGITS_2 = '\\d{2}';
 
 export function yyyy(str, date) {
-  return str.replaceAll('{yyyy}', String(date.getFullYear()));
+  return str.replaceAll('{yyyy}', String(date.getUTCFullYear()));
 }
 
 export function MM(str, date) {
-  return str.replaceAll('{MM}', String(date.getMonth() + 1).padStart(2, '0'));
+  return str.replaceAll('{MM}', String(date.getUTCMonth() + 1).padStart(2, '0'));
 }
 
 export function dd(str, date) {
-  return str.replaceAll('{dd}', String(date.getDate()).padStart(2, '0'));
+  return str.replaceAll('{dd}', String(date.getUTCDate()).padStart(2, '0'));
 }
 
 export function hh(str, date) {
-  return str.replaceAll('{hh}', String(date.getHours()).padStart(2, '0'));
+  return str.replaceAll('{hh}', String(date.getUTCHours()).padStart(2, '0'));
 }
 
 export function mm(str, date) {
-  return str.replaceAll('{mm}', String(date.getMinutes()).padStart(2, '0'));
+  return str.replaceAll('{mm}', String(date.getUTCMinutes()).padStart(2, '0'));
 }
 
 export function ss(str, date) {
-  return str.replaceAll('{ss}', String(date.getSeconds()).padStart(2, '0'));
+  return str.replaceAll('{ss}', String(date.getUTCSeconds()).padStart(2, '0'));
 }
 
 export function regexFromPattern(pattern = '') {

package/src/utils/file-path-builder/file-path-builder.js

@@ -1,5 +1,8 @@
-import { eachDayOfInterval, eachHourOfInterval } from 'date-fns';
 import { yyyy, MM, dd, hh, mm, ss } from '../date-regex/date-regex.js';
+
+const MS_PER_HOUR = 60 * 60 * 1000;
+const MS_PER_DAY = 24 * MS_PER_HOUR;
+
 /**
  * Give a string with date patterns replaces patterns with actual data values
  *
@@ -15,17 +18,17 @@ export function buildPath(filePattern, date) {
 }
 
 /**
- * Given a date range returns an array of date objects
+ * Given a date range returns an array of date objects, one per UTC calendar day.
  *
  * @param {Date} from The from Date object
  * @param {Date} to The to Date object
  * @returns {Date[]} An array of dates within the to from from time range
  */
 export function datesInRange(from, to) {
-  return eachDayOfInterval({
-    start: zeroDateMins(new Date(from)),
-    end: zeroDateMins(new Date(to)),
-  });
+  const startMs = Date.UTC(from.getUTCFullYear(), from.getUTCMonth(), from.getUTCDate());
+  const endMs = Date.UTC(to.getUTCFullYear(), to.getUTCMonth(), to.getUTCDate());
+  const count = Math.round((endMs - startMs) / MS_PER_DAY) + 1;
+  return Array.from({ length: count }, (_, index) => new Date(startMs + index * MS_PER_DAY));
 }
 
 /**
@@ -45,29 +48,17 @@ export function monthsInRange(from, to) {
 }
 
 /**
- * Given a date range returns an array of date objects
+ * Given a date range returns an array of date objects, one per UTC hour.
  *
  * @param {Date} from The from Date object
  * @param {Date} to The to Date object
  * @returns {Date[]} An array of dates by hours within the to from from time range
  */
 export function hoursInRange(from, to) {
-  return eachHourOfInterval({
-    start: new Date(from),
-    end: new Date(to),
-  });
-}
-
-/**
- * Sets a Date object minutes and seconds to 0
- *
- * @param {Date} date Date object
- * @returns {Date} A Date object with minutes and seconds set to 0
- */
-export function zeroDateMins(date) {
-  const zeroedDate = new Date(date);
-  zeroedDate.setMinutes(0, 0);
-  return zeroedDate;
+  const startMs = Date.UTC(from.getUTCFullYear(), from.getUTCMonth(), from.getUTCDate(), from.getUTCHours());
+  const endMs = Date.UTC(to.getUTCFullYear(), to.getUTCMonth(), to.getUTCDate(), to.getUTCHours());
+  const count = Math.round((endMs - startMs) / MS_PER_HOUR) + 1;
+  return Array.from({ length: count }, (_, index) => new Date(startMs + index * MS_PER_HOUR));
 }
 
 function noonUtcForMonthOffset(startYear, startMonth, offset) {

package/src/utils/fs-purge/fs-purge.js

@@ -0,0 +1,57 @@
+import { glob } from 'glob';
+import { unlink } from 'node:fs/promises';
+import { logger } from '../logger.js';
+
+export default class FSPurge {
+  /**
+   * @param {object} options
+   * @param {string} options.pattern Glob pattern for files to consider for purging
+   * @param {number} [options.lastAccessTTLMinutes=60] Minutes since last access before a file is eligible for purge
+   * @param {number} [options.refreshIntervalMin=60] How often to run a sweep, in minutes
+   */
+  constructor({ pattern, lastAccessTTLMinutes = 60, refreshIntervalMin = 60 }) {
+    this.lastAccessTTLMinutes = lastAccessTTLMinutes;
+    this.pattern = pattern;
+    this.refreshIntervalMin = refreshIntervalMin;
+    this.lastSweep = null;
+  }
+
+  start() {
+    this.refreshRef = setInterval(this.sweep.bind(this), this.refreshIntervalMin * (1000 * 60));
+  }
+
+  stop() {
+    clearInterval(this.refreshRef);
+  }
+
+  async sweep() {
+    const now = new Date();
+    if (this.lastSweep && now - this.lastSweep < this.refreshIntervalMin * 60_000) return;
+    this.lastSweep = now;
+
+    const startSweep = now;
+    const filesToBeRemoved = await this.getExpiredFiles();
+    logger.info(`Removing ${filesToBeRemoved.length} files in sweep. Read time: ${(new Date() - startSweep) / 1000}s`);
+
+    await Promise.allSettled(
+      filesToBeRemoved.map((file) => {
+        return unlink(file)
+          .then(() => logger.debug(`Deleted file ${file}.`))
+          .catch((error) => logger.error(`Error deleting file ${file}. ${error.toString()}`));
+      }),
+    );
+  }
+
+  async getExpiredFiles() {
+    const expiresAt = new Date();
+    expiresAt.setMinutes(expiresAt.getMinutes() - this.lastAccessTTLMinutes);
+
+    const files = await glob(this.pattern, {
+      stat: true,
+      withFileTypes: true,
+      nodir: true,
+    });
+
+    return files.filter((file) => file.atime < expiresAt).map((file) => file.fullpath());
+  }
+}

package/src/utils/logger.js

@@ -1,3 +1,3 @@
 import pino from 'pino';
-const logger = pino();
+const logger = pino({}, process.stderr);
 export { logger };