s3-querier 1.3.0 → 1.3.1

package/README.md

@@ -141,16 +141,82 @@ Downloaded files are cached to `bucketsDir` on disk. Subsequent queries that ref
 
 ## Plugins
 
-The `plugins` option accepts an array of plugin objects that can extend query parsing and file processing. A plugin may implement:
+The `plugins` option accepts an array of plugin objects. Plugins can hook into every phase of query execution — from S3 listing through download to SQL execution.
 
-- `processQuery(context)` — transform the query context before execution
-- `processFile(filePath)` — process each downloaded file (e.g. convert Avro to JSON)
+### Plugin interface
 
-The built-in Avro plugin is an example:
+| Method | Phase | Description |
+| --- | --- | --- |
+| `processQuery(context)` | pre-download | Transform the query context. Return the (possibly mutated) context. |
+| `finalizeQuery(query, fileSettings, downloadedPaths, bucketsDir)` | post-download | Rewrite the SQL string after downloads complete. Return the final SQL. |
+| `preListFiles({ prefix, bucket })` | S3 listing | Called before listing. Return a callback or nothing. |
+| `preDownloadFiles({ bucket, from, to })` | S3 download | Called before downloading. Return a callback or nothing. |
+| `preQuery({ sql, downloadedPaths, bucketsDir })` | DuckDB execution | Called before the query runs. Return a callback or nothing. |
+| `postQuery({ result, downloadedPaths, bucketsDir })` | DuckDB execution | Called after the query completes. |
+
+The `pre*` methods use a closure pattern: return a callback to receive the after-state for that phase. This lets you capture a start timestamp and receive the result in one place without shared mutable variables:
 
 ```js
-import s3Querier from 's3-querier';
-import AvroPlugin from 's3-querier/src/plugins/avro/avro-plugin.js';
+preQuery({ sql }) {
+  const start = Date.now();
+  return ({ result }) => {
+    console.log(`Query took ${Date.now() - start}ms — ${result.length} rows`);
+  };
+}
+```
+
+Post-phase callbacks are fire-and-forget — errors are logged and swallowed, so a failing plugin never rejects the caller's query.
+
+### FSPurgePlugin
+
+`FSPurgePlugin` sweeps the local file cache after each query, evicting files that haven't been accessed recently. Import it alongside the default export:
+
+```js
+import s3Querier, { FSPurgePlugin } from 's3-querier';
+
+const purgePlugin = new FSPurgePlugin({
+  bucketsDir: '/tmp/s3-cache',
+  lastAccessTTLMinutes: 60, // evict files not accessed in the last hour (default: 60)
+  refreshIntervalMin: 60,   // minimum minutes between sweeps (default: 60)
+});
+
+const results = await s3Querier({
+  // ...
+  plugins: [purgePlugin],
+});
+```
+
+### StatsPlugin
+
+`StatsPlugin` fires a single `onStats` callback for listing, download, and query events. Use it for logging, metrics, or custom dashboards:
+
+```js
+import s3Querier, { StatsPlugin } from 's3-querier';
+
+const statsPlugin = new StatsPlugin((event) => console.log(event));
+
+await s3Querier({ /* ... */ plugins: [statsPlugin] });
+// { type: 'listing',  prefix, bucket, fileCount, durationMs, cacheHit }
+// { type: 'download', bucket, from, to, cacheHits, cacheMisses, enqueuedHits, bytesDownloaded, durationMs }
+// { type: 'query',    sql, durationMs, rowCount }
+```
+
+Each call fires a single event with a discriminated `type`:
+
+| `type` | Fields |
+| --- | --- |
+| `'listing'` | `prefix`, `bucket`, `fileCount`, `durationMs`, `cacheHit` |
+| `'download'` | `bucket`, `from`, `to`, `cacheHits`, `cacheMisses`, `enqueuedHits`, `bytesDownloaded`, `durationMs` |
+| `'query'` | `sql`, `durationMs`, `rowCount` |
+
+Events fire independently — one listing event per S3 prefix, one download event per bucket per query, one query event per execution. Aggregation is left to the caller.
+
+### AvroPlugin
+
+The built-in Avro plugin converts Avro files to JSON before querying:
+
+```js
+import s3Querier, { AvroPlugin } from 's3-querier';
 
 const results = await s3Querier({
   // ...

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "s3-querier",
-  "version": "1.3.0",
+  "version": "1.3.1",
   "description": "Query S3-compatible storage with DuckDB and SQL",
   "type": "module",
   "main": "src/s3-querier.js",

package/src/mcp/server.js

@@ -1,5 +1,17 @@
 #!/usr/bin/env node
-
 import { S3QuerierMCP } from './s3querier-mcp.js';
+import { FSPurgePlugin, StatsPlugin } from '../../s3-querier.js';
+import { logger } from '../utils/logger.js';
+
+const { S3_BUCKETS_DIR = '/tmp/s3-querier', S3_PURGE_CACHE = 'true', S3_PURGE_TTL_MINUTES = '60' } = process.env;
+
+await new S3QuerierMCP({ plugins: buildDefaultPlugins() }).start();
 
-await new S3QuerierMCP().start();
+function buildDefaultPlugins() {
+  const plugins = [];
+  if (S3_PURGE_CACHE !== 'false') {
+    plugins.push(new FSPurgePlugin({ bucketsDir: S3_BUCKETS_DIR, lastAccessTTLMinutes: Number(S3_PURGE_TTL_MINUTES) }));
+  }
+  plugins.push(new StatsPlugin((event) => logger.info(event)));
+  return plugins;
+}

package/src/mcp/tools/query/query.js

@@ -2,8 +2,7 @@ import { readFileSync } from 'node:fs';
 import { z } from 'zod';
 
 import BaseTool from '../base-tool.js';
-import s3Querier, { bigintReplacer, FSPurgePlugin, StatsPlugin } from '../../../s3-querier.js';
-import { logger } from '../../../utils/logger.js';
+import s3Querier, { bigintReplacer } from '../../../s3-querier.js';
 
 const {
   S3_ACCESS_KEY_ID,
@@ -14,9 +13,6 @@ const {
   S3_BUCKETS_DIR = '/tmp/s3-querier',
 } = process.env;
 
-const purgePlugin = new FSPurgePlugin({ bucketsDir: S3_BUCKETS_DIR });
-const statsPlugin = new StatsPlugin((event) => logger.info(event));
-
 const sqlDescription = readFileSync(new URL('../../descriptions/sql-param.md', import.meta.url), 'utf8');
 const toolDescription = readFileSync(new URL('../../descriptions/tool.md', import.meta.url), 'utf8');
 
@@ -63,7 +59,7 @@ export default class QueryTool extends BaseTool {
       accessKeyId: S3_ACCESS_KEY_ID,
       secretAccessKey: S3_SECRET_ACCESS_KEY,
       format: 'jsonRecords',
-      plugins: [purgePlugin, statsPlugin],
+      plugins: this.config.plugins ?? [],
     });
 
     return {

package/src/s3-querier.js

@@ -5,6 +5,7 @@ export { bigintReplacer } from './utils/bigint-replacer.js';
 import { query as execQuery } from './duck-db/index.js';
 import QueryParserPlugin from './plugins/query-parser/query-parser.js';
 import QueryFinalizerPlugin from './plugins/query-finalizer/query-finalizer.js';
+export { default as AvroPlugin } from './plugins/avro/avro-plugin.js';
 export { default as FSPurgePlugin } from './plugins/fs-purge/fs-purge-plugin.js';
 export { default as StatsPlugin } from './plugins/stats/stats-plugin.js';
 import { processQuery, runFinalizers, runPreQuery, runPostQuery } from './plugins/lifecycle.js';