s3-querier 1.0.0

package/README.md

@@ -0,0 +1,203 @@
+# s3-querier
+
+Query S3-compatible storage directly with DuckDB SQL. S3 Querier handles listing files, downloading them locally, and executing your query — turning a data lake into a queryable resource with a single function call.
+
+## Requirements
+
+- Node.js >= 22
+- S3-compatible storage (AWS S3, MinIO, IBM COS, etc.) with HMAC or IBM IAM credentials
+
+## Installation
+
+```bash
+npm install s3-querier
+```
+
+## Usage
+
+```js
+import s3Querier from 's3-querier';
+
+const results = await s3Querier({
+  accessKeyId: 'your-access-key',
+  secretAccessKey: 'your-secret-key',
+  defaultEndpoint: 'https://s3.amazonaws.com',
+  defaultBucket: 'my-bucket',
+  bucketsDir: '/tmp/s3-cache',
+  from: new Date('2025-01-01').getTime(),
+  to: new Date('2025-01-31').getTime(),
+  query: `SELECT * FROM read_parquet('events/year={yyyy}/month={MM}/day={dd}/data.parquet')`,
+  format: 'jsonRecords',
+});
+```
+
+## API
+
+### `s3Querier(options)`
+
+Returns a `Promise` that resolves to the query results.
+
+| Option | Type | Required | Description |
+| --- | --- | --- | --- |
+| `accessKeyId` | `string` | ✓ | HMAC access key ID |
+| `secretAccessKey` | `string` | ✓ | HMAC secret access key |
+| `defaultEndpoint` | `string` | ✓ | S3 endpoint URL |
+| `defaultBucket` | `string` | ✓ | Default bucket name |
+| `bucketsDir` | `string` | ✓ | Local directory for caching downloaded files |
+| `query` | `string` | ✓ | DuckDB SQL query |
+| `from` | `number` | | Start of date range as a Unix timestamp (ms). Required when using date tokens. |
+| `to` | `number` | | End of date range as a Unix timestamp (ms). Required when using date tokens. |
+| `format` | `string` | | Output format. `'jsonRecords'` returns `[{ col: val }]`. Default is columnar `[{ name, fields: [val, ...] }]`. |
+| `plugins` | `array` | | Additional plugins to extend query processing. |
+
+### Environment Variables
+
+| Variable | Default | Description |
+| --- | --- | --- |
+| `MAX_MB_DOWNLOAD` | `1000` | Maximum total download size in MB per query. Queries exceeding this limit throw an error. |
+
+## Query Syntax
+
+### Static Files
+
+```sql
+SELECT * FROM read_parquet('reports/summary.parquet') LIMIT 10;
+```
+
+### Date Tokens
+
+When `from` and `to` are provided, date tokens are expanded into a list of matching file paths.
+
+```sql
+SELECT *
+FROM read_parquet('events/year={yyyy}/month={MM}/day={dd}/data.parquet', union_by_name=1);
+```
+
+| Token | Description | Example |
+| --- | --- | --- |
+| `{yyyy}` | 4-digit year | `2025` |
+| `{MM}` | 2-digit month | `01`–`12` |
+| `{dd}` | 2-digit day | `01`–`31` |
+| `{hh}` | 2-digit hour | `00`–`23` |
+| `{mm}` | 2-digit minute | `00`–`59` |
+| `{ss}` | 2-digit second | `00`–`59` |
+
+### Glob Patterns
+
+```sql
+SELECT * FROM read_parquet('reports/2025/*.parquet', union_by_name=1);
+```
+
+### Location Tokens
+
+Override the default endpoint and bucket per file reference within a query.
+
+```sql
+SELECT *
+FROM read_parquet('{endpoint:https://s3.us-east.example.com}/{bucket:my-bucket}/data.parquet');
+```
+
+### Cross-Bucket Joins
+
+```sql
+SELECT s.id, s.event_type, r.description
+FROM read_parquet('{bucket:events-bucket}/reports/summary.parquet') s
+JOIN read_parquet('{bucket:reference-bucket}/lookup.parquet') r ON s.id = r.id;
+```
+
+### Cache Control
+
+Append `?cache=false` to force a fresh download, bypassing the local cache.
+
+```sql
+SELECT * FROM read_parquet('reports/summary.parquet?cache=false');
+```
+
+## BigInt
+
+> [!WARNING]
+> DuckDB returns `BigInt` for `COUNT(*)`, `SUM`, and other integer aggregations. `BigInt` is not JSON-serializable — `JSON.stringify` will throw.
+
+The safest fix is to cast in SQL:
+
+```sql
+SELECT CAST(COUNT(*) AS INTEGER) AS total FROM read_parquet('data.parquet')
+```
+
+If you can't control the query, use the exported `bigintReplacer` with `JSON.stringify`:
+
+```js
+import s3Querier, { bigintReplacer } from 's3-querier';
+
+const results = await s3Querier({ ..., format: 'jsonRecords' });
+const json = JSON.stringify(results, bigintReplacer);
+```
+
+Note: `bigintReplacer` converts `BigInt` to `Number`, which loses precision for values above `Number.MAX_SAFE_INTEGER` (~9 quadrillion). For large integer IDs or counters, prefer the SQL cast.
+
+## Caching
+
+Downloaded files are cached to `bucketsDir` on disk. Subsequent queries that reference the same files skip the download entirely. The listing cache (S3 object listings) is held in memory per process using an LRU cache, with today's prefix always re-fetched to pick up new files.
+
+## Plugins
+
+The `plugins` option accepts an array of plugin objects that can extend query parsing and file processing. A plugin may implement:
+
+- `processQuery(context)` — transform the query context before execution
+- `processFile(filePath)` — process each downloaded file (e.g. convert Avro to JSON)
+
+The built-in Avro plugin is an example:
+
+```js
+import s3Querier from 's3-querier';
+import AvroPlugin from 's3-querier/src/plugins/avro/avro-plugin.js';
+
+const results = await s3Querier({
+  // ...
+  plugins: [new AvroPlugin()],
+  query: `SELECT * FROM read_json('data.avro+json')`,
+});
+```
+
+## Examples
+
+The `examples/` directory contains a local interactive demo and standalone scripts. All examples target a local MinIO instance — you'll need [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/install/) installed. Both are bundled with Docker Desktop on Mac and Windows; on Linux, install the Compose plugin separately.
+
+### Interactive demo
+
+Starts MinIO, seeds it with sample parquet data, and launches an Express server with a Monaco SQL editor in the browser.
+
+```bash
+npm run demo:up     # start MinIO and seed data (runs once)
+npm run demo:start  # start the Express server
+```
+
+Then open [http://localhost:3000](http://localhost:3000). The editor has five pre-loaded example queries you can run or modify. When you're done:
+
+```bash
+npm run demo:down   # stop MinIO
+```
+
+### Standalone scripts
+
+Run any script directly after MinIO is up:
+
+```bash
+npm run demo:up                               # if not already running
+node examples/scripts/basic-query.js         # fetch the first 10 sales rows
+node examples/scripts/glob-pattern.js        # filter to Jan–Feb with a brace glob
+node examples/scripts/date-range.js          # use {from}/{to} date tokens
+node examples/scripts/ibm-cos.js             # IBM Cloud Object Storage (requires env vars)
+```
+
+For the IBM COS script, set these environment variables first:
+
+```bash
+export IBM_COS_API_KEY=your-api-key
+export IBM_COS_ENDPOINT=https://s3.us-south.cloud-object-storage.appdomain.cloud
+export IBM_COS_BUCKET=your-bucket
+```
+
+## License
+
+MIT

package/docs/s3-querier.md

@@ -0,0 +1,196 @@
+# S3 Querier
+
+S3 Querier allows you to query data lake content directly using [DuckDB](https://duckdb.org/) queries. By parsing these queries, determining the necessary files, and dynamically downloading and processing them, S3 Querier transforms what is otherwise an opaque storage system into a user-friendly, queryable resource.
+
+## Planning Your Queries
+
+When querying data from a data lake, be mindful of how your queries are constructed. S3 Querier downloads files from S3-compatible storage before they can be queried using DuckDB, so query speed is directly influenced by the size and number of files involved.
+
+### Key Considerations
+
+1. **File Size And Query Efficiency**
+   Large files increase query time because they take longer to download. To optimize performance:
+
+   - Query only the columns you need. Avoid `SELECT *` without a `LIMIT`.
+   - Avoid overly broad queries that download unnecessary files, such as `read_parquet('my-bucket/*.parquet')`.
+
+2. **1GB File Size Limit**
+   This service enforces a 1GB limit per query. Queries that access files accumulating beyond this limit will fail.
+
+3. **Partitioning And Filtering**
+   Partition your data in the lake where possible. This lets you filter queries to target only relevant partitions, reducing unnecessary downloads.
+
+### Tips For Better Query Planning
+
+- **Test Locally First**
+  [Install the DuckDB CLI](https://duckdb.org/docs/installation/?version=stable&environment=cli&platform=macos&download_method=direct) and experiment with your queries on local parquet files before running them against S3. This gives a fast feedback loop for understanding data structure and refining queries.
+
+- **Be Mindful Of Time Ranges In Date Tokens**
+  Long time ranges require fetching more files and slow execution. Use narrow time ranges whenever possible.
+
+- **Create Secondary Representations Of Your Data**
+  For larger datasets, break files into smaller chunks to avoid hitting the file size limit.
+
+- **Monitor Query Times**
+  If a query is slow, revisit the query logic and the files it accesses.
+
+## Example Queries
+
+Below are some examples of common use cases.
+
+**Key Concepts:**
+
+- Static files versus dynamic files. A static file is a file for which you know the exact location in S3. A dynamic file uses one or more file tokens to match a range of files. [See the file tokens section](#file-tokens-overview) for details.
+- In most cases you'll want to use `union_by_name=1` when using `read_parquet` or `read_csv`. Read more about [why this is important](https://duckdb.org/2025/01/10/union-by-name.html).
+
+### Querying A Single, Static File
+
+```sql
+SELECT * FROM
+  read_parquet('file1.parquet', union_by_name=1)
+LIMIT 10;
+```
+
+### Getting Multiple, Static Files
+
+When querying multiple files they should share the same or a similar schema. Use `union_by_name=1` to handle minor schema differences.
+
+```sql
+SELECT * FROM
+  read_parquet(['file1.parquet', 'file2.parquet'], union_by_name=1)
+LIMIT 10;
+```
+
+### Querying Time-Related Files
+
+Use date tokens to query files spanning a specific date range. When S3 Querier receives a `from` and `to` parameter, it automatically expands the file list to match the date tokens in your query.
+
+#### Example
+
+Given `from=2025-08-03` and `to=2025-08-06`, the following query:
+
+```sql
+SELECT id
+FROM read_parquet('jobs_failed/year={yyyy}/month={MM}/day={dd}/servers.parquet', union_by_name=1);
+```
+
+Will resolve and download:
+
+```
+jobs_failed/year=2025/month=08/day=03/servers.parquet
+jobs_failed/year=2025/month=08/day=04/servers.parquet
+jobs_failed/year=2025/month=08/day=05/servers.parquet
+jobs_failed/year=2025/month=08/day=06/servers.parquet
+```
+
+For more details, see [tips about date ranges](#tips-for-better-query-planning).
+
+### Querying Files From Multiple Locations
+
+Use location tokens to query files across different endpoints or buckets in a single query.
+
+```sql
+WITH us_south_data AS (
+  SELECT id, timestamp
+  FROM read_parquet('{endpoint:https://s3.us-south.example.com}/{bucket:my-bucket}/my_time_series/{yyyy}{MM}{dd}{hh}{mm}{ss}.parquet')
+)
+SELECT id, timestamp
+FROM read_parquet('{endpoint:https://s3.us-east.example.com}/{bucket:my-bucket}/my_time_series_2/{yyyy}{MM}{dd}{hh}{mm}{ss}.parquet') AS us_east_data
+JOIN us_south_data ON us_east_data.id = us_south_data.id;
+```
+
+### Tips And Utilities
+
+See the DuckDB docs for [`read_parquet` parameters](https://duckdb.org/docs/stable/data/parquet/overview.html#parameters) and [`read_csv` parameters](https://duckdb.org/docs/stable/data/csv/overview#parameters).
+
+#### Getting The File Name Of The File(s) Being Queried
+
+Pass `filename=1` to `read_parquet` or `read_csv` to include the source file path as a column in results.
+
+```sql
+SELECT id, filename
+FROM read_parquet('year={yyyy}/month={MM}/my-file.parquet', filename=1);
+```
+
+| id  | filename                            |
+| --- | ----------------------------------- |
+| 1   | year=2025/month=01/my-file.parquet  |
+
+#### Extracting Partition Values From Hive-Style Paths
+
+If your data uses Hive-style partitioning (e.g., `year=2025/month=04/day=20`), use `hive_partitioning=1` to extract partition keys as columns.
+
+```sql
+SELECT year, month, day, id
+FROM read_parquet('jobs_failed/year=2025/month=01/day=19/my-file.parquet', hive_partitioning=1);
+```
+
+| id | year | month | day |
+| -- | ---- | ----- | --- |
+| 1  | 2025 | 01    | 19  |
+
+---
+
+## File Tokens Overview
+
+File tokens allow you to create dynamic queries with patterns that vary based on time, non-time components, or storage location. There are three types: **Glob Syntax**, **Time Formatting Tokens**, and **Location Tokens**.
+
+### Glob Syntax
+
+Glob syntax handles file name segments that vary but are not time-related.
+
+```
+jobs_failed/window=202308032130/0.parquet
+jobs_failed/window=202308032230/3.parquet
+jobs_failed/window=202308032330/6.parquet
+```
+
+```sql
+SELECT id
+FROM read_parquet('jobs_failed/window=202308032130/*.parquet', union_by_name=1);
+```
+
+> [!WARNING]
+> **Use globs with caution.** They can match more files than expected, causing unnecessary downloads and degraded performance. Always verify the file list your glob will match before running a broad query.
+
+---
+
+### Time Formatting Tokens
+
+Time tokens dynamically match files based on time-related patterns in their names, based on [Unicode Technical Standard #35](https://unicode.org/reports/tr35/).
+
+| **Token**         | **Usage**      | **Example Output** |
+| ----------------- | -------------- | ------------------ |
+| **Year** `{yyyy}` | 4-digit year   | 1970, ..., 2030    |
+| **Month** `{MM}`  | 2-digit month  | 01...12            |
+| **Day** `{dd}`    | 2-digit day    | 01...31            |
+| **Hour** `{hh}`   | 2-digit hour   | 00...23            |
+| **Minute** `{mm}` | 2-digit minute | 00...59            |
+| **Second** `{ss}` | 2-digit second | 00...59            |
+
+```sql
+SELECT id
+FROM read_parquet('jobs_failed/window={yyyy}{MM}{dd}{hh}{mm}/*.parquet', union_by_name=1);
+```
+
+---
+
+### Location Tokens
+
+Location tokens let you vary the storage endpoint and bucket within a query.
+
+| **Token**                  | **Usage**                        | **Example**                                                  |
+| -------------------------- | -------------------------------- | ------------------------------------------------------------ |
+| **Endpoint** `{endpoint:}` | Specifies a storage endpoint URL | `{endpoint:http://s3.example.com}/my-bucket/file.parquet`    |
+| **Bucket** `{bucket:}`     | Specifies a storage bucket       | `{bucket:my-bucket}/file.parquet`                            |
+
+```sql
+SELECT id
+FROM read_parquet('{endpoint:http://s3.example.com}/{bucket:my-bucket}/jobs_failed/window={yyyy}{MM}{dd}{hh}{mm}/*.parquet');
+```
+
+**Benefits:**
+
+1. **Cross-Endpoint Queries** — Query data stored on different S3-compatible endpoints in a single query.
+2. **Cross-Bucket Queries** — Access data from multiple buckets without separate queries.
+3. **Dynamic Query Construction** — Combine location tokens with glob syntax and time tokens for fully dynamic, cross-location queries.

package/package.json

@@ -0,0 +1,71 @@
+{
+  "name": "s3-querier",
+  "version": "1.0.0",
+  "description": "Query S3-compatible storage with DuckDB and SQL",
+  "type": "module",
+  "main": "src/s3-querier.js",
+  "exports": {
+    ".": "./src/s3-querier.js"
+  },
+  "files": [
+    "src/**/*.js",
+    "!src/**/*.test.js",
+    "docs/",
+    "README.md"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+ssh://git@github.com/grommett/s3-querier.git"
+  },
+  "author": "david@pinkiering.com",
+  "scripts": {
+    "test": "node --test \"./src/**/*.test.js\"",
+    "test:e2e": "docker compose -f e2e/docker-compose.yml up -d --wait && node e2e/setup/seed.js && node --test e2e/*.e2e.js; docker compose -f e2e/docker-compose.yml down",
+    "test:coverage:html": "c8 -x coverage -x **/*.test.js --all -r html node --test \"./src/**/*.test.js\"",
+    "prettify": "prettier \"./src/**/*.js\" --write",
+    "lint": "eslint \"./src/**/*.js\"",
+    "lint:fix": "eslint --fix \"./src/**/*.js\"",
+    "prepare": "husky",
+    "demo:up": "docker compose -f examples/demo/docker-compose.yml up -d --wait && node examples/demo/seed.js",
+    "demo:down": "docker compose -f examples/demo/docker-compose.yml down",
+    "demo:start": "node examples/demo/server.js"
+  },
+  "engines": {
+    "node": ">=22"
+  },
+  "keywords": [
+    "s3",
+    "duckdb",
+    "parquet",
+    "query"
+  ],
+  "license": "MIT",
+  "dependencies": {
+    "@aws-sdk/client-s3": "^3.0.0",
+    "@derekstride/tree-sitter-sql": "^0.3.11",
+    "@duckdb/node-api": "^1.5.3-r.3",
+    "avsc": "^5.7.7",
+    "date-fns": "^4.0.0",
+    "lru-cache": "^11.0.0",
+    "peggy": "^5.1.0",
+    "pino": "^10.3.1",
+    "tree-sitter": "^0.21.1"
+  },
+  "lint-staged": {
+    "src/**/*.js": [
+      "prettier --write",
+      "eslint"
+    ]
+  },
+  "devDependencies": {
+    "@eslint/js": "^10.0.1",
+    "c8": "^11.0.0",
+    "eslint": "^10.5.0",
+    "esmock": "^2.7.5",
+    "express": "^5.2.1",
+    "globals": "^17.6.0",
+    "husky": "^9.1.7",
+    "lint-staged": "^17.0.7",
+    "prettier": "^3.0.3"
+  }
+}

package/src/duck-db/index.js

@@ -0,0 +1,57 @@
+import { DuckDBInstance } from '@duckdb/node-api';
+import { logger } from '../utils/logger.js';
+
+const db = await DuckDBInstance.create(':memory:', {
+  threads: 4,
+});
+
+const formatStrategies = {
+  jsonRecords: formatJsonRecords,
+  default: formatColumnar,
+};
+
+/**
+ * Execute a SQL query and return results in the specified format
+ *
+ * @param {string} sql - The SQL query to execute
+ * @param {object} options - Query options
+ * @param {string} options.format - Output format: 'jsonRecords' for row objects, otherwise columnar (default)
+ * @returns {Promise<Array>} Query results in the requested format
+ */
+export async function query(sql, options = {}) {
+  const { format } = options;
+  const queryStart = new Date();
+
+  try {
+    const connection = await db.connect();
+    const reader = await connection.runAndReadAll(sql);
+    const columnsResult = reader.getColumnsObjectJS();
+
+    const formatter = formatStrategies[format] ?? formatStrategies.default;
+    const result = formatter(columnsResult);
+
+    const queryTime = new Date() - queryStart;
+    logger.info(`Query completed in : ${queryTime / 1000} seconds`);
+    return result ?? [];
+  } catch (error) {
+    logger.error(error);
+    throw error;
+  }
+}
+
+function formatColumnar(columnsResult) {
+  return Object.keys(columnsResult).map((key) => ({ name: key, fields: columnsResult[key] }));
+}
+
+function formatJsonRecords(columnsResult) {
+  const keys = Object.keys(columnsResult);
+  if (keys.length === 0) return [];
+  const rowCount = columnsResult[keys[0]].length;
+  return Array.from({ length: rowCount }, (_, rowIndex) => {
+    const row = {};
+    keys.forEach((key) => {
+      row[key] = columnsResult[key][rowIndex];
+    });
+    return row;
+  });
+}

package/src/plugins/avro/avro-plugin.js

@@ -0,0 +1,64 @@
+import { createWriteStream } from 'node:fs';
+import fsPromise from 'node:fs/promises';
+import avro from 'avsc';
+
+import QueryParserPlugin from '../query-parser/query-parser.js';
+
+const AVRO_EXTENSION = /\.avro(\?|$)/i;
+
+class AvroPlugin extends QueryParserPlugin {
+  name = 'AvroPlugin';
+
+  processQuery(context) {
+    const { query, settings, endpoint, defaultBucket } = context;
+    const avroSettings = this.getFiles({ endpoint, defaultBucket, query });
+    const processedQuery = replaceAvroExtension(query);
+    return { ...context, settings: [...settings, ...avroSettings], query: processedQuery };
+  }
+
+  getFiles({ endpoint, defaultBucket, query }) {
+    return super
+      .getFiles({ endpoint, defaultBucket, query })
+      .filter((setting) => AVRO_EXTENSION.test(setting.file))
+      .map((setting) => ({ ...setting, sqlFileReference: setting.file.replace(/\.avro/gi, '.json') }));
+  }
+
+  /**
+   * Converts an avro file to json file
+   *
+   * @param {string} file
+   * @returns {Promise<string>} A promise that resolves to the processed file's name
+   */
+  processFile(file) {
+    if (!file.includes('.avro')) return Promise.resolve(file);
+    const errorMsg = `Error converting avro to json for ${file}`;
+
+    return new Promise((resolve, reject) => {
+      const jsonFile = file.replace('.avro', '.json');
+      fileExists(jsonFile)
+        .then((exists) => {
+          if (exists) return resolve(jsonFile);
+          const fileStream = createWriteStream(jsonFile);
+          avro.createFileDecoder(file).pipe(fileStream);
+          fileStream.on('close', () => resolve(jsonFile));
+          fileStream.on('error', () => reject(new Error(errorMsg)));
+        })
+        .catch(() => reject(new Error(errorMsg)));
+    });
+  }
+}
+
+function replaceAvroExtension(query) {
+  return query.replace(/\.avro/gi, '.json');
+}
+
+async function fileExists(file) {
+  try {
+    await fsPromise.stat(file);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export default AvroPlugin;

package/src/plugins/query-finalizer/query-finalizer.js

@@ -0,0 +1,41 @@
+import { removeFileDatePatterns } from '../../utils/date-regex/date-regex.js';
+import {
+  removeFileSettingTokens,
+  removeDoubleFwdSlash,
+  removeCacheSettings,
+} from '../../utils/file-settings/file-settings.js';
+
+export default class QueryFinalizerPlugin {
+  name = 'CorePlugin';
+
+  processQuery(context) {
+    const { settings, bucketsDir, query } = context;
+    const processedQuery = QueryFinalizerPlugin.prepareQuery(settings, bucketsDir, query);
+    return { ...context, query: processedQuery };
+  }
+
+  static prepareQuery(settings, bucketsDir, query) {
+    let prepared = query;
+
+    settings.forEach((setting) => {
+      const searchPattern = setting.sqlFileReference.replace(/\?cache=(true|false)/i, '');
+      const fileRegexStr = QueryFinalizerPlugin.prepareFileRegexStr(searchPattern);
+      prepared = prepared.replace(new RegExp(fileRegexStr, 'gi'), `${bucketsDir}/${setting.bucket}/${setting.file}`);
+    });
+    prepared = removeFileSettingTokens(prepared);
+    prepared = removeFileDatePatterns(prepared);
+    prepared = removeCacheSettings(prepared);
+    prepared = removeDoubleFwdSlash(prepared);
+
+    return prepared;
+  }
+
+  static prepareFileRegexStr(fileStr) {
+    return fileStr
+      .replace(/\*/g, '\\*')
+      .replace(/\./g, '\\.')
+      .replace(/\{/g, '\\{')
+      .replace(/\}/g, '\\}')
+      .replace(/\+/g, '\\+');
+  }
+}

package/src/plugins/query-parser/query-parser.js

@@ -0,0 +1,33 @@
+import { extractFileReferences } from '../../utils/sql-parser/sql-parser.js';
+import { parseFilePath } from '../../utils/path-parser/path-parser.js';
+
+class QueryParserPlugin {
+  name = 'BasePlugin';
+
+  processQuery(context) {
+    const { settings, endpoint, defaultBucket, query } = context;
+    const fileSettings = this.getFiles({ endpoint, defaultBucket, query });
+    return { ...context, settings: [...settings, ...fileSettings] };
+  }
+
+  getFiles({ endpoint, defaultBucket, query }) {
+    return extractFileReferences(query).map((ref) => toFileSetting(ref, endpoint, defaultBucket));
+  }
+
+  static processFile(file) {
+    return Promise.resolve(file);
+  }
+}
+
+function toFileSetting({ raw }, defaultEndpoint, defaultBucket) {
+  const parsed = parseFilePath(raw);
+  return {
+    endpoint: parsed.endpoint ?? defaultEndpoint,
+    bucket: parsed.bucket ?? defaultBucket,
+    file: parsed.file,
+    cache: parsed.cache,
+    sqlFileReference: raw,
+  };
+}
+
+export default QueryParserPlugin;

package/src/s3/auth/ibm-iam-client.js

@@ -0,0 +1,21 @@
+import { S3Client } from '@aws-sdk/client-s3';
+import { IbmIamTokenManager } from './ibm-iam-token-manager.js';
+
+export function buildIbmIamClient(config, apiKey) {
+  const tokenManager = new IbmIamTokenManager(apiKey);
+  const client = new S3Client({ ...config, credentials: { accessKeyId: 'ibm-iam', secretAccessKey: 'ibm-iam' } });
+  client.middlewareStack.add(ibmIamMiddleware(tokenManager), {
+    step: 'finalizeRequest',
+    priority: 'low',
+    name: 'ibmIamAuth',
+  });
+  return client;
+}
+
+function ibmIamMiddleware(tokenManager) {
+  return (next) => async (args) => {
+    const token = await tokenManager.getToken();
+    args.request.headers['Authorization'] = `Bearer ${token}`;
+    return next(args);
+  };
+}