s3-querier 1.1.0 → 1.1.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "s3-querier",
-  "version": "1.1.0",
+  "version": "1.1.2",
   "description": "Query S3-compatible storage with DuckDB and SQL",
   "type": "module",
   "main": "src/s3-querier.js",
@@ -9,11 +9,12 @@
   },
   "exports": {
     ".": "./src/s3-querier.js",
-    "./mcp": "./src/mcp/datalake-mcp.js"
+    "./mcp": "./src/mcp/s3querier-mcp.js"
   },
   "files": [
     "src/**/*.js",
     "!src/**/*.test.js",
+    "src/mcp/descriptions/",
     "docs/",
     "README.md"
   ],
@@ -25,7 +26,7 @@
   "scripts": {
     "test": "node --test \"./src/**/*.test.js\"",
     "test:e2e": "docker compose -f e2e/docker-compose.yml up -d --wait && node e2e/setup/seed.js && node --test e2e/*.e2e.js; docker compose -f e2e/docker-compose.yml down",
-    "test:coverage:html": "c8 -x coverage -x **/*.test.js --all -r html node --test \"./src/**/*.test.js\"",
+    "test:coverage:html": "c8 -x coverage -x **/*.test.js -x src/s3-querier.js -x e2e -x examples -x '*.config.js' --all -r html node --test \"./src/**/*.test.js\"",
     "prettify": "prettier \"./src/**/*.js\" --write",
     "lint": "eslint \"./src/**/*.js\"",
     "lint:fix": "eslint --fix \"./src/**/*.js\"",

package/src/mcp/descriptions/list-files.md

@@ -0,0 +1,34 @@
+List files and sub-directories in an S3 bucket at a given prefix level.
+
+Returns:
+- directories: sub-prefixes to drill into (e.g. "env=production/", "year=2026/")
+- files: objects at this level with path, size, and column names for parquet files
+- truncated: true if there are more results — narrow the prefix or increase maxResults
+
+NAVIGATION STRATEGY
+
+Start with an empty prefix to see the top-level structure. Drill into directories
+one level at a time. Each call only shows what is directly under the given prefix,
+not the full recursive tree.
+
+HIVE-PARTITIONED DATA
+
+Today's date is {{TODAY}}.
+
+Many datasets use Hive-style partitioning with partition keys in the path:
+  year=YYYY/month=MM/day=DD/hour=HH/minute=MM/
+
+To find the most recent data:
+1. List down to the partition root (e.g. env=production/) — one call per structural level
+2. Then STOP listing. Do NOT list year, month, or day directories individually.
+3. Construct the date segment directly from today: year=YYYY/month=MM/day=DD/
+4. Append it to the partition root and list from the day directory to find the latest hour.
+
+Example: if you reach env=production/ and today is 2026-06-14, your next call is
+  prefix: "…/env=production/year=2026/month=06/day=14/"
+Then list that to find the available hours, pick the highest, and continue.
+
+Only fall back to listing year/month/day if the constructed path returns empty directories and files.
+
+Always check for a "latest/" directory first — it often contains the most recent
+snapshot and avoids navigating the full partition tree.

package/src/mcp/descriptions/sql-param.md

@@ -0,0 +1,81 @@
+A DuckDB SQL query. File paths inside read_parquet() or read_csv() are resolved against S3
+and downloaded before the query runs.
+
+REQUIRED: always call read_parquet() or read_csv() — plain table names are not supported.
+Prefer union_by_name=1 when reading multiple files.
+
+SCHEMA DISCOVERY: Never guess column names. Before writing any query that joins files or
+references specific columns, run SELECT * FROM read_parquet('path') LIMIT 1 on each file
+to inspect the actual schema. Only then write the real query.
+
+FILE PATH TOKENS
+
+Date tokens — expanded using the `from` and `to` parameters:
+  {yyyy}  4-digit year      e.g. 2025
+  {MM}    2-digit month     e.g. 08
+  {dd}    2-digit day       e.g. 03
+  {hh}    2-digit hour      e.g. 14
+  {mm}    2-digit minute    e.g. 30
+  {ss}    2-digit second    e.g. 00
+
+Location tokens — override endpoint or bucket per path:
+  {endpoint:https://s3.example.com}
+  {bucket:my-bucket}
+
+Glob syntax — wildcard matching for non-time path segments:
+  jobs/window=202308032130/*.parquet
+
+QUERYING TIME-PARTITIONED DATA OVER A RANGE
+
+When data is partitioned by time and you need multiple hours, days, or months, use date
+tokens in the SQL with `from`/`to` as separate parameters. ONE query with tokens downloads
+all matching files across the range — do not make multiple tool calls with hardcoded dates.
+
+  ✗ WRONG — three separate tool calls with hardcoded hours:
+      sql: SELECT * FROM read_parquet('events/year=2026/month=06/day=15/hour=12/data.parquet')
+      sql: SELECT * FROM read_parquet('events/year=2026/month=06/day=15/hour=13/data.parquet')
+      sql: SELECT * FROM read_parquet('events/year=2026/month=06/day=15/hour=14/data.parquet')
+
+  ✓ CORRECT — one tool call, tokens expand across all hours in the range:
+      sql:  SELECT * FROM read_parquet('events/year={yyyy}/month={MM}/day={dd}/hour={hh}/data.parquet', union_by_name=1)
+      from: 2026-06-15T12:00:00Z
+      to:   2026-06-15T14:59:59Z
+
+Tokens also expand inside the filename, not just in path directory segments:
+  data/year={yyyy}/month={MM}/day={dd}/hour={hh}/file_{yyyy}{MM}{dd}{hh}00.parquet
+  → s3-querier downloads one file per hour in the from/to range.
+
+Hardcoding a date is fine for a single known file or a fixed point-in-time lookup where
+you are not spanning multiple time segments.
+
+HIVE-PARTITIONED DATA
+
+For paths partitioned only by year and month (no day segment), use {yyyy} and {MM} together:
+  sales/year={yyyy}/month={MM}/data.parquet
+
+s3-querier generates one prefix per calendar month in the from/to range, so a Q1 query
+(from=2024-01-01, to=2024-03-31) fetches exactly months 01, 02, 03 — not all of 2024.
+
+Do NOT use DuckDB character-class globs like month=0[1-3] — DuckDB does not support them.
+Use {yyyy}/{MM} tokens instead, which s3-querier expands correctly.
+
+EXAMPLES
+
+Single file:
+  SELECT * FROM read_parquet('reports/summary.parquet') LIMIT 10
+
+Hour-partitioned files — tokens in path and filename (requires from/to):
+  SELECT * FROM read_parquet(
+    'events/year={yyyy}/month={MM}/day={dd}/hour={hh}/file_{yyyy}{MM}{dd}{hh}00.parquet',
+    union_by_name=1)
+
+Day-partitioned files (requires from/to):
+  SELECT id FROM read_parquet('events/year={yyyy}/month={MM}/day={dd}/data.parquet', union_by_name=1)
+
+Month-partitioned files (no day segment — use {yyyy}/{MM} only):
+  SELECT * FROM read_parquet('sales/year={yyyy}/month={MM}/data.parquet', union_by_name=1)
+
+Cross-endpoint join:
+  WITH east AS (SELECT id FROM read_parquet('{endpoint:https://s3.us-east.example.com}/{bucket:logs}/data/{yyyy}{MM}{dd}.parquet'))
+  SELECT * FROM read_parquet('{endpoint:https://s3.eu-west.example.com}/{bucket:logs}/data/{yyyy}{MM}{dd}.parquet') AS west
+  JOIN east ON west.id = east.id

package/src/mcp/descriptions/tool.md

@@ -0,0 +1,8 @@
+Download files from S3-compatible storage and execute a DuckDB SQL query against them.
+
+Queries must use DuckDB table functions such as read_parquet() or read_csv() with file paths
+that reference objects in S3. S3 Querier resolves those paths, downloads the matching files,
+and runs the query locally with DuckDB.
+
+Use the `s3-querier://docs` resource for full documentation including token syntax, examples,
+and query planning tips.