s3-querier 1.0.1 → 1.0.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "s3-querier",
-  "version": "1.0.1",
+  "version": "1.0.3",
   "description": "Query S3-compatible storage with DuckDB and SQL",
   "type": "module",
   "main": "src/s3-querier.js",

package/src/plugins/query-finalizer/query-finalizer.js

@@ -1,4 +1,4 @@
-import { removeFileDatePatterns } from '../../utils/date-regex/date-regex.js';
+import { regexFromPattern } from '../../utils/date-regex/date-regex.js';
 import {
   removeFileSettingTokens,
   removeDoubleFwdSlash,
@@ -9,33 +9,60 @@ export default class QueryFinalizerPlugin {
   name = 'CorePlugin';
 
   processQuery(context) {
-    const { settings, bucketsDir, query } = context;
-    const processedQuery = QueryFinalizerPlugin.prepareQuery(settings, bucketsDir, query);
-    return { ...context, query: processedQuery };
+    return context;
   }
 
-  static prepareQuery(settings, bucketsDir, query) {
-    let prepared = query;
-
-    settings.forEach((setting) => {
-      const searchPattern = setting.sqlFileReference.replace(/\?cache=(true|false)/i, '');
-      const fileRegexStr = QueryFinalizerPlugin.prepareFileRegexStr(searchPattern);
-      prepared = prepared.replace(new RegExp(fileRegexStr, 'gi'), `${bucketsDir}/${setting.bucket}/${setting.file}`);
-    });
+  /**
+   * Replaces each SQL file reference with the exact local paths downloaded from S3.
+   * Called after all downloads complete so that DuckDB receives precise file paths
+   * rather than glob patterns that would scan the entire local cache.
+   *
+   * @param {string} rawQuery - SQL with original file references and date/location tokens
+   * @param {object[]} fileSettings - Pre-merge per-file settings from processQuery
+   * @param {string[]} downloadedPaths - Absolute local paths of all downloaded files
+   * @param {string} bucketsDir - Root directory where files are cached locally
+   * @returns {string} Finalized SQL ready for DuckDB execution
+   */
+  finalizeQuery(rawQuery, fileSettings, downloadedPaths, bucketsDir) {
+    let prepared = fileSettings.reduce(
+      (query, setting) => applyFileSetting(query, setting, downloadedPaths, bucketsDir),
+      rawQuery,
+    );
     prepared = removeFileSettingTokens(prepared);
-    prepared = removeFileDatePatterns(prepared);
     prepared = removeCacheSettings(prepared);
     prepared = removeDoubleFwdSlash(prepared);
-
     return prepared;
   }
+}
 
-  static prepareFileRegexStr(fileStr) {
-    return fileStr
-      .replace(/\*/g, '\\*')
-      .replace(/\./g, '\\.')
-      .replace(/\{/g, '\\{')
-      .replace(/\}/g, '\\}')
-      .replace(/\+/g, '\\+');
-  }
+/** Helpers */
+
+function applyFileSetting(query, { sqlFileReference, file, bucket }, downloadedPaths, bucketsDir) {
+  const localDir = `${bucketsDir}/${bucket}/`;
+  const filePattern = regexFromPattern(file);
+  const matchingPaths = downloadedPaths.filter((localPath) => matchesPattern(localPath, localDir, filePattern));
+  const searchStr = sqlFileReference.replace(/\?cache=(true|false)/i, '');
+
+  if (matchingPaths.length === 0) throw new Error(`No files found for: ${file}`);
+  if (matchingPaths.length > 1) return replaceWithArray(query, searchStr, matchingPaths);
+
+  return query.replace(new RegExp(escapeForRegex(searchStr), 'gi'), matchingPaths[0]);
+}
+
+function matchesPattern(localPath, localDir, filePattern) {
+  return localPath.startsWith(localDir) && filePattern.test(localPath.slice(localDir.length));
+}
+
+function replaceWithArray(query, searchStr, paths) {
+  const arrayLiteral = `[${paths.map((path) => `'${path}'`).join(', ')}]`;
+  return query.replace(new RegExp(`['"]${escapeForRegex(searchStr)}['"]`, 'gi'), arrayLiteral);
+}
+
+function escapeForRegex(str) {
+  return str
+    .replace(/\*/g, '\\*')
+    .replace(/\./g, '\\.')
+    .replace(/\{/g, '\\{')
+    .replace(/\}/g, '\\}')
+    .replace(/\+/g, '\\+');
 }

package/src/s3/s3.js

@@ -3,7 +3,7 @@ import { dirname } from 'node:path';
 import { S3Client, ListObjectsV2Command, GetObjectCommand } from '@aws-sdk/client-s3';
 
 import { logger } from '../utils/logger.js';
-import { datesInRange, hoursInRange, buildPath } from '../utils/file-path-builder/file-path-builder.js';
+import { datesInRange, hoursInRange, monthsInRange, buildPath } from '../utils/file-path-builder/file-path-builder.js';
 import { regexFromPattern } from '../utils/date-regex/date-regex.js';
 import { buildIbmIamClient } from './auth/ibm-iam-client.js';
 
@@ -139,12 +139,14 @@ export default class S3 {
    * @returns {(from:Date, to:Date, filePattern:string) => string[]} A function that creates a list of prefixes
    */
   prefixStrategy(from, to, filePattern) {
-    const hasDateToken = filePattern.match(/\{(yyyy|MM|dd|hh|mm)\}/g);
-    const hasGlob = filePattern.match(/\*/g);
+    const hasDayToken = /\{(dd|hh|mm)\}/.test(filePattern);
+    const hasMonthToken = /\{(yyyy|MM)\}/.test(filePattern);
+    const hasGlob = filePattern.includes('*');
     const hourDiff = (new Date(to) - new Date(from)) / 1000 / 60 / 60;
 
-    if (hasDateToken && hourDiff < 24) return this.prefixHours;
-    if (hasDateToken) return this.prefixDays;
+    if (hasDayToken && hourDiff < 24) return this.prefixHours;
+    if (hasDayToken) return this.prefixDays;
+    if (hasMonthToken) return this.prefixMonths;
     if (hasGlob) return this.prefixGlob;
     return (_from, _to, pattern) => [pattern];
   }
@@ -182,6 +184,22 @@ export default class S3 {
     });
   }
 
+  /**
+   * Returns a list of prefixes based on a range of months, one per calendar month.
+   * Used for Hive-style paths with {yyyy}/{MM} tokens but no {dd}.
+   *
+   * @param {Date} from From date
+   * @param {Date} to To date
+   * @param {string} filePattern The file pattern to use
+   * @returns {string[]} The list of prefixes for filtering
+   */
+  prefixMonths(from, to, filePattern) {
+    const monthRange = monthsInRange(new Date(from), new Date(to));
+    const splitToken = filePattern.includes('{MM}') ? '{MM}' : '{yyyy}';
+    const [trimmed] = filePattern.split(splitToken);
+    return monthRange.map((date) => buildPath(`${trimmed}${splitToken}`, date));
+  }
+
   /**
    * Returns a single entry array with a file pattern trimmed to the first glob
    *
@@ -368,7 +386,7 @@ export default class S3 {
    */
   async objectToFile(key) {
     const file = `${this.mount}/${key}`;
-    const tmp = `${file}.${process.pid}.${Date.now()}.tmp`;
+    const tmp = `${file}.${process.pid}.${Date.now()}.${Math.random().toString(36).slice(2)}.tmp`;
     try {
       const response = await this.s3.send(new GetObjectCommand({ Bucket: this.bucket, Key: key }));
       const chunks = [];
@@ -422,7 +440,7 @@ export default class S3 {
   }
 }
 
-function buildS3Client({ apiKey, accessKeyId, secretAccessKey, endpoint, region }) {
+export function buildS3Client({ apiKey, accessKeyId, secretAccessKey, endpoint, region = 'us-east-1' }) {
   const config = { ...(endpoint && { endpoint }), region, forcePathStyle: true };
 
   if (apiKey) return buildIbmIamClient(config, apiKey);

package/src/s3-querier.js

@@ -40,8 +40,16 @@ export default function s3Querier({
   format,
 }) {
   const systemPlugins = [new QueryParserPlugin(), ...plugins, new QueryFinalizerPlugin()];
-  const processed = processQuery(systemPlugins, { query, endpoint: defaultEndpoint, defaultBucket, bucketsDir });
-  const { query: processedQuery, settings: downloadSettings } = processed;
+  const {
+    query: rawQuery,
+    fileSettings,
+    settings: downloadSettings,
+  } = processQuery(systemPlugins, {
+    query,
+    endpoint: defaultEndpoint,
+    defaultBucket,
+    bucketsDir,
+  });
 
   const downloadPromises = startDownloads({
     apiKey,
@@ -58,7 +66,9 @@ export default function s3Querier({
     results.forEach((result) => {
       if (result.status === 'rejected') throw result.reason;
     });
-    return execQuery(processedQuery, { format });
+    const downloadedPaths = results.flatMap((result) => result.value);
+    const finalQuery = runFinalizers({ plugins: systemPlugins, rawQuery, fileSettings, downloadedPaths, bucketsDir });
+    return execQuery(finalQuery, { format });
   });
 }
 
@@ -72,14 +82,35 @@ export default function s3Querier({
  * @returns
  */
 function processQuery(plugins = [], { query = '', endpoint, defaultBucket, bucketsDir }) {
-  const processedQuery = plugins.reduce(
-    (result, plugin) => {
-      return plugin.processQuery(result);
-    },
-    { endpoint, defaultBucket, bucketsDir, query, settings: [] },
-  );
+  const processedQuery = plugins.reduce((result, plugin) => plugin.processQuery(result), {
+    endpoint,
+    defaultBucket,
+    bucketsDir,
+    query,
+    settings: [],
+  });
+  const fileSettings = processedQuery.settings;
   processedQuery.settings = mergeSettings(processedQuery.settings);
-  return processedQuery;
+  return { ...processedQuery, fileSettings };
+}
+
+/**
+ * Passes the raw query through each plugin's `finalizeQuery` lifecycle method,
+ * substituting exact downloaded paths in place of glob patterns.
+ *
+ * @param {object} params
+ * @param {object[]} params.plugins - Plugin instances to run finalizers on.
+ * @param {string} params.rawQuery - SQL with original file references and date/location tokens.
+ * @param {object[]} params.fileSettings - Pre-merge per-file settings from processQuery.
+ * @param {string[]} params.downloadedPaths - Absolute local paths of all downloaded files.
+ * @param {string} params.bucketsDir - Root directory where files are cached locally.
+ * @returns {string} Finalized SQL ready for DuckDB execution.
+ */
+function runFinalizers({ plugins, rawQuery, fileSettings, downloadedPaths, bucketsDir }) {
+  return plugins.reduce((query, plugin) => {
+    if (!plugin.finalizeQuery) return query;
+    return plugin.finalizeQuery(query, fileSettings, downloadedPaths, bucketsDir);
+  }, rawQuery);
 }
 
 /**

package/src/utils/file-path-builder/file-path-builder.js

@@ -28,6 +28,22 @@ export function datesInRange(from, to) {
   });
 }
 
+/**
+ * Given a date range returns one date object per calendar month.
+ * Returns noon-UTC dates so that getMonth()/getFullYear() produce the correct
+ * UTC month in any timezone.
+ *
+ * @param {Date} from The from Date object
+ * @param {Date} to The to Date object
+ * @returns {Date[]} An array of dates, one per month within the range
+ */
+export function monthsInRange(from, to) {
+  const startYear = from.getUTCFullYear();
+  const startMonth = from.getUTCMonth();
+  const count = (to.getUTCFullYear() - startYear) * 12 + (to.getUTCMonth() - startMonth) + 1;
+  return Array.from({ length: count }, (_, index) => noonUtcForMonthOffset(startYear, startMonth, index));
+}
+
 /**
  * Given a date range returns an array of date objects
  *
@@ -53,3 +69,9 @@ export function zeroDateMins(date) {
   zeroedDate.setMinutes(0, 0);
   return zeroedDate;
 }
+
+function noonUtcForMonthOffset(startYear, startMonth, offset) {
+  const year = startYear + Math.floor((startMonth + offset) / 12);
+  const month = (startMonth + offset) % 12;
+  return new Date(Date.UTC(year, month, 1, 12, 0, 0));
+}