s3-querier 1.0.2 → 1.0.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "s3-querier",
-  "version": "1.0.2",
+  "version": "1.0.3",
   "description": "Query S3-compatible storage with DuckDB and SQL",
   "type": "module",
   "main": "src/s3-querier.js",

package/src/plugins/query-finalizer/query-finalizer.js

@@ -1,4 +1,4 @@
-import { removeFileDatePatterns } from '../../utils/date-regex/date-regex.js';
+import { regexFromPattern } from '../../utils/date-regex/date-regex.js';
 import {
   removeFileSettingTokens,
   removeDoubleFwdSlash,
@@ -9,33 +9,60 @@ export default class QueryFinalizerPlugin {
   name = 'CorePlugin';
 
   processQuery(context) {
-    const { settings, bucketsDir, query } = context;
-    const processedQuery = QueryFinalizerPlugin.prepareQuery(settings, bucketsDir, query);
-    return { ...context, query: processedQuery };
+    return context;
   }
 
-  static prepareQuery(settings, bucketsDir, query) {
-    let prepared = query;
-
-    settings.forEach((setting) => {
-      const searchPattern = setting.sqlFileReference.replace(/\?cache=(true|false)/i, '');
-      const fileRegexStr = QueryFinalizerPlugin.prepareFileRegexStr(searchPattern);
-      prepared = prepared.replace(new RegExp(fileRegexStr, 'gi'), `${bucketsDir}/${setting.bucket}/${setting.file}`);
-    });
+  /**
+   * Replaces each SQL file reference with the exact local paths downloaded from S3.
+   * Called after all downloads complete so that DuckDB receives precise file paths
+   * rather than glob patterns that would scan the entire local cache.
+   *
+   * @param {string} rawQuery - SQL with original file references and date/location tokens
+   * @param {object[]} fileSettings - Pre-merge per-file settings from processQuery
+   * @param {string[]} downloadedPaths - Absolute local paths of all downloaded files
+   * @param {string} bucketsDir - Root directory where files are cached locally
+   * @returns {string} Finalized SQL ready for DuckDB execution
+   */
+  finalizeQuery(rawQuery, fileSettings, downloadedPaths, bucketsDir) {
+    let prepared = fileSettings.reduce(
+      (query, setting) => applyFileSetting(query, setting, downloadedPaths, bucketsDir),
+      rawQuery,
+    );
     prepared = removeFileSettingTokens(prepared);
-    prepared = removeFileDatePatterns(prepared);
     prepared = removeCacheSettings(prepared);
     prepared = removeDoubleFwdSlash(prepared);
-
     return prepared;
   }
+}
 
-  static prepareFileRegexStr(fileStr) {
-    return fileStr
-      .replace(/\*/g, '\\*')
-      .replace(/\./g, '\\.')
-      .replace(/\{/g, '\\{')
-      .replace(/\}/g, '\\}')
-      .replace(/\+/g, '\\+');
-  }
+/** Helpers */
+
+function applyFileSetting(query, { sqlFileReference, file, bucket }, downloadedPaths, bucketsDir) {
+  const localDir = `${bucketsDir}/${bucket}/`;
+  const filePattern = regexFromPattern(file);
+  const matchingPaths = downloadedPaths.filter((localPath) => matchesPattern(localPath, localDir, filePattern));
+  const searchStr = sqlFileReference.replace(/\?cache=(true|false)/i, '');
+
+  if (matchingPaths.length === 0) throw new Error(`No files found for: ${file}`);
+  if (matchingPaths.length > 1) return replaceWithArray(query, searchStr, matchingPaths);
+
+  return query.replace(new RegExp(escapeForRegex(searchStr), 'gi'), matchingPaths[0]);
+}
+
+function matchesPattern(localPath, localDir, filePattern) {
+  return localPath.startsWith(localDir) && filePattern.test(localPath.slice(localDir.length));
+}
+
+function replaceWithArray(query, searchStr, paths) {
+  const arrayLiteral = `[${paths.map((path) => `'${path}'`).join(', ')}]`;
+  return query.replace(new RegExp(`['"]${escapeForRegex(searchStr)}['"]`, 'gi'), arrayLiteral);
+}
+
+function escapeForRegex(str) {
+  return str
+    .replace(/\*/g, '\\*')
+    .replace(/\./g, '\\.')
+    .replace(/\{/g, '\\{')
+    .replace(/\}/g, '\\}')
+    .replace(/\+/g, '\\+');
 }

package/src/s3/s3.js

@@ -386,7 +386,7 @@ export default class S3 {
    */
   async objectToFile(key) {
     const file = `${this.mount}/${key}`;
-    const tmp = `${file}.${process.pid}.${Date.now()}.tmp`;
+    const tmp = `${file}.${process.pid}.${Date.now()}.${Math.random().toString(36).slice(2)}.tmp`;
     try {
       const response = await this.s3.send(new GetObjectCommand({ Bucket: this.bucket, Key: key }));
       const chunks = [];

package/src/s3-querier.js

@@ -40,8 +40,16 @@ export default function s3Querier({
   format,
 }) {
   const systemPlugins = [new QueryParserPlugin(), ...plugins, new QueryFinalizerPlugin()];
-  const processed = processQuery(systemPlugins, { query, endpoint: defaultEndpoint, defaultBucket, bucketsDir });
-  const { query: processedQuery, settings: downloadSettings } = processed;
+  const {
+    query: rawQuery,
+    fileSettings,
+    settings: downloadSettings,
+  } = processQuery(systemPlugins, {
+    query,
+    endpoint: defaultEndpoint,
+    defaultBucket,
+    bucketsDir,
+  });
 
   const downloadPromises = startDownloads({
     apiKey,
@@ -58,7 +66,9 @@ export default function s3Querier({
     results.forEach((result) => {
       if (result.status === 'rejected') throw result.reason;
     });
-    return execQuery(processedQuery, { format });
+    const downloadedPaths = results.flatMap((result) => result.value);
+    const finalQuery = runFinalizers({ plugins: systemPlugins, rawQuery, fileSettings, downloadedPaths, bucketsDir });
+    return execQuery(finalQuery, { format });
   });
 }
 
@@ -72,14 +82,35 @@ export default function s3Querier({
  * @returns
  */
 function processQuery(plugins = [], { query = '', endpoint, defaultBucket, bucketsDir }) {
-  const processedQuery = plugins.reduce(
-    (result, plugin) => {
-      return plugin.processQuery(result);
-    },
-    { endpoint, defaultBucket, bucketsDir, query, settings: [] },
-  );
+  const processedQuery = plugins.reduce((result, plugin) => plugin.processQuery(result), {
+    endpoint,
+    defaultBucket,
+    bucketsDir,
+    query,
+    settings: [],
+  });
+  const fileSettings = processedQuery.settings;
   processedQuery.settings = mergeSettings(processedQuery.settings);
-  return processedQuery;
+  return { ...processedQuery, fileSettings };
+}
+
+/**
+ * Passes the raw query through each plugin's `finalizeQuery` lifecycle method,
+ * substituting exact downloaded paths in place of glob patterns.
+ *
+ * @param {object} params
+ * @param {object[]} params.plugins - Plugin instances to run finalizers on.
+ * @param {string} params.rawQuery - SQL with original file references and date/location tokens.
+ * @param {object[]} params.fileSettings - Pre-merge per-file settings from processQuery.
+ * @param {string[]} params.downloadedPaths - Absolute local paths of all downloaded files.
+ * @param {string} params.bucketsDir - Root directory where files are cached locally.
+ * @returns {string} Finalized SQL ready for DuckDB execution.
+ */
+function runFinalizers({ plugins, rawQuery, fileSettings, downloadedPaths, bucketsDir }) {
+  return plugins.reduce((query, plugin) => {
+    if (!plugin.finalizeQuery) return query;
+    return plugin.finalizeQuery(query, fileSettings, downloadedPaths, bucketsDir);
+  }, rawQuery);
 }
 
 /**