s3-querier 1.0.0

package/src/utils/file-settings/file-settings.js

@@ -0,0 +1,65 @@
+const endPointRegexStr = '\\{endpoint:(?<endpoint>[a-z0-9-:/.]+)\\}';
+const bucketRegexStr = '\\{bucket:(?<bucket>[a-z0-9-:/._]+)\\}';
+
+/**
+ * Merges endpoints and bucket file values
+ *
+ * @param {DownloadSetting[]} settings File settings (endpoint/bucket) parsed from the query
+ * @returns {object[]} Settings merged on endpoint/buckets
+ */
+export function mergeSettings(settings = []) {
+  const settingsMerged = settings.reduce((acc, setting) => {
+    const { file, cache, endpoint, bucket } = setting;
+    const key = `${endpoint}/${bucket}`;
+    const tokenRegex = /\{|\*/;
+
+    if (acc[key]) {
+      if (tokenRegex.test(file)) acc[key].filePatterns.push({ file, cache });
+      if (!tokenRegex.test(file)) acc[key].staticFiles.push({ file, cache });
+      return acc;
+    }
+
+    acc[key] = {};
+    acc[key].filePatterns = tokenRegex.test(file) ? [{ file, cache }] : [];
+    acc[key].staticFiles = !tokenRegex.test(file) ? [{ file, cache }] : [];
+    acc[key].endpoint = endpoint;
+    acc[key].bucket = bucket;
+    return acc;
+  }, {});
+
+  return Object.values(settingsMerged);
+}
+
+/**
+ * Removes file setting tokens
+ *
+ * @param {string} query The query
+ * @returns {string} The query with file setting tokens removed
+ */
+export function removeFileSettingTokens(query = '') {
+  const endPointRegex = new RegExp(endPointRegexStr, 'gi');
+  const bucketPointRegex = new RegExp(bucketRegexStr, 'gi');
+  query = query.replace(endPointRegex, '');
+  query = query.replace(bucketPointRegex, '');
+  return query;
+}
+
+/**
+ * Removes double / in file paths. Edge case where bucket paths start with /
+ *
+ * @param {string} query The query
+ * @returns {string} The query with file setting tokens removed
+ */
+export function removeDoubleFwdSlash(query = '') {
+  return query.replace(/\/\/+/g, '/');
+}
+
+/**
+ * Removes ?cache=(true|false) from the file paths in the query
+ *
+ * @param {string} query The query
+ * @returns {string} The query with file setting tokens removed
+ */
+export function removeCacheSettings(query = '') {
+  return query.replace(/\?cache=(true|false)/gi, '');
+}

package/src/utils/logger.js

@@ -0,0 +1,3 @@
+import pino from 'pino';
+const logger = pino();
+export { logger };

package/src/utils/path-parser/path-parser-grammar.js

@@ -0,0 +1,39 @@
+export const GRAMMAR = `
+FilePattern
+  = tokens:Token+ cache:CacheParam? {
+      if (cache) return [...tokens, cache];
+      return tokens;
+    }
+
+Token
+  = LocationToken
+  / DateToken
+  / GlobToken
+  / LiteralText
+
+LocationToken
+  = "{" _ "endpoint:" _ value:LocationValue _ "}" "/"? { return { type: "endpoint", value }; }
+  / "{" _ "bucket:"   _ value:LocationValue _ "}" "/"? { return { type: "bucket",   value }; }
+
+LocationValue
+  = chars:[^} \t]+ { return chars.join(""); }
+
+_ = [ \t]*
+
+DateToken
+  = "{yyyy}" { return { type: "date", unit: "year"   }; }
+  / "{MM}"   { return { type: "date", unit: "month"  }; }
+  / "{dd}"   { return { type: "date", unit: "day"    }; }
+  / "{hh}"   { return { type: "date", unit: "hour"   }; }
+  / "{mm}"   { return { type: "date", unit: "minute" }; }
+  / "{ss}"   { return { type: "date", unit: "second" }; }
+
+GlobToken
+  = "*" { return { type: "glob" }; }
+
+CacheParam
+  = "?cache=" v:("true" / "false") { return { type: "cache", value: v === "true" }; }
+
+LiteralText
+  = chars:[^{*?]+ { return { type: "literal", value: chars.join("") }; }
+`;

package/src/utils/path-parser/path-parser.js

@@ -0,0 +1,58 @@
+import peggy from 'peggy';
+import { GRAMMAR } from './path-parser-grammar.js';
+
+const DATE_UNIT_TOKENS = {
+  year: '{yyyy}',
+  month: '{MM}',
+  day: '{dd}',
+  hour: '{hh}',
+  minute: '{mm}',
+  second: '{ss}',
+};
+
+const compiledParser = peggy.generate(GRAMMAR);
+
+/**
+ * Parses an S3 file path string (without surrounding SQL quotes) into its
+ * constituent tokens, then extracts endpoint, bucket, file path, and cache setting.
+ *
+ * @param {string} raw File path string as it appears inside the SQL quotes
+ * @returns {{ endpoint: string|null, bucket: string|null, file: string, cache: boolean }}
+ */
+export function parseFilePath(raw) {
+  const tokens = compiledParser.parse(raw);
+  return {
+    endpoint: extractEndpoint(tokens),
+    bucket: extractBucket(tokens),
+    file: buildFilePath(tokens),
+    cache: extractCache(tokens),
+  };
+}
+
+/** Reconstructs the file path from tokens, dropping endpoint/bucket/cache */
+function buildFilePath(tokens) {
+  return tokens
+    .filter((token) => token.type !== 'endpoint' && token.type !== 'bucket' && token.type !== 'cache')
+    .map(tokenToString)
+    .join('');
+}
+
+function tokenToString(token) {
+  if (token.type === 'literal') return token.value;
+  if (token.type === 'date') return DATE_UNIT_TOKENS[token.unit];
+  if (token.type === 'glob') return '*';
+  return '';
+}
+
+function extractEndpoint(tokens) {
+  return tokens.find((token) => token.type === 'endpoint')?.value ?? null;
+}
+
+function extractBucket(tokens) {
+  return tokens.find((token) => token.type === 'bucket')?.value ?? null;
+}
+
+function extractCache(tokens) {
+  const cacheToken = tokens.find((token) => token.type === 'cache');
+  return cacheToken ? cacheToken.value : true;
+}

package/src/utils/sql-parser/sql-parser.js

@@ -0,0 +1,31 @@
+import { createRequire } from 'module';
+
+const require = createRequire(import.meta.url);
+const Parser = require('tree-sitter');
+const SQL = require('@derekstride/tree-sitter-sql');
+
+const FILE_QUERY = new Parser.Query(
+  SQL,
+  `
+  (invocation
+    (object_reference (identifier) @func (#match? @func "^read_"))
+    (term (literal) @file (#match? @file "^'"))
+  )
+`,
+);
+
+const parser = new Parser();
+parser.setLanguage(SQL);
+
+export function extractFileReferences(query) {
+  const tree = parser.parse(query);
+  const seen = new Set();
+  return FILE_QUERY.captures(tree.rootNode)
+    .filter((capture) => capture.name === 'file')
+    .map((capture) => ({ raw: capture.node.text.slice(1, -1) }))
+    .filter(({ raw }) => {
+      if (seen.has(raw)) return false;
+      seen.add(raw);
+      return true;
+    });
+}