squirreling 0.12.11 → 0.12.12

package/README.md

@@ -141,7 +141,7 @@ Squirreling mostly follows the SQL standard. The following features are supporte
 - `SELECT` statements with `DISTINCT`, `WHERE`, `ORDER BY`, `LIMIT`, `OFFSET`
 - `WITH` clause for Common Table Expressions (CTEs)
 - Subqueries in `SELECT`, `FROM`, `WHERE`, and correlated subqueries
-- `JOIN` operations: `INNER JOIN`, `LEFT JOIN`, `RIGHT JOIN`, `FULL JOIN`, `CROSS JOIN`, `POSITIONAL JOIN`
+- `JOIN` operations: `INNER JOIN`, `LEFT JOIN`, `RIGHT JOIN`, `FULL JOIN`, `CROSS JOIN`, `POSITIONAL JOIN`, `LATERAL VIEW [OUTER] EXPLODE(...)`
 - `GROUP BY` and `HAVING` clauses
 - Set operations: `UNION`, `UNION ALL`, `INTERSECT`, `INTERSECT ALL`, `EXCEPT`, `EXCEPT ALL`
 - Expressions: `CASE`, `CAST`, `BETWEEN`, `IN`, `LIKE`, `IS NULL`, `IS NOT NULL`
@@ -161,7 +161,7 @@ Squirreling mostly follows the SQL standard. The following features are supporte
 - Date: `CURRENT_DATE`, `CURRENT_TIME`, `CURRENT_TIMESTAMP`, `DATE_PART`, `DATE_TRUNC`, `EXTRACT`, `INTERVAL`
 - Json: `JSON_VALUE`, `JSON_QUERY`, `JSON_EXTRACT`, `JSON_OBJECT`, `JSON_ARRAY_LENGTH`, `JSON_VALID`, `JSON_TYPE`
 - Array: `ARRAY_LENGTH`, `ARRAY_POSITION`, `ARRAY_SORT`, `CARDINALITY`
-- Table functions: `UNNEST`, `JSON_EACH`
+- Table functions: `UNNEST`, `EXPLODE`, `JSON_EACH`
 - Regex: `REGEXP_SUBSTR`, `REGEXP_EXTRACT`, `REGEXP_REPLACE`, `REGEXP_MATCHES`
 - Spatial: `ST_GeomFromText`, `ST_MakeEnvelope`, `ST_AsText`, `ST_Intersects`, `ST_Contains`, `ST_ContainsProperly`, `ST_Within`, `ST_Overlaps`, `ST_Touches`, `ST_Equals`, `ST_Crosses`, `ST_Covers`, `ST_CoveredBy`, `ST_DWithin`
 - Conditional: `COALESCE`, `NULLIF`, `GREATEST`, `LEAST`

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "squirreling",
-  "version": "0.12.11",
+  "version": "0.12.12",
   "description": "Squirreling Async SQL Engine",
   "author": "Hyperparam",
   "homepage": "https://hyperparam.app",

package/src/execute/execute.js

@@ -3,7 +3,7 @@ import { derivedAlias } from '../expression/alias.js'
 import { evaluateExpr } from '../expression/evaluate.js'
 import { parseSql } from '../parse/parse.js'
 import { planSql, planStatement } from '../plan/plan.js'
-import { fromAlias } from '../plan/columns.js'
+import { statementScope } from '../plan/columns.js'
 import { validateScan, validateTable } from '../validation/tables.js'
 import { executeHashAggregate, executeScalarAggregate } from './aggregates.js'
 import { executeHashJoin, executeNestedLoopJoin, executePositionalJoin } from './join.js'
@@ -72,18 +72,6 @@ export function executeStatement({ query, context, outerScope }) {
   return executePlan({ plan, context: scope ? { ...context, scope } : context })
 }
 
-/**
- * Extracts the table aliases from a statement's FROM and JOIN clauses.
- *
- * @param {Statement} stmt
- * @returns {string[] | undefined}
- */
-function statementScope(stmt) {
-  if (stmt.type === 'with') return statementScope(stmt.query)
-  if (stmt.type === 'compound') return undefined
-  return [fromAlias(stmt.from), ...stmt.joins.map(j => j.alias ?? j.table)]
-}
-
 /**
  * Executes a query plan and returns query results with row count estimates
  *
@@ -119,6 +107,8 @@ export function executePlan({ plan, context }) {
     return executeLimit(plan, context)
   } else if (plan.type === 'SetOperation') {
     return executeSetOperation(plan, context)
+  } else if (plan.type === 'Subquery') {
+    return executePlan({ plan: plan.child, context: { ...context, scope: plan.scope } })
   } else if (plan.type === 'TableFunction') {
     return executeTableFunction(plan, context)
   } else if (plan.type === 'Window') {
@@ -137,7 +127,7 @@ export function executePlan({ plan, context }) {
  * @returns {QueryResults}
  */
 function executeTableFunction(plan, context) {
-  if (plan.funcName === 'UNNEST') {
+  if (plan.funcName === 'UNNEST' || plan.funcName === 'EXPLODE') {
     return executeUnnest(plan, context)
   } else if (plan.funcName === 'JSON_EACH') {
     return executeJsonEach(plan, context)

package/src/execute/join.js

@@ -200,6 +200,7 @@ export function executeHashJoin(plan, context) {
     async *rows() {
       const leftTable = plan.leftAlias
       const rightTable = plan.rightAlias
+      const { leftKeys, rightKeys, residual } = plan
 
       // Buffer right rows and build hash map
       /** @type {AsyncRow[]} */
@@ -209,16 +210,16 @@ export function executeHashJoin(plan, context) {
         rightRows.push(row)
       }
 
-      /** @type {Map<any, AsyncRow[]>} */
+      /** @type {Map<string | number | bigint | boolean, AsyncRow[]>} */
       const hashMap = new Map()
       for (const rightRow of rightRows) {
-        const keyValue = await evaluateExpr({
-          node: plan.rightKey,
-          row: rightRow,
-          context,
-        })
-        if (keyValue == null) continue
-        const key = keyify(keyValue)
+        const keyValues = await Promise.all(
+          rightKeys.map(node => evaluateExpr({ node, row: rightRow, context }))
+        )
+        // SQL semantics: NULL never equals anything, so a row with any NULL
+        // join key is excluded from the hash table.
+        if (keyValues.some(v => v == null)) continue
+        const key = keyify(...keyValues)
         let bucket = hashMap.get(key)
         if (!bucket) {
           bucket = []
@@ -243,20 +244,28 @@ export function executeHashJoin(plan, context) {
           leftCols = leftRow.columns
         }
 
-        const keyValue = await evaluateExpr({
-          node: plan.leftKey,
-          row: leftRow,
-          context,
-        })
-        const key = keyify(keyValue)
-        const matchingRightRows = hashMap.get(key)
-
-        if (matchingRightRows?.length) {
-          for (const rightRow of matchingRightRows) {
-            matchedRightRows?.add(rightRow)
-            yield mergeRows(leftRow, rightRow, leftTable, rightTable)
+        const keyValues = await Promise.all(
+          leftKeys.map(node => evaluateExpr({ node, row: leftRow, context }))
+        )
+        let matched = false
+        if (!keyValues.some(v => v == null)) {
+          const key = keyify(...keyValues)
+          const candidates = hashMap.get(key)
+          if (candidates?.length) {
+            for (const rightRow of candidates) {
+              const merged = mergeRows(leftRow, rightRow, leftTable, rightTable)
+              if (residual) {
+                const ok = await evaluateExpr({ node: residual, row: merged, context })
+                if (!ok) continue
+              }
+              matched = true
+              matchedRightRows?.add(rightRow)
+              yield merged
+            }
           }
-        } else if (plan.joinType === 'LEFT' || plan.joinType === 'FULL') {
+        }
+
+        if (!matched && (plan.joinType === 'LEFT' || plan.joinType === 'FULL')) {
           const nullRight = createNullRow(rightCols)
           yield mergeRows(leftRow, nullRight, leftTable, rightTable)
         }

package/src/expression/evaluate.js

@@ -614,7 +614,10 @@ export async function evaluateExpr({ node, row, rowIndex, rows, context }) {
       return Math.trunc(num)
     }
     if (toType === 'BIGINT') {
-      return BigInt(val)
+      if (typeof val === 'bigint') return val
+      const num = Number(val)
+      if (isNaN(num)) return null
+      return BigInt(Math.trunc(num))
     }
     if (toType === 'FLOAT' || toType === 'REAL' || toType === 'DOUBLE') {
       const num = Number(val)

package/src/parse/joins.js

@@ -1,8 +1,9 @@
 import { expectNoAggregate } from '../validation/aggregates.js'
+import { isTableFunction, validateFunctionArgs } from '../validation/functions.js'
 import { ParseError } from '../validation/parseErrors.js'
 import { parseExpression } from './expression.js'
-import { isTableFunctionStart, parseFromFunction, parseTableAlias } from './parse.js'
-import { current, expect, match } from './state.js'
+import { isTableFunctionStart, parseFromFunction, parseTableAlias, tableFunctionColumnCount, tableFunctionDefaultColumns } from './parse.js'
+import { consume, current, expect, match } from './state.js'
 
 /**
  * @import { ExprNode, JoinClause, JoinType, ParserState } from '../types.js'
@@ -19,6 +20,96 @@ export function parseJoins(state) {
   while (true) {
     const tok = current(state)
 
+    // LATERAL VIEW [OUTER] func(args) tableAlias AS colAlias[, ...] (Spark/Hive style)
+    if (current(state).type === 'keyword' && current(state).value === 'LATERAL') {
+      const lateralStart = tok.positionStart
+      consume(state)
+      expect(state, 'keyword', 'VIEW')
+      const isOuter = match(state, 'keyword', 'OUTER')
+      const funcTok = current(state)
+      if (funcTok.type !== 'identifier' || !isTableFunction(funcTok.value.toUpperCase())) {
+        throw new ParseError({
+          message: 'LATERAL VIEW requires a table function like EXPLODE',
+          positionStart: funcTok.positionStart,
+          positionEnd: funcTok.positionEnd,
+        })
+      }
+      consume(state)
+      const funcName = funcTok.value.toUpperCase()
+      expect(state, 'paren', '(')
+      /** @type {ExprNode[]} */
+      const args = []
+      if (!match(state, 'paren', ')')) {
+        while (true) {
+          args.push(parseExpression(state))
+          if (!match(state, 'comma')) break
+        }
+        expect(state, 'paren', ')')
+      }
+      validateFunctionArgs(funcName, args.length, funcTok.positionStart, state.lastPos, state.functions)
+
+      const aliasTok = current(state)
+      if (aliasTok.type !== 'identifier') {
+        throw new ParseError({
+          message: 'LATERAL VIEW requires a table alias before AS',
+          positionStart: aliasTok.positionStart,
+          positionEnd: aliasTok.positionEnd,
+        })
+      }
+      consume(state)
+      const tableAlias = aliasTok.value
+
+      expect(state, 'keyword', 'AS')
+      /** @type {string[]} */
+      const columnAliases = []
+      const colStart = state.lastPos
+      while (true) {
+        const colTok = expect(state, 'identifier')
+        columnAliases.push(colTok.value)
+        if (!match(state, 'comma')) break
+      }
+      const maxCols = tableFunctionColumnCount(funcName)
+      if (columnAliases.length > maxCols) {
+        const colLabels = tableFunctionDefaultColumns(funcName).join(', ')
+        throw new ParseError({
+          message: maxCols === 1
+            ? `${funcName} produces a single column; only one column alias is allowed`
+            : `${funcName} produces at most ${maxCols} columns (${colLabels}); too many column aliases`,
+          positionStart: colStart,
+          positionEnd: state.lastPos,
+        })
+      }
+
+      /** @type {import('../ast.js').FromFunction} */
+      const fromFunction = {
+        type: 'function',
+        funcName,
+        args,
+        alias: tableAlias,
+        columnAliases,
+        positionStart: funcTok.positionStart,
+        positionEnd: state.lastPos,
+      }
+
+      /** @type {JoinType} */
+      const joinType = isOuter ? 'LEFT' : 'CROSS'
+      /** @type {ExprNode | undefined} */
+      const condition = isOuter
+        ? { type: 'literal', value: true, positionStart: lateralStart, positionEnd: state.lastPos }
+        : undefined
+
+      joins.push({
+        joinType,
+        table: funcName,
+        alias: tableAlias,
+        on: condition,
+        fromFunction,
+        positionStart: lateralStart,
+        positionEnd: state.lastPos,
+      })
+      continue
+    }
+
     // Comma-join: implicit CROSS JOIN LATERAL, currently only for table functions.
     if (match(state, 'comma')) {
       if (!isTableFunctionStart(state)) {

package/src/plan/columns.js

@@ -18,6 +18,22 @@ export function fromAlias(from) {
   return 'table'
 }
 
+/**
+ * Returns the FROM/JOIN aliases visible inside a statement's body — its
+ * lexical scope. Used to set context.scope when entering a derived-table
+ * subplan, so correlated subqueries inside resolve outer references against
+ * the right aliases. Returns undefined for compound statements (UNION etc.)
+ * which have no single scope.
+ *
+ * @param {Statement} stmt
+ * @returns {string[] | undefined}
+ */
+export function statementScope(stmt) {
+  if (stmt.type === 'with') return statementScope(stmt.query)
+  if (stmt.type === 'compound') return undefined
+  return [fromAlias(stmt.from), ...stmt.joins.map(j => j.alias ?? j.table)]
+}
+
 /**
  * Returns the output column names for a FROM table function, applying any
  * column aliases over the function's default column names.

package/src/plan/plan.js

@@ -4,7 +4,7 @@ import { findAggregate } from '../validation/aggregates.js'
 import { ParseError } from '../validation/parseErrors.js'
 import { ColumnNotFoundError, TableNotFoundError } from '../validation/tables.js'
 import { validateNoIdentifiers, validateScan, validateTableRefs } from '../validation/tables.js'
-import { collectScopeColumns, extractColumns, fromAlias, inferSelectSourceColumns, inferStatementColumns, tableFunctionColumnNames } from './columns.js'
+import { collectScopeColumns, extractColumns, fromAlias, inferSelectSourceColumns, inferStatementColumns, statementScope, tableFunctionColumnNames } from './columns.js'
 
 /**
  * @import { AsyncDataSource, ExprNode, DerivedColumn, IdentifierNode, JoinClause, OrderByItem, PlanSqlOptions, ScanOptions, SelectColumn, SelectStatement, SetOperationStatement, Statement, WindowFunctionNode } from '../types.js'
@@ -363,6 +363,14 @@ function planFrom({ select, ctePlans, cteColumns, hints, tables, outerScope }) {
         throw new ColumnNotFoundError({ missingColumn, availableColumns, ...select.from })
       }
     }
+    // Wrap with the inner SELECT's scope so correlated subqueries inside the
+    // derived table resolve outer references against the inner aliases, not
+    // the enclosing query's. Compound subqueries (UNION etc.) have no single
+    // scope and pass through unwrapped.
+    const innerScope = statementScope(select.from.query)
+    if (innerScope) {
+      return { type: 'Subquery', scope: innerScope, child: subPlan }
+    }
     return subPlan
   }
 }
@@ -447,18 +455,21 @@ function planJoin({ left, joins, leftTable, ctePlans, cteColumns, perTableColumn
     if (join.joinType === 'POSITIONAL') {
       plan = { type: 'PositionalJoin', leftAlias: currentLeftTable, rightAlias: rightTable, left: plan, right: rightScan }
     } else {
-      const keys = join.on && extractSimpleJoinKeys({ condition: join.on, leftTable: currentLeftTable, rightTable })
+      const keys = join.on && extractEquiKeys({ condition: join.on, leftTable: currentLeftTable, rightTable })
       if (keys) {
-        plan = {
+        /** @type {import('./types.d.ts').HashJoinNode} */
+        const hashJoin = {
           type: 'HashJoin',
           joinType: join.joinType,
           leftAlias: currentLeftTable,
           rightAlias: rightTable,
-          leftKey: keys.leftKey,
-          rightKey: keys.rightKey,
+          leftKeys: keys.leftKeys,
+          rightKeys: keys.rightKeys,
           left: plan,
           right: rightScan,
         }
+        if (keys.residual) hashJoin.residual = keys.residual
+        plan = hashJoin
       } else {
         plan = {
           type: 'NestedLoopJoin',
@@ -605,28 +616,89 @@ function normalizeIdentifiers(node, sourceColumns) {
 }
 
 /**
- * Extracts left and right key expressions from a simple equality join condition.
- * Returns undefined if the condition is not a simple equality between identifiers.
+ * Splits a join ON expression into equi-key pairs and a residual predicate so
+ * the planner can route AND-of-equis (with optional range/inequality
+ * conjuncts) to the hash-join path. Conjuncts of the form
+ * `<left-ref> = <right-ref>` between two identifiers become hash keys; every
+ * other conjunct stays as part of the residual that will run after the hash
+ * lookup. Returns undefined when no equi conjunct is present so the caller
+ * falls back to the nested-loop path.
  *
  * @param {object} options
  * @param {ExprNode} options.condition
  * @param {string} options.leftTable
  * @param {string} options.rightTable
- * @returns {{ leftKey: ExprNode, rightKey: ExprNode } | undefined}
+ * @returns {{ leftKeys: ExprNode[], rightKeys: ExprNode[], residual?: ExprNode } | undefined}
+ */
+function extractEquiKeys({ condition, leftTable, rightTable }) {
+  /** @type {ExprNode[]} */
+  const conjuncts = []
+  collectConjuncts(condition, conjuncts)
+  /** @type {ExprNode[]} */
+  const leftKeys = []
+  /** @type {ExprNode[]} */
+  const rightKeys = []
+  /** @type {ExprNode[]} */
+  const residuals = []
+  for (const conjunct of conjuncts) {
+    const eq = classifyEquiConjunct(conjunct, leftTable, rightTable)
+    if (eq) {
+      leftKeys.push(eq.leftKey)
+      rightKeys.push(eq.rightKey)
+    } else {
+      residuals.push(conjunct)
+    }
+  }
+  if (!leftKeys.length) return undefined
+  /** @type {ExprNode | undefined} */
+  let residual
+  for (const r of residuals) {
+    residual = residual === undefined
+      ? r
+      : { type: 'binary', op: 'AND', left: residual, right: r, positionStart: residual.positionStart, positionEnd: r.positionEnd }
+  }
+  return residual ? { leftKeys, rightKeys, residual } : { leftKeys, rightKeys }
+}
+
+/**
+ * Walks an ON expression, flattening top-level AND conjuncts. Non-AND nodes
+ * are pushed verbatim. Used to expose individual predicates so equi-keys can
+ * be lifted out of an AND chain.
+ *
+ * @param {ExprNode} node
+ * @param {ExprNode[]} out
  */
-function extractSimpleJoinKeys({ condition, leftTable, rightTable }) {
-  if (condition.type !== 'binary' || condition.op !== '=') return
-  const { left, right } = condition
-  if (left.type !== 'identifier' || right.type !== 'identifier') return
+function collectConjuncts(node, out) {
+  if (node.type === 'binary' && node.op === 'AND') {
+    collectConjuncts(node.left, out)
+    collectConjuncts(node.right, out)
+    return
+  }
+  out.push(node)
+}
 
-  // Check if keys are in swapped order (right table ref on left side)
+/**
+ * Returns the (leftKey, rightKey) pair for an equi conjunct, oriented so
+ * leftKey references the left input and rightKey references the right input.
+ * Returns undefined when the conjunct is not a `<identifier> = <identifier>`
+ * predicate. When the prefixes don't unambiguously identify a side, falls
+ * through to the original orientation — matches the prior single-equi
+ * behavior so unprefixed columns still produce a hash join.
+ *
+ * @param {ExprNode} conjunct
+ * @param {string} leftTable
+ * @param {string} rightTable
+ * @returns {{ leftKey: ExprNode, rightKey: ExprNode } | undefined}
+ */
+function classifyEquiConjunct(conjunct, leftTable, rightTable) {
+  if (conjunct.type !== 'binary' || conjunct.op !== '=') return undefined
+  const { left, right } = conjunct
+  if (left.type !== 'identifier' || right.type !== 'identifier') return undefined
   const leftRefsRight = left.prefix === rightTable
   const rightRefsLeft = right.prefix === leftTable
-
   if (leftRefsRight && rightRefsLeft) {
     return { leftKey: right, rightKey: left }
   }
-
   return { leftKey: left, rightKey: right }
 }
 

package/src/plan/types.d.ts

@@ -14,6 +14,7 @@ export type QueryPlan =
   | NestedLoopJoinNode
   | PositionalJoinNode
   | SetOperationNode
+  | SubqueryNode
   | TableFunctionNode
   | WindowNode
 
@@ -85,8 +86,11 @@ export interface HashJoinNode {
   joinType: JoinType
   leftAlias: string
   rightAlias: string
-  leftKey: ExprNode
-  rightKey: ExprNode
+  leftKeys: ExprNode[]
+  rightKeys: ExprNode[]
+  // Non-equi conjuncts from the ON clause (e.g. range predicates) applied to
+  // each merged candidate after the hash lookup succeeds.
+  residual?: ExprNode
   left: QueryPlan
   right: QueryPlan
 }
@@ -119,6 +123,17 @@ export interface SetOperationNode {
   right: QueryPlan
 }
 
+// Wraps a derived-table or CTE subplan with the lexical alias scope of its
+// inner SELECT, so the executor can set context.scope while traversing the
+// subtree. Correlated subqueries inside the subtree resolve outer references
+// against this scope, not whichever ancestor most recently went through
+// executeStatement.
+export interface SubqueryNode {
+  type: 'Subquery'
+  scope: string[]
+  child: QueryPlan
+}
+
 // Table-valued function (e.g. UNNEST) used in FROM clause
 export interface TableFunctionNode {
   type: 'TableFunction'

package/src/validation/functions.js

@@ -49,7 +49,7 @@ export function isRegexpFunc(name) {
  * @returns {boolean}
  */
 export function isTableFunction(name) {
-  return ['UNNEST', 'JSON_EACH'].includes(name)
+  return ['UNNEST', 'EXPLODE', 'JSON_EACH'].includes(name)
 }
 
 /**
@@ -187,6 +187,7 @@ export const FUNCTION_SIGNATURES = {
 
   // Table functions (used in FROM clause)
   UNNEST: { min: 1, max: 1, signature: 'array' },
+  EXPLODE: { min: 1, max: 1, signature: 'array' },
   JSON_EACH: { min: 1, max: 1, signature: 'value' },
 
   // Conditional functions

package/src/validation/keywords.js

@@ -3,7 +3,7 @@ export const KEYWORDS = new Set([
   'HAVING', 'ORDER', 'ASC', 'DESC', 'NULLS', 'LIMIT', 'OFFSET', 'AS', 'ALL',
   'DISTINCT', 'TRUE', 'FALSE', 'NULL', 'LIKE', 'IN', 'EXISTS', 'BETWEEN',
   'CASE', 'WHEN', 'THEN', 'ELSE', 'END', 'JOIN', 'INNER', 'LEFT', 'RIGHT',
-  'FULL', 'OUTER', 'CROSS', 'POSITIONAL', 'LATERAL', 'ON', 'INTERVAL', 'DAY', 'MONTH', 'YEAR',
+  'FULL', 'OUTER', 'CROSS', 'POSITIONAL', 'LATERAL', 'VIEW', 'ON', 'INTERVAL', 'DAY', 'MONTH', 'YEAR',
   'HOUR', 'MINUTE', 'SECOND', 'FILTER', 'WITHIN',
   'UNION', 'INTERSECT', 'EXCEPT',
 ])