squirreling 0.16.8 → 0.16.10

package/README.md

@@ -12,7 +12,7 @@
 
 Squirreling is a streaming async SQL engine in pure JavaScript. Built for the browser from the ground up: streaming input and output, pluggable data sources, and lazy async cell evaluation. This makes Squirreling ideal for querying data from network sources, APIs, or LLMs where latency and cost matter.
 
-> Part of **[HypStack](https://hypstack.ai/)**, an open-source stack for AI observability.
+> Powers **[HypAware](https://hypaware.ai/?utm_source=github-readme)**, the improvement loop for all your agents.
 
 - **Standard SQL**: Full SQL support for querying data (read-only)
 - **Async UDFs**: User-defined functions can call APIs or models
@@ -146,6 +146,7 @@ Squirreling mostly follows the SQL standard. The following features are supporte
 - `JOIN` operations: `INNER JOIN`, `LEFT JOIN`, `RIGHT JOIN`, `FULL JOIN`, `CROSS JOIN`, `POSITIONAL JOIN`, `LATERAL VIEW [OUTER] EXPLODE(...)`, with `ON` or `USING (col, ...)` conditions
 - `GROUP BY` and `HAVING` clauses, including `GROUP BY ALL`
 - Set operations: `UNION`, `UNION ALL`, `INTERSECT`, `INTERSECT ALL`, `EXCEPT`, `EXCEPT ALL`
+- `SHOW TABLES` to list the available tables
 - Expressions: `CASE`, `CAST`, `TRY_CAST`, `BETWEEN`, `IN`, `LIKE`, `IS NULL`, `IS NOT NULL`, string concatenation `||`
 - Subscript access: zero-based array indexing `col[0]`, struct field access `col['field']`, and chains like `col[0].field`
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "squirreling",
-  "version": "0.16.8",
+  "version": "0.16.10",
   "description": "Squirreling Async SQL Engine",
   "author": "Hyperparam",
   "homepage": "https://hyperparam.app",
@@ -40,12 +40,12 @@
   },
   "devDependencies": {
     "@eslint/js": "10.0.1",
-    "@types/node": "26.6.3",
-    "@vitest/coverage-v8": "5.0.2",
-    "eslint": "10.11.0",
-    "eslint-plugin-jsdoc": "65.0.0",
-    "globals": "17.12.0",
+    "@types/node": "26.6.5",
+    "@vitest/coverage-v8": "5.0.5",
+    "eslint": "10.13.0",
+    "eslint-plugin-jsdoc": "65.4.0",
+    "globals": "17.13.0",
     "typescript": "7.0.2",
-    "vitest": "5.0.2"
+    "vitest": "5.0.5"
   }
 }

package/src/ast.d.ts

@@ -41,7 +41,12 @@ export interface WithStatement extends AstBase {
   query: Statement
 }
 
-export type Statement = SelectStatement | SetOperationStatement | WithStatement
+// SHOW TABLES statement: lists the names of the available tables
+export interface ShowTablesStatement extends AstBase {
+  type: 'show tables'
+}
+
+export type Statement = SelectStatement | SetOperationStatement | WithStatement | ShowTablesStatement
 
 export interface CTEDefinition extends AstBase {
   name: string

package/src/execute/execute.js

@@ -15,6 +15,7 @@ import { executeHashJoin, executeNestedLoopJoin, executePositionalJoin } from '.
 import { referencesRowScope } from './rowScope.js'
 import { normalizeScanColumnResult } from './scanColumn.js'
 import { executeSort } from './sort.js'
+import { executeSemiJoin } from './semiJoin.js'
 import { addBounds, minBounds, stableRowKey } from './utils.js'
 import { executeWindow } from './window.js'
 import { yieldToEventLoop } from './yield.js'
@@ -37,6 +38,9 @@ const YIELD_INTERVAL = 4000
 export function executeSql({ tables, query, functions, signal }) {
   const parsed = typeof query === 'string' ? parseSql({ query, functions }) : query
 
+  // Listing names does not require table contents or inferred schemas.
+  if (parsed.type === 'show tables') return executeShowTables({ tables, signal })
+
   // Normalize tables: convert arrays to AsyncDataSource
   /** @type {Record<string, AsyncDataSource>} */
   const normalizedTables = {}
@@ -97,12 +101,16 @@ export function executePlan({ plan, context }) {
     return executeScan(plan, context)
   } else if (plan.type === 'SingleRow') {
     return executeSingleRow()
+  } else if (plan.type === 'ShowTables') {
+    return executeShowTables(context)
   } else if (plan.type === 'Count') {
     return executeCount(plan, context)
   } else if (plan.type === 'Filter') {
     return executeFilter(plan, context)
   } else if (plan.type === 'Project') {
     return executeProject(plan, context)
+  } else if (plan.type === 'SemiJoin') {
+    return executeSemiJoin(plan, context)
   } else if (plan.type === 'HashJoin') {
     return executeHashJoin(plan, context)
   } else if (plan.type === 'NestedLoopJoin') {
@@ -148,6 +156,31 @@ function executeSingleRow() {
   }
 }
 
+/**
+ * Executes SHOW TABLES: yields one row per available table name, sorted.
+ *
+ * @param {Pick<ExecuteSqlOptions, 'tables' | 'signal'>} context
+ * @returns {QueryResults}
+ */
+function executeShowTables(context) {
+  const columns = ['name']
+  const names = Object.keys(context.tables).sort()
+  return {
+    columns,
+    numRows: names.length,
+    maxRows: names.length,
+    async *rows() {
+      for (const name of names) {
+        context.signal?.throwIfAborted()
+        yield {
+          columns,
+          cells: { name: () => Promise.resolve(name) },
+        }
+      }
+    },
+  }
+}
+
 /**
  * Executes a table-valued function (e.g. UNNEST, JSON_EACH).
  * Evaluates the argument once against the outer row (for lateral joins) or an

package/src/execute/semiJoin.js

@@ -0,0 +1,77 @@
+import { evaluateExpr } from '../expression/evaluate.js'
+import { executePlan } from './execute.js'
+import { sqlEquals } from './utils.js'
+import { yieldToEventLoop } from './yield.js'
+
+/**
+ * @import { ExecuteContext, QueryResults, SqlPrimitive } from '../types.js'
+ * @import { SemiJoinNode } from '../plan/types.js'
+ */
+
+/**
+ * Executes WHERE IN / NOT IN with one build of the right input per iteration.
+ * The membership set belongs to this operator, never to the context or AST.
+ *
+ * @param {SemiJoinNode} plan
+ * @param {ExecuteContext} context
+ * @returns {QueryResults}
+ */
+export function executeSemiJoin(plan, context) {
+  const left = executePlan({ plan: plan.left, context })
+  return {
+    columns: left.columns,
+    maxRows: left.maxRows,
+    async *rows() {
+      /** @type {Map<string, Set<SqlPrimitive>>} */
+      const values = new Map()
+      let hasNull = false
+      let count = 0
+      let built = false
+      let rowIndex = 0
+      for await (const row of left.rows()) {
+        context.signal?.throwIfAborted()
+        if (++rowIndex % 4000 === 0) await yieldToEventLoop()
+        const value = await evaluateExpr({ node: plan.expr, row, rowIndex, context })
+        if (!built) {
+          for await (const inner of executePlan({ plan: plan.right, context }).rows()) {
+            if (++count % 4000 === 0) {
+              await yieldToEventLoop()
+              context.signal?.throwIfAborted()
+            }
+            const value = await inner.cells[inner.columns[0]]()
+            if (value == null) hasNull = true
+            else {
+              const type = typeof value
+              let group = values.get(type)
+              if (!group) values.set(type, group = new Set())
+              group.add(value)
+            }
+          }
+          built = true
+        }
+        const matched = count === 0 ? false : value == null ? null : contains(values, value) ? true : hasNull ? null : false
+        if (matched !== null && matched !== plan.not) yield row
+      }
+    },
+  }
+}
+
+/**
+ * Same-type primitives use a Set. Cross-type values and Dates retain the
+ * evaluator's SQL equality, including coercion and timestamp comparison.
+ *
+ * @param {Map<string, Set<SqlPrimitive>>} values
+ * @param {SqlPrimitive} value
+ * @returns {boolean}
+ */
+function contains(values, value) {
+  const type = typeof value
+  if (type !== 'object' && !(typeof value === 'number' && Number.isNaN(value)) && values.get(type)?.has(value)) return true
+  for (const [valueType, group] of values) {
+    if (valueType === type && type !== 'object') continue
+    for (const candidate of group) {
+      if (sqlEquals(value, candidate)) return true
+    }
+  }
+  return false
+}

package/src/index.d.ts

@@ -35,6 +35,7 @@ export type {
   ScanResults,
   SelectStatement,
   SetOperationStatement,
+  ShowTablesStatement,
   SqlType,
   SqlPrimitive,
   Statement,

package/src/parse/extractTables.js

@@ -46,6 +46,10 @@ function walkStatement(stmt, cteScope, refs) {
     for (const o of stmt.orderBy) walkExpr(o.expr, cteScope, refs)
     return
   }
+  if (stmt.type === 'show tables') {
+    // SHOW TABLES references no source tables
+    return
+  }
   // select
   if (!stmt.from) {
     // FROM-less SELECT (e.g. `SELECT 1`), no source tables

package/src/parse/parse.js

@@ -47,6 +47,17 @@ export function parseSql({ query, functions }) {
  */
 export function parseStatement(state) {
   const positionStart = state.lastPos
+  const showTok = current(state)
+  if (showTok.type === 'identifier' && showTok.value.toUpperCase() === 'SHOW') {
+    consume(state)
+    const tok = current(state)
+    if (tok.type !== 'identifier' || tok.value.toUpperCase() !== 'TABLES') {
+      throw parseError(state, 'TABLES')
+    }
+    consume(state)
+    match(state, 'semicolon')
+    return { type: 'show tables', positionStart: showTok.positionStart, positionEnd: state.lastPos }
+  }
   if (match(state, 'keyword', 'WITH')) {
     const recursiveTok = current(state)
     if (recursiveTok.type === 'identifier' && recursiveTok.value.toUpperCase() === 'RECURSIVE') {

package/src/plan/columns.js

@@ -40,7 +40,7 @@ export function fromAlias(from) {
  */
 export function statementScope(stmt) {
   if (stmt.type === 'with') return statementScope(stmt.query)
-  if (stmt.type === 'compound') return undefined
+  if (stmt.type === 'compound' || stmt.type === 'show tables') return undefined
   const alias = fromAlias(stmt.from)
   const joins = stmt.joins.map(j => j.alias ?? j.table)
   return alias === undefined ? joins : [alias, ...joins]
@@ -329,6 +329,7 @@ export function collectColumnsFromExpr(expr, columns, aliases, context) {
  * @param {ColumnCollectionContext} [context]
  */
 function collectCorrelatedColumnsFromStatement(stmt, columns, context) {
+  if (stmt.type === 'show tables') return
   if (stmt.type === 'compound') {
     collectCorrelatedColumnsFromStatement(stmt.left, columns, context)
     collectCorrelatedColumnsFromStatement(stmt.right, columns, context)
@@ -386,6 +387,7 @@ function collectCorrelatedColumnsFromStatement(stmt, columns, context) {
  * @param {ColumnCollectionContext} [context]
  */
 function collectFromFunctionArgs(stmt, columns, context) {
+  if (stmt.type === 'show tables') return
   if (stmt.type === 'compound') {
     collectFromFunctionArgs(stmt.left, columns, context)
     collectFromFunctionArgs(stmt.right, columns, context)
@@ -414,6 +416,9 @@ function collectFromFunctionArgs(stmt, columns, context) {
  * @returns {string[]}
  */
 export function inferStatementColumns({ stmt, cteColumns, tables }) {
+  if (stmt.type === 'show tables') {
+    return ['name']
+  }
   if (stmt.type === 'with') {
     return inferStatementColumns({ stmt: stmt.query, cteColumns, tables })
   }

package/src/plan/plan.js

@@ -9,7 +9,7 @@ import { collectColumnsFromExpr, collectScopeColumns, extractColumns, fromAlias,
 
 /**
  * @import { AsyncDataSource, DerivedColumn, ExprNode, FromFunction, IdentifierNode, JoinClause, OrderByItem, PlanSqlOptions, ScanOptions, SelectColumn, SelectStatement, SetOperationStatement, Statement, WindowFunctionNode } from '../types.js'
- * @import { HashJoinNode, QueryPlan, TableFunctionNode, WindowSpec } from './types.js'
+ * @import { HashJoinNode, QueryPlan, SemiJoinNode, TableFunctionNode, WindowSpec } from './types.js'
  */
 
 /**
@@ -38,6 +38,9 @@ export function planSql({ query, functions, tables, ctePlans, cteColumns }) {
  * @returns {QueryPlan}
  */
 export function planStatement({ stmt, ctePlans, cteColumns, tables, parentColumns, outerScope }) {
+  if (stmt.type === 'show tables') {
+    return { type: 'ShowTables' }
+  }
   if (stmt.type === 'with') {
     // Build CTE plans in order (each CTE can reference preceding CTEs)
     ctePlans ??= new Map()
@@ -227,9 +230,46 @@ function planSelect({ select, ctePlans, cteColumns, tables, parentColumns, outer
   if (hints.columns?.length === 0 && select.from?.type === 'subquery') {
     hints.columns = undefined
   }
+  // A WHERE membership test is a semi-join, not a row-by-row expression
+  // subquery. Keep the original SELECT for column demand analysis above.
+  /** @type {Omit<SemiJoinNode, 'left'>[]} */
+  const semiJoins = []
+  if (select.where && !outerScope?.length) {
+    /** @type {ExprNode[]} */
+    const conjuncts = []
+    collectConjuncts(select.where, conjuncts)
+    /** @type {ExprNode | undefined} */
+    let residual
+    for (const conjunct of conjuncts) {
+      const not = conjunct.type === 'unary' && conjunct.op === 'NOT'
+      const expr = not ? conjunct.argument : conjunct
+      if (expr.type !== 'in') {
+        residual = andPredicate(residual, conjunct)
+        continue
+      }
+      /** @type {IdentifierNode[]} */
+      const correlated = []
+      collectColumnsFromExpr({ type: 'subquery', subquery: expr.subquery, positionStart: expr.positionStart, positionEnd: expr.positionEnd }, correlated, undefined, {
+        tables, cteColumns, outerAliases: new Set(scopeAliases),
+      })
+      if (correlated.length) {
+        residual = andPredicate(residual, conjunct)
+        continue
+      }
+      const right = planStatement({ stmt: expr.subquery, tables, ctePlans, cteColumns })
+      const scope = statementScope(expr.subquery)
+      semiJoins.push({
+        type: 'SemiJoin',
+        expr: expr.expr,
+        not,
+        right: scope ? { type: 'Subquery', child: right, scope } : right,
+      })
+    }
+    select = { ...select, where: residual }
+  }
   if (!select.joins.length) {
     hints.where = select.where
-    if (!needsBuffering && !select.distinct) {
+    if (!needsBuffering && !select.distinct && !semiJoins.length) {
       hints.limit = select.limit
       hints.offset = select.offset
     }
@@ -259,6 +299,8 @@ function planSelect({ select, ctePlans, cteColumns, tables, parentColumns, outer
     plan = { type: 'Filter', condition: where, child: plan }
   }
 
+  for (const semiJoin of semiJoins) plan = { ...semiJoin, left: plan }
+
   if (useGrouping) {
     // Aggregation path: GROUP BY or scalar aggregate
     // HAVING is integrated into aggregate nodes for access to group context
@@ -343,7 +385,7 @@ function planSelect({ select, ctePlans, cteColumns, tables, parentColumns, outer
       plan = { type: 'Distinct', child: plan }
     }
 
-    if (!(isOwnScan && !needsBuffering && !select.distinct) && (select.limit !== undefined || select.offset)) {
+    if (!(isOwnScan && !needsBuffering && !select.distinct && !semiJoins.length) && (select.limit !== undefined || select.offset)) {
       if (select.limit !== undefined) pushLimitIntoSort(plan, select.limit, select.offset)
       plan = { type: 'Limit', limit: select.limit, offset: select.offset, child: plan }
     }
@@ -834,11 +876,10 @@ function collectConjuncts(node, out) {
 }
 
 /**
- * Push total, table-local comparisons into fresh scans of a direct INNER
- * equijoin. Scan execution already enforces hints a source cannot apply.
- * Validate ON before moving anything; an unsafe ON also blocks WHERE
- * pushdown. Safe ON terms can move even when WHERE must remain above the
- * join, since ON already excludes those pairs before WHERE is evaluated.
+ * Push each table-local conjunct into fresh scans of a direct INNER
+ * equijoin. Scan execution enforces hints a source cannot apply. Unresolved
+ * and cross-table conjuncts remain residual predicates. Predicate evaluation
+ * order and error short-circuiting are not guaranteed.
  *
  * @param {QueryPlan} plan
  * @param {ExprNode | undefined} where
@@ -862,30 +903,54 @@ function pushJoinPredicates(plan, where, tables) {
    */
   function owner(expr) {
     if (expr.type === 'literal') return 'literal'
-    if (expr.type !== 'identifier' || !expr.prefix) return 'unsafe'
-    const index = aliases.indexOf(expr.prefix)
+    if (expr.type !== 'identifier') return 'unsafe'
+    const matches = columns.map((names, i) => names.includes(expr.name) ? i : -1).filter(i => i >= 0)
+    const index = expr.prefix ? aliases.indexOf(expr.prefix) : matches.length === 1 ? matches[0] : -1
     if (index < 0 || !columns[index].includes(expr.name)) return 'unsafe'
     return index === 0 ? 'left' : 'right'
   }
 
   // Avoid changing the evaluation of unresolved or unqualified ON keys.
   if (plan.leftKeys.some(key => owner(key) !== 'left') || plan.rightKeys.some(key => owner(key) !== 'right')) return where
+  /**
+   * Resolve every reference in a conjunct independently. Subqueries and
+   * windows have their own scope and must remain above the join.
+   *
+   * @param {ExprNode} expr
+   * @returns {'left' | 'right' | 'literal' | 'unsafe'}
+   */
+  function expressionOwner(expr) {
+    if (!expr) return 'literal'
+    if (expr.type === 'identifier' || expr.type === 'literal') return owner(expr)
+    /** @type {ExprNode[]} */
+    let children
+    if (expr.type === 'binary') children = [expr.left, expr.right]
+    else if (expr.type === 'unary') children = [expr.argument]
+    else if (expr.type === 'function' && !expr.filter) children = expr.args
+    else if (expr.type === 'cast') children = [expr.expr]
+    else if (expr.type === 'in valuelist') children = [expr.expr, ...expr.values]
+    else if (expr.type === 'subscript') children = [expr.expr, expr.index]
+    else if (expr.type === 'case') children = [expr.caseExpr, ...expr.whenClauses.flatMap(w => [w.condition, w.result]), expr.elseResult]
+    else return 'unsafe'
+    /** @type {'left' | 'right' | 'literal'} */
+    let target = 'literal'
+    for (const child of children) {
+      const side = expressionOwner(child)
+      if (side === 'unsafe' || side !== 'literal' && target !== 'literal' && side !== target) return 'unsafe'
+      if (side !== 'literal') target = side
+    }
+    return target
+  }
+
   /**
    * @param {ExprNode[]} conjuncts
-   * @returns {Array<'left' | 'right' | 'residual'> | undefined}
+   * @returns {Array<'left' | 'right' | 'residual'>}
    */
   function classify(conjuncts) {
-    /** @type {Array<'left' | 'right' | 'residual'>} */
-    const targets = []
-    for (const expr of conjuncts) {
-      if (expr.type !== 'binary' || !['=', '==', '!=', '<>', '<', '<=', '>', '>='].includes(expr.op)) return undefined
-      const left = owner(expr.left)
-      const right = owner(expr.right)
-      if (left === 'unsafe' || right === 'unsafe') return undefined
-      // Only column/literal terms move; cross-table comparisons remain residual.
-      targets.push(left !== 'literal' && right === 'literal' ? left : right !== 'literal' && left === 'literal' ? right : 'residual')
-    }
-    return targets
+    return conjuncts.map(function target(expr) {
+      const side = expressionOwner(expr)
+      return side === 'left' || side === 'right' ? side : 'residual'
+    })
   }
 
   /**
@@ -912,7 +977,6 @@ function pushJoinPredicates(plan, where, tables) {
   const onConjuncts = []
   if (plan.residual) collectConjuncts(plan.residual, onConjuncts)
   const onTargets = classify(onConjuncts)
-  if (!onTargets) return where
   const onResidual = push(onConjuncts, onTargets)
   if (onResidual) plan.residual = onResidual
   else delete plan.residual
@@ -922,7 +986,7 @@ function pushJoinPredicates(plan, where, tables) {
   const whereConjuncts = []
   collectConjuncts(where, whereConjuncts)
   const whereTargets = classify(whereConjuncts)
-  return whereTargets ? push(whereConjuncts, whereTargets) : where
+  return push(whereConjuncts, whereTargets)
 }
 
 /**

package/src/plan/types.d.ts

@@ -11,10 +11,12 @@ export type QueryPlan =
   | LimitNode
   | HashAggregateNode
   | ScalarAggregateNode
+  | SemiJoinNode
   | HashJoinNode
   | NestedLoopJoinNode
   | PositionalJoinNode
   | SetOperationNode
+  | ShowTablesNode
   | SubqueryNode
   | TableFunctionNode
   | WindowNode
@@ -35,6 +37,11 @@ export interface SingleRowNode {
   type: 'SingleRow'
 }
 
+// SHOW TABLES: yields one row per available table name
+export interface ShowTablesNode {
+  type: 'ShowTables'
+}
+
 // Count node for COUNT(*) optimization
 export interface CountNode {
   type: 'Count'
@@ -94,6 +101,15 @@ export interface ScalarAggregateNode {
 }
 
 // Join nodes
+// WHERE IN / NOT IN: retain left rows using membership in the right output.
+export interface SemiJoinNode {
+  type: 'SemiJoin'
+  left: QueryPlan
+  right: QueryPlan
+  expr: ExprNode
+  not: boolean
+}
+
 export interface HashJoinNode {
   type: 'HashJoin'
   joinType: JoinType