squirreling 0.4.8 → 0.5.0

package/src/parse/state.js

@@ -1,4 +1,4 @@
-import { syntaxError } from '../errors.js'
+import { syntaxError } from '../parseErrors.js'
 
 /**
  * @import { ParserState, Token, TokenType } from '../types.js'
@@ -31,12 +31,22 @@ export function peekToken(state, offset) {
  */
 export function consume(state) {
   const tok = current(state)
+  state.lastPos = tok.positionEnd
   if (state.pos < state.tokens.length - 1) {
     state.pos += 1
   }
   return tok
 }
 
+/**
+ * Gets the position after the last consumed token.
+ * @param {ParserState} state
+ * @returns {number}
+ */
+export function lastPosition(state) {
+  return state.lastPos ?? 0
+}
+
 /**
  * @param {ParserState} state
  * @param {TokenType} type
@@ -83,12 +93,12 @@ export function expectIdentifier(state) {
  * Helper function to create consistent parser error messages.
  * @param {ParserState} state
  * @param {string} expected - Description of what was expected
- * @returns {Error}
+ * @returns {import('../parseErrors.js').ParseError}
  */
 export function parseError(state, expected) {
   const tok = current(state)
   const prevToken = state.tokens[state.pos - 1]
   const after = prevToken ? prevToken.originalValue ?? prevToken.value : undefined
   const received = tok.type === 'eof' ? 'end of query' : `"${tok.originalValue ?? tok.value}"`
-  return syntaxError({ expected, received, position: tok.position, after })
+  return syntaxError({ expected, received, positionStart: tok.positionStart, positionEnd: tok.positionEnd, after })
 }

package/src/parse/tokenize.js

@@ -2,7 +2,7 @@ import {
   invalidLiteralError,
   unexpectedCharError,
   unterminatedError,
-} from '../errors.js'
+} from '../parseErrors.js'
 
 /**
  * @import { Token } from '../types.d.ts'
@@ -117,24 +117,26 @@ export function tokenize(sql) {
         return {
           type: 'number',
           value: text,
-          position: startPos,
+          positionStart: startPos,
+          positionEnd: i,
           numericValue: BigInt(text.slice(0, -1)),
         }
       } catch {
-        throw invalidLiteralError({ type: 'bigint', value: text.slice(0, -1), position: startPos })
+        throw invalidLiteralError({ type: 'bigint', value: text.slice(0, -1), positionStart: startPos, positionEnd: i })
       }
     }
     if (isAlpha(peek())) {
-      throw invalidLiteralError({ type: 'number', value: text + peek(), position: startPos })
+      throw invalidLiteralError({ type: 'number', value: text + peek(), positionStart: startPos, positionEnd: i + 1 })
     }
     const num = parseFloat(text)
     if (isNaN(num)) {
-      throw invalidLiteralError({ type: 'number', value: text, position: startPos })
+      throw invalidLiteralError({ type: 'number', value: text, positionStart: startPos, positionEnd: i })
     }
     return {
       type: 'number',
       value: text,
-      position: startPos,
+      positionStart: startPos,
+      positionEnd: i,
       numericValue: num,
     }
   }
@@ -204,13 +206,15 @@ export function tokenize(sql) {
           type: 'keyword',
           value: upper,
           originalValue: text,
-          position: pos,
+          positionStart: pos,
+          positionEnd: i,
         })
       } else {
         tokens.push({
           type: 'identifier',
           value: text,
-          position: pos,
+          positionStart: pos,
+          positionEnd: i,
         })
       }
       continue
@@ -222,7 +226,7 @@ export function tokenize(sql) {
       let text = ''
       while (i <= length) {
         if (i === length) {
-          throw unterminatedError('string', pos)
+          throw unterminatedError('string', pos, length)
         }
         const c = nextChar()
         if (c === quote) {
@@ -239,7 +243,8 @@ export function tokenize(sql) {
       tokens.push({
         type: 'string',
         value: text,
-        position: pos,
+        positionStart: pos,
+        positionEnd: i,
       })
       continue
     }
@@ -250,7 +255,7 @@ export function tokenize(sql) {
       let text = ''
       while (i <= length) {
         if (i === length) {
-          throw unterminatedError('identifier', pos)
+          throw unterminatedError('identifier', pos, length)
         }
         const c = nextChar()
         if (c === quote) {
@@ -267,7 +272,8 @@ export function tokenize(sql) {
       tokens.push({
         type: 'identifier',
         value: text,
-        position: pos,
+        positionStart: pos,
+        positionEnd: i,
       })
       continue
     }
@@ -283,7 +289,8 @@ export function tokenize(sql) {
       tokens.push({
         type: 'operator',
         value: op,
-        position: pos,
+        positionStart: pos,
+        positionEnd: i,
       })
       continue
     }
@@ -294,7 +301,8 @@ export function tokenize(sql) {
       tokens.push({
         type: 'operator',
         value: ch,
-        position: pos,
+        positionStart: pos,
+        positionEnd: i,
       })
       continue
     }
@@ -304,7 +312,8 @@ export function tokenize(sql) {
       tokens.push({
         type: 'comma',
         value: ',',
-        position: pos,
+        positionStart: pos,
+        positionEnd: i,
       })
       continue
     }
@@ -314,7 +323,8 @@ export function tokenize(sql) {
       tokens.push({
         type: 'dot',
         value: '.',
-        position: pos,
+        positionStart: pos,
+        positionEnd: i,
       })
       continue
     }
@@ -324,7 +334,8 @@ export function tokenize(sql) {
       tokens.push({
         type: 'paren',
         value: ch,
-        position: pos,
+        positionStart: pos,
+        positionEnd: i,
       })
       continue
     }
@@ -334,21 +345,23 @@ export function tokenize(sql) {
       tokens.push({
         type: 'semicolon',
         value: ';',
-        position: pos,
+        positionStart: pos,
+        positionEnd: i,
       })
       continue
     }
 
     if (tokens.length === 0) {
-      throw unexpectedCharError(ch, pos, true)
+      throw unexpectedCharError({ char: ch, positionStart: pos, expectsSelect: true })
     }
-    throw unexpectedCharError(ch, pos)
+    throw unexpectedCharError({ char: ch, positionStart: pos })
   }
 
   tokens.push({
     type: 'eof',
     value: '',
-    position: length,
+    positionStart: length,
+    positionEnd: length,
   })
 
   return tokens

package/src/parseErrors.js

@@ -0,0 +1,117 @@
+// ============================================================================
+// PARSE ERRORS - Issues during SQL tokenization and parsing
+// ============================================================================
+
+/**
+ * Structured parse error with position range.
+ */
+export class ParseError extends Error {
+  /**
+   * @param {string} message - Human-readable error message
+   * @param {number} positionStart - Start position (0-based character offset)
+   * @param {number} positionEnd - End position (exclusive, 0-based character offset)
+   */
+  constructor(message, positionStart, positionEnd) {
+    super(message)
+    this.name = 'ParseError'
+    this.positionStart = positionStart
+    this.positionEnd = positionEnd
+  }
+}
+
+/**
+ * General syntax error for unexpected tokens.
+ *
+ * @param {Object} options
+ * @param {string} options.expected - Description of what was expected
+ * @param {string} options.received - What was actually found
+ * @param {number} options.positionStart - Start character position in query
+ * @param {number} options.positionEnd - End character position in query
+ * @param {string} [options.after] - What token came before (for context)
+ * @returns {ParseError}
+ */
+export function syntaxError({ expected, received, positionStart, positionEnd, after }) {
+  const afterClause = after ? ` after "${after}"` : ''
+  return new ParseError(`Expected ${expected}${afterClause} but found ${received} at position ${positionStart}`, positionStart, positionEnd)
+}
+
+/**
+ * Error for unterminated literals (strings, identifiers).
+ *
+ * @param {'string' | 'identifier'} type - Type of unterminated literal
+ * @param {number} positionStart - Starting position
+ * @param {number} positionEnd - End position
+ * @returns {ParseError}
+ */
+export function unterminatedError(type, positionStart, positionEnd) {
+  const name = type === 'string' ? 'string literal' : 'identifier'
+  return new ParseError(`Unterminated ${name} starting at position ${positionStart}`, positionStart, positionEnd)
+}
+
+/**
+ * Error for invalid literals (numbers, intervals, etc).
+ *
+ * @param {Object} options
+ * @param {string} options.type - Type of invalid literal (e.g., 'number', 'interval value', 'interval unit')
+ * @param {string} options.value - The invalid value
+ * @param {number} options.positionStart - Start position in query
+ * @param {number} options.positionEnd - End position in query
+ * @param {string} [options.validValues] - List of valid values (for enums like interval units)
+ * @returns {ParseError}
+ */
+export function invalidLiteralError({ type, value, positionStart, positionEnd, validValues }) {
+  const suffix = validValues ? `. Valid values: ${validValues}` : ''
+  return new ParseError(`Invalid ${type} ${value} at position ${positionStart}${suffix}`, positionStart, positionEnd)
+}
+
+/**
+ * Error for unexpected characters during tokenization.
+ *
+ * @param {Object} options
+ * @param {string} options.char - The unexpected character
+ * @param {number} options.positionStart - Position in query
+ * @param {boolean} [options.expectsSelect=false] - Whether SELECT was expected (first token)
+ * @returns {ParseError}
+ */
+export function unexpectedCharError({ char, positionStart, expectsSelect = false }) {
+  const positionEnd = positionStart + 1
+  if (expectsSelect) {
+    return new ParseError(`Expected SELECT but found "${char}" at position ${positionStart}. Queries must start with SELECT.`, positionStart, positionEnd)
+  }
+  return new ParseError(`Unexpected character "${char}" at position ${positionStart}`, positionStart, positionEnd)
+}
+
+/**
+ * Error for unknown/unsupported functions.
+ *
+ * @param {Object} options
+ * @param {string} options.funcName - The unknown function name
+ * @param {number} options.positionStart - Start position in query
+ * @param {number} options.positionEnd - End position in query
+ * @param {string} [options.validFunctions] - List of valid functions
+ * @returns {ParseError}
+ */
+export function unknownFunctionError({ funcName, positionStart, positionEnd, validFunctions }) {
+  const supported = validFunctions ||
+    'COUNT, SUM, AVG, MIN, MAX, UPPER, LOWER, CONCAT, LENGTH, SUBSTRING, TRIM, REPLACE, FLOOR, CEIL, ABS, MOD, EXP, LN, LOG10, POWER, SQRT, JSON_OBJECT, JSON_VALUE, JSON_QUERY, JSON_ARRAYAGG'
+
+  return new ParseError(
+    `Unknown function "${funcName}" at position ${positionStart}. Supported: ${supported}`,
+    positionStart,
+    positionEnd
+  )
+}
+
+/**
+ * Error for missing required clause or structure.
+ *
+ * @param {Object} options
+ * @param {string} options.missing - What is missing (e.g., 'WHEN clause', 'FROM clause', 'ON condition')
+ * @param {string} options.context - Where it's missing from (e.g., 'CASE expression', 'SELECT statement', 'JOIN')
+ * @param {number} [options.positionStart] - Start position in query
+ * @param {number} [options.positionEnd] - End position in query
+ * @returns {ParseError}
+ */
+export function missingClauseError({ missing, context, positionStart, positionEnd }) {
+  return new ParseError(`${context} requires ${missing}`, positionStart ?? 0, positionEnd ?? 0)
+}

package/src/types.d.ts

@@ -6,6 +6,11 @@
 export interface QueryHints {
   columns?: string[] // columns needed
   where?: ExprNode // where clause
+  // important: only apply limit/offset if where is fully applied by the data source
+  // otherwise, the data source must return at least enough rows to ensure the engine
+  // can apply limit/offset correctly after filtering
+  // even with offset, the datasource must return rows starting from offset 0
+  // but doesn't need to resolve async rows before the offset
   limit?: number
   offset?: number
 }
@@ -68,54 +73,59 @@ export type BinaryOp = 'AND' | 'OR' | 'LIKE' | ComparisonOp | ArithmeticOp
 
 export type ComparisonOp = '=' | '!=' | '<>' | '<' | '>' | '<=' | '>='
 
-export interface LiteralNode {
+export interface ExprNodeBase {
+  positionStart: number
+  positionEnd: number
+}
+
+export interface LiteralNode extends ExprNodeBase {
   type: 'literal'
   value: SqlPrimitive
 }
 
-export interface IdentifierNode {
+export interface IdentifierNode extends ExprNodeBase {
   type: 'identifier'
   name: string
 }
 
-export interface UnaryNode {
+export interface UnaryNode extends ExprNodeBase {
   type: 'unary'
   op: 'NOT' | 'IS NULL' | 'IS NOT NULL' | '-'
   argument: ExprNode
 }
 
-export interface BinaryNode {
+export interface BinaryNode extends ExprNodeBase {
   type: 'binary'
   op: BinaryOp
   left: ExprNode
   right: ExprNode
 }
 
-export interface FunctionNode {
+export interface FunctionNode extends ExprNodeBase {
   type: 'function'
   name: string
   args: ExprNode[]
 }
 
-export interface CastNode {
+export interface CastNode extends ExprNodeBase {
   type: 'cast'
   expr: ExprNode
   toType: string
 }
 
-export interface InSubqueryNode {
+export interface InSubqueryNode extends ExprNodeBase {
   type: 'in'
   expr: ExprNode
   subquery: SelectStatement
 }
 
-export interface InValuesNode {
+export interface InValuesNode extends ExprNodeBase {
   type: 'in valuelist'
   expr: ExprNode
   values: ExprNode[]
 }
 
-export interface ExistsNode {
+export interface ExistsNode extends ExprNodeBase {
   type: 'exists' | 'not exists'
   subquery: SelectStatement
 }
@@ -125,21 +135,21 @@ export interface WhenClause {
   result: ExprNode
 }
 
-export interface CaseNode {
+export interface CaseNode extends ExprNodeBase {
   type: 'case'
   caseExpr?: ExprNode
   whenClauses: WhenClause[]
   elseResult?: ExprNode
 }
 
-export interface SubqueryNode {
+export interface SubqueryNode extends ExprNodeBase {
   type: 'subquery'
   subquery: SelectStatement
 }
 
 export type IntervalUnit = 'DAY' | 'MONTH' | 'YEAR' | 'HOUR' | 'MINUTE' | 'SECOND'
 
-export interface IntervalNode {
+export interface IntervalNode extends ExprNodeBase {
   type: 'interval'
   value: number
   unit: IntervalUnit
@@ -167,6 +177,18 @@ export interface StarColumn {
 
 export type AggregateFunc = 'COUNT' | 'SUM' | 'AVG' | 'MIN' | 'MAX' | 'JSON_ARRAYAGG'
 
+export type MathFunc =
+  | 'FLOOR'
+  | 'CEIL'
+  | 'CEILING'
+  | 'ABS'
+  | 'MOD'
+  | 'EXP'
+  | 'LN'
+  | 'LOG10'
+  | 'POWER'
+  | 'SQRT'
+
 export type StringFunc =
   | 'UPPER'
   | 'LOWER'
@@ -228,6 +250,7 @@ export interface JoinClause {
 export interface ParserState {
   tokens: Token[]
   pos: number
+  lastPos?: number
 }
 
 // Tokenizer types
@@ -246,7 +269,8 @@ export type TokenType =
 export interface Token {
   type: TokenType
   value: string
-  position: number
+  positionStart: number
+  positionEnd: number
   numericValue?: number | bigint
   originalValue?: string
 }

package/src/validation.js

@@ -1,6 +1,6 @@
 
 /**
- * @import {AggregateFunc, BinaryOp, ComparisonOp, IntervalUnit, StringFunc} from './types.js'
+ * @import {AggregateFunc, BinaryOp, ComparisonOp, IntervalUnit, MathFunc, StringFunc} from './types.js'
  * @param {string} name
  * @returns {name is AggregateFunc}
  */
@@ -8,6 +8,17 @@ export function isAggregateFunc(name) {
   return ['COUNT', 'SUM', 'AVG', 'MIN', 'MAX', 'JSON_ARRAYAGG'].includes(name)
 }
 
+/**
+ * @param {string} name
+ * @returns {name is MathFunc}
+ */
+export function isMathFunc(name) {
+  return [
+    'FLOOR', 'CEIL', 'CEILING', 'ABS', 'MOD',
+    'EXP', 'LN', 'LOG10', 'POWER', 'SQRT',
+  ].includes(name)
+}
+
 /**
  * @param {string} name
  * @returns {name is IntervalUnit}

package/src/validationErrors.js

@@ -0,0 +1,127 @@
+import { ExecutionError } from './executionErrors.js'
+
+// ============================================================================
+// VALIDATION ERRORS - Function argument and type validation
+// ============================================================================
+
+/**
+ * Function signatures for helpful error messages.
+ * Maps function name to its parameter signature.
+ * @type {Record<string, string>}
+ */
+const FUNCTION_SIGNATURES = {
+  // String functions
+  UPPER: 'string',
+  LOWER: 'string',
+  LENGTH: 'string',
+  TRIM: 'string',
+  REPLACE: 'string, search, replacement',
+  SUBSTRING: 'string, start[, length]',
+  SUBSTR: 'string, start[, length]',
+  CONCAT: 'value1, value2[, ...]',
+
+  // Date/time functions
+  RANDOM: '',
+  RAND: '',
+  CURRENT_DATE: '',
+  CURRENT_TIME: '',
+  CURRENT_TIMESTAMP: '',
+
+  // Math functions
+  FLOOR: 'number',
+  CEIL: 'number',
+  CEILING: 'number',
+  ABS: 'number',
+  MOD: 'dividend, divisor',
+  EXP: 'number',
+  LN: 'number',
+  LOG10: 'number',
+  POWER: 'base, exponent',
+  SQRT: 'number',
+
+  // JSON functions
+  JSON_VALUE: 'expression, path',
+  JSON_QUERY: 'expression, path',
+  JSON_OBJECT: 'key1, value1[, ...]',
+  JSON_ARRAYAGG: 'expression',
+
+  // Aggregate functions
+  COUNT: 'expression',
+  SUM: 'expression',
+  AVG: 'expression',
+  MIN: 'expression',
+  MAX: 'expression',
+}
+
+/**
+ * Error for wrong number of function arguments.
+ *
+ * @param {Object} options
+ * @param {string} options.funcName - The function name
+ * @param {number | string} options.expected - Expected count (number or range like "2 or 3")
+ * @param {number} options.received - Actual argument count
+ * @param {number} options.positionStart - Start position in query
+ * @param {number} options.positionEnd - End position in query
+ * @param {number} [options.rowNumber] - 1-based row number where error occurred
+ * @returns {ExecutionError}
+ */
+export function argCountError({ funcName, expected, received, positionStart, positionEnd, rowNumber }) {
+  const signature = FUNCTION_SIGNATURES[funcName] ?? ''
+  let expectedStr = `${expected} arguments`
+  if (expected === 0) expectedStr = 'no arguments'
+  if (expected === 1) expectedStr = '1 argument'
+  if (typeof expected === 'string' && expected.endsWith(' 1')) {
+    expectedStr = `${expected} argument`
+  }
+
+  return new ExecutionError(`${funcName}(${signature}) function requires ${expectedStr}, got ${received}`, positionStart, positionEnd, rowNumber)
+}
+
+/**
+ * Error for invalid argument type or value.
+ *
+ * @param {Object} options
+ * @param {string} options.funcName - The function name
+ * @param {string} options.message - Specific error message
+ * @param {number} options.positionStart - Start position in query
+ * @param {number} options.positionEnd - End position in query
+ * @param {string} [options.hint] - Recovery hint
+ * @param {number} [options.rowNumber] - 1-based row number where error occurred
+ * @returns {ExecutionError}
+ */
+export function argValueError({ funcName, message, positionStart, positionEnd, hint, rowNumber }) {
+  const signature = FUNCTION_SIGNATURES[funcName] ?? ''
+  const suffix = hint ? `. ${hint}` : ''
+  return new ExecutionError(`${funcName}(${signature}): ${message}${suffix}`, positionStart, positionEnd, rowNumber)
+}
+
+/**
+ * Error for aggregate function misuse (e.g., SUM(*)).
+ *
+ * @param {Object} options
+ * @param {string} options.funcName - The aggregate function
+ * @param {string} options.issue - What's wrong (e.g., "(*) is not supported")
+ * @returns {Error}
+ */
+export function aggregateError({ funcName, issue }) {
+  return new Error(`${funcName}${issue}. Only COUNT supports *. Use a column name for ${funcName}.`)
+}
+
+/**
+ * Error for unsupported CAST type.
+ *
+ * @param {Object} options
+ * @param {string} options.toType - The unsupported target type
+ * @param {number} options.positionStart - Start position in query
+ * @param {number} options.positionEnd - End position in query
+ * @param {string} [options.fromType] - The source type (optional)
+ * @param {number} [options.rowNumber] - 1-based row number where error occurred
+ * @returns {ExecutionError}
+ */
+export function castError({ toType, positionStart, positionEnd, fromType, rowNumber }) {
+  const message = fromType
+    ? `Cannot CAST ${fromType} to ${toType}`
+    : `Unsupported CAST to type ${toType}`
+
+  return new ExecutionError(`${message}. Supported types: TEXT, VARCHAR, INTEGER, INT, BIGINT, FLOAT, REAL, DOUBLE, BOOLEAN`, positionStart, positionEnd, rowNumber)
+}