just-bash 0.1.5__py3-none-any.whl

just_bash/query_engine/tokenizer.py

@@ -0,0 +1,329 @@
+"""Tokenizer for jq expressions.
+
+Converts a jq expression string into a sequence of tokens.
+"""
+
+from .types import Token, TokenType
+
+# Keywords mapping
+KEYWORDS: dict[str, TokenType] = {
+    "and": TokenType.AND,
+    "or": TokenType.OR,
+    "not": TokenType.NOT,
+    "if": TokenType.IF,
+    "then": TokenType.THEN,
+    "elif": TokenType.ELIF,
+    "else": TokenType.ELSE,
+    "end": TokenType.END,
+    "as": TokenType.AS,
+    "try": TokenType.TRY,
+    "catch": TokenType.CATCH,
+    "true": TokenType.TRUE,
+    "false": TokenType.FALSE,
+    "null": TokenType.NULL,
+    "reduce": TokenType.REDUCE,
+    "foreach": TokenType.FOREACH,
+}
+
+
+def tokenize(input_str: str) -> list[Token]:
+    """Tokenize a jq expression string into a list of tokens.
+
+    Args:
+        input_str: The jq expression to tokenize
+
+    Returns:
+        A list of Token objects
+
+    Raises:
+        ValueError: If an unexpected character is encountered
+    """
+    tokens: list[Token] = []
+    pos = 0
+    length = len(input_str)
+
+    def peek(offset: int = 0) -> str:
+        """Look at character at current position + offset."""
+        idx = pos + offset
+        return input_str[idx] if idx < length else ""
+
+    def advance() -> str:
+        """Advance position and return the character."""
+        nonlocal pos
+        c = input_str[pos] if pos < length else ""
+        pos += 1
+        return c
+
+    def is_eof() -> bool:
+        """Check if at end of input."""
+        return pos >= length
+
+    def is_digit(c: str) -> bool:
+        """Check if character is a digit."""
+        return c >= "0" and c <= "9"
+
+    def is_alpha(c: str) -> bool:
+        """Check if character is alphabetic or underscore."""
+        return (c >= "a" and c <= "z") or (c >= "A" and c <= "Z") or c == "_"
+
+    def is_alnum(c: str) -> bool:
+        """Check if character is alphanumeric or underscore."""
+        return is_alpha(c) or is_digit(c)
+
+    while not is_eof():
+        start = pos
+        c = advance()
+
+        # Whitespace - skip
+        if c in " \t\n\r":
+            continue
+
+        # Comments - skip to end of line
+        if c == "#":
+            while not is_eof() and peek() != "\n":
+                advance()
+            continue
+
+        # Two-character operators (must check before single-char)
+
+        # .. (recurse)
+        if c == "." and peek() == ".":
+            advance()
+            tokens.append(Token(TokenType.DOTDOT, None, start))
+            continue
+
+        # == (equality)
+        if c == "=" and peek() == "=":
+            advance()
+            tokens.append(Token(TokenType.EQ, None, start))
+            continue
+
+        # != (not equal)
+        if c == "!" and peek() == "=":
+            advance()
+            tokens.append(Token(TokenType.NE, None, start))
+            continue
+
+        # <= (less than or equal)
+        if c == "<" and peek() == "=":
+            advance()
+            tokens.append(Token(TokenType.LE, None, start))
+            continue
+
+        # >= (greater than or equal)
+        if c == ">" and peek() == "=":
+            advance()
+            tokens.append(Token(TokenType.GE, None, start))
+            continue
+
+        # // and //= (alternative and update alternative)
+        if c == "/" and peek() == "/":
+            advance()
+            if peek() == "=":
+                advance()
+                tokens.append(Token(TokenType.UPDATE_ALT, None, start))
+            else:
+                tokens.append(Token(TokenType.ALT, None, start))
+            continue
+
+        # += (update add)
+        if c == "+" and peek() == "=":
+            advance()
+            tokens.append(Token(TokenType.UPDATE_ADD, None, start))
+            continue
+
+        # -= (update subtract)
+        if c == "-" and peek() == "=":
+            advance()
+            tokens.append(Token(TokenType.UPDATE_SUB, None, start))
+            continue
+
+        # *= (update multiply)
+        if c == "*" and peek() == "=":
+            advance()
+            tokens.append(Token(TokenType.UPDATE_MUL, None, start))
+            continue
+
+        # /= (update divide) - only if not //
+        if c == "/" and peek() == "=" and not (pos > 1 and input_str[pos - 2] == "/"):
+            advance()
+            tokens.append(Token(TokenType.UPDATE_DIV, None, start))
+            continue
+
+        # %= (update modulo)
+        if c == "%" and peek() == "=":
+            advance()
+            tokens.append(Token(TokenType.UPDATE_MOD, None, start))
+            continue
+
+        # |= (update pipe)
+        if c == "|" and peek() == "=":
+            advance()
+            tokens.append(Token(TokenType.UPDATE_PIPE, None, start))
+            continue
+
+        # = (assignment) - single = that's not part of ==
+        if c == "=" and peek() != "=":
+            tokens.append(Token(TokenType.ASSIGN, None, start))
+            continue
+
+        # Single-character tokens
+        if c == ".":
+            tokens.append(Token(TokenType.DOT, None, start))
+            continue
+
+        if c == "|":
+            tokens.append(Token(TokenType.PIPE, None, start))
+            continue
+
+        if c == ",":
+            tokens.append(Token(TokenType.COMMA, None, start))
+            continue
+
+        if c == ":":
+            tokens.append(Token(TokenType.COLON, None, start))
+            continue
+
+        if c == ";":
+            tokens.append(Token(TokenType.SEMICOLON, None, start))
+            continue
+
+        if c == "(":
+            tokens.append(Token(TokenType.LPAREN, None, start))
+            continue
+
+        if c == ")":
+            tokens.append(Token(TokenType.RPAREN, None, start))
+            continue
+
+        if c == "[":
+            tokens.append(Token(TokenType.LBRACKET, None, start))
+            continue
+
+        if c == "]":
+            tokens.append(Token(TokenType.RBRACKET, None, start))
+            continue
+
+        if c == "{":
+            tokens.append(Token(TokenType.LBRACE, None, start))
+            continue
+
+        if c == "}":
+            tokens.append(Token(TokenType.RBRACE, None, start))
+            continue
+
+        if c == "?":
+            tokens.append(Token(TokenType.QUESTION, None, start))
+            continue
+
+        if c == "+":
+            tokens.append(Token(TokenType.PLUS, None, start))
+            continue
+
+        if c == "-":
+            # Could be negative number or minus operator
+            if is_digit(peek()):
+                num = c
+                while not is_eof() and (is_digit(peek()) or peek() == "."):
+                    num += advance()
+                # Handle scientific notation
+                if not is_eof() and peek() in "eE":
+                    num += advance()
+                    if not is_eof() and peek() in "+-":
+                        num += advance()
+                    while not is_eof() and is_digit(peek()):
+                        num += advance()
+                try:
+                    value = float(num) if "." in num or "e" in num or "E" in num else int(num)
+                    tokens.append(Token(TokenType.NUMBER, value, start))
+                except ValueError:
+                    tokens.append(Token(TokenType.NUMBER, float(num), start))
+                continue
+            tokens.append(Token(TokenType.MINUS, None, start))
+            continue
+
+        if c == "*":
+            tokens.append(Token(TokenType.STAR, None, start))
+            continue
+
+        if c == "/":
+            tokens.append(Token(TokenType.SLASH, None, start))
+            continue
+
+        if c == "%":
+            tokens.append(Token(TokenType.PERCENT, None, start))
+            continue
+
+        if c == "<":
+            tokens.append(Token(TokenType.LT, None, start))
+            continue
+
+        if c == ">":
+            tokens.append(Token(TokenType.GT, None, start))
+            continue
+
+        # Numbers
+        if is_digit(c):
+            num = c
+            while not is_eof() and (is_digit(peek()) or peek() == "." or peek() in "eE"):
+                if peek() in "eE":
+                    num += advance()
+                    if not is_eof() and peek() in "+-":
+                        num += advance()
+                else:
+                    num += advance()
+            try:
+                value = float(num) if "." in num or "e" in num or "E" in num else int(num)
+                tokens.append(Token(TokenType.NUMBER, value, start))
+            except ValueError:
+                tokens.append(Token(TokenType.NUMBER, float(num), start))
+            continue
+
+        # Strings
+        if c == '"':
+            s = ""
+            while not is_eof() and peek() != '"':
+                if peek() == "\\":
+                    advance()  # consume backslash
+                    if is_eof():
+                        break
+                    escaped = advance()
+                    if escaped == "n":
+                        s += "\n"
+                    elif escaped == "r":
+                        s += "\r"
+                    elif escaped == "t":
+                        s += "\t"
+                    elif escaped == "\\":
+                        s += "\\"
+                    elif escaped == '"':
+                        s += '"'
+                    elif escaped == "(":
+                        s += "\\("  # Keep for string interpolation
+                    else:
+                        s += escaped
+                else:
+                    s += advance()
+            if not is_eof():
+                advance()  # consume closing quote
+            tokens.append(Token(TokenType.STRING, s, start))
+            continue
+
+        # Identifiers and keywords (including $variables and @formats)
+        if is_alpha(c) or c == "$" or c == "@":
+            ident = c
+            while not is_eof() and is_alnum(peek()):
+                ident += advance()
+
+            # Check for keyword
+            keyword_type = KEYWORDS.get(ident)
+            if keyword_type:
+                tokens.append(Token(keyword_type, None, start))
+            else:
+                tokens.append(Token(TokenType.IDENT, ident, start))
+            continue
+
+        raise ValueError(f"Unexpected character '{c}' at position {start}")
+
+    tokens.append(Token(TokenType.EOF, None, pos))
+    return tokens

just_bash/query_engine/types.py

@@ -0,0 +1,373 @@
+"""Type definitions for the query engine.
+
+Contains TokenType enum, Token dataclass, and all AST node dataclasses.
+"""
+
+from dataclasses import dataclass, field
+from enum import Enum, auto
+from typing import Any, Union
+
+
+class TokenType(Enum):
+    """Token types for the jq expression lexer."""
+
+    # Structural
+    DOT = auto()
+    PIPE = auto()
+    COMMA = auto()
+    COLON = auto()
+    SEMICOLON = auto()
+    LPAREN = auto()
+    RPAREN = auto()
+    LBRACKET = auto()
+    RBRACKET = auto()
+    LBRACE = auto()
+    RBRACE = auto()
+    QUESTION = auto()
+
+    # Arithmetic
+    PLUS = auto()
+    MINUS = auto()
+    STAR = auto()
+    SLASH = auto()
+    PERCENT = auto()
+
+    # Comparison
+    EQ = auto()  # ==
+    NE = auto()  # !=
+    LT = auto()  # <
+    LE = auto()  # <=
+    GT = auto()  # >
+    GE = auto()  # >=
+
+    # Logical
+    AND = auto()  # and
+    OR = auto()  # or
+    NOT = auto()  # not
+
+    # Alternative
+    ALT = auto()  # //
+
+    # Assignment
+    ASSIGN = auto()  # =
+
+    # Update operators
+    UPDATE_ADD = auto()  # +=
+    UPDATE_SUB = auto()  # -=
+    UPDATE_MUL = auto()  # *=
+    UPDATE_DIV = auto()  # /=
+    UPDATE_MOD = auto()  # %=
+    UPDATE_ALT = auto()  # //=
+    UPDATE_PIPE = auto()  # |=
+
+    # Values
+    IDENT = auto()
+    NUMBER = auto()
+    STRING = auto()
+
+    # Keywords
+    IF = auto()
+    THEN = auto()
+    ELIF = auto()
+    ELSE = auto()
+    END = auto()
+    AS = auto()
+    TRY = auto()
+    CATCH = auto()
+    TRUE = auto()
+    FALSE = auto()
+    NULL = auto()
+    REDUCE = auto()
+    FOREACH = auto()
+
+    # Special
+    DOTDOT = auto()  # ..
+    EOF = auto()
+
+
+@dataclass
+class Token:
+    """A lexer token."""
+
+    type: TokenType
+    value: str | int | float | None = None
+    pos: int = 0
+
+
+@dataclass
+class QueryExecutionLimits:
+    """Execution limits to prevent runaway queries."""
+
+    max_iterations: int = 10000
+
+
+@dataclass
+class EvalContext:
+    """Evaluation context with variables, limits, and environment."""
+
+    vars: dict[str, Any] = field(default_factory=dict)
+    limits: QueryExecutionLimits = field(default_factory=QueryExecutionLimits)
+    env: dict[str, str] = field(default_factory=dict)
+    root: Any = None
+    current_path: list[str | int] = field(default_factory=list)
+
+
+# AST Node Types
+@dataclass
+class IdentityNode:
+    """The identity filter (.)"""
+
+    type: str = field(default="Identity", init=False)
+
+
+@dataclass
+class FieldNode:
+    """Field access (.name)"""
+
+    name: str
+    base: "AstNode | None" = None
+    type: str = field(default="Field", init=False)
+
+
+@dataclass
+class IndexNode:
+    """Array/object index access (.[n] or .["key"])"""
+
+    index: "AstNode"
+    base: "AstNode | None" = None
+    type: str = field(default="Index", init=False)
+
+
+@dataclass
+class SliceNode:
+    """Array slice (.[start:end])"""
+
+    start: "AstNode | None" = None
+    end: "AstNode | None" = None
+    base: "AstNode | None" = None
+    type: str = field(default="Slice", init=False)
+
+
+@dataclass
+class IterateNode:
+    """Iterator (.[] or .[])"""
+
+    base: "AstNode | None" = None
+    type: str = field(default="Iterate", init=False)
+
+
+@dataclass
+class PipeNode:
+    """Pipe operator (|)"""
+
+    left: "AstNode"
+    right: "AstNode"
+    type: str = field(default="Pipe", init=False)
+
+
+@dataclass
+class CommaNode:
+    """Comma operator for multiple outputs"""
+
+    left: "AstNode"
+    right: "AstNode"
+    type: str = field(default="Comma", init=False)
+
+
+@dataclass
+class LiteralNode:
+    """Literal value (number, string, true, false, null)"""
+
+    value: Any
+    type: str = field(default="Literal", init=False)
+
+
+@dataclass
+class ArrayNode:
+    """Array construction ([...])"""
+
+    elements: "AstNode | None" = None
+    type: str = field(default="Array", init=False)
+
+
+@dataclass
+class ObjectEntry:
+    """A single object entry with key and value."""
+
+    key: "AstNode | str"
+    value: "AstNode"
+
+
+@dataclass
+class ObjectNode:
+    """Object construction ({...})"""
+
+    entries: list[ObjectEntry] = field(default_factory=list)
+    type: str = field(default="Object", init=False)
+
+
+@dataclass
+class ParenNode:
+    """Parenthesized expression"""
+
+    expr: "AstNode"
+    type: str = field(default="Paren", init=False)
+
+
+@dataclass
+class BinaryOpNode:
+    """Binary operation (+, -, *, /, %, ==, !=, <, <=, >, >=, and, or, //)"""
+
+    op: str  # "+", "-", "*", "/", "%", "==", "!=", "<", "<=", ">", ">=", "and", "or", "//"
+    left: "AstNode"
+    right: "AstNode"
+    type: str = field(default="BinaryOp", init=False)
+
+
+@dataclass
+class UnaryOpNode:
+    """Unary operation (- or not)"""
+
+    op: str  # "-" or "not"
+    operand: "AstNode"
+    type: str = field(default="UnaryOp", init=False)
+
+
+@dataclass
+class ElifBranch:
+    """An elif branch in a conditional."""
+
+    cond: "AstNode"
+    then: "AstNode"
+
+
+@dataclass
+class CondNode:
+    """Conditional (if-then-elif-else-end)"""
+
+    cond: "AstNode"
+    then: "AstNode"
+    elifs: list[ElifBranch] = field(default_factory=list)
+    else_: "AstNode | None" = None
+    type: str = field(default="Cond", init=False)
+
+
+@dataclass
+class TryNode:
+    """Try-catch expression"""
+
+    body: "AstNode"
+    catch: "AstNode | None" = None
+    type: str = field(default="Try", init=False)
+
+
+@dataclass
+class CallNode:
+    """Function call"""
+
+    name: str
+    args: list["AstNode"] = field(default_factory=list)
+    type: str = field(default="Call", init=False)
+
+
+@dataclass
+class VarBindNode:
+    """Variable binding (expr as $var | body)"""
+
+    name: str
+    value: "AstNode"
+    body: "AstNode"
+    type: str = field(default="VarBind", init=False)
+
+
+@dataclass
+class VarRefNode:
+    """Variable reference ($var)"""
+
+    name: str
+    type: str = field(default="VarRef", init=False)
+
+
+@dataclass
+class RecurseNode:
+    """Recursive descent (..)"""
+
+    type: str = field(default="Recurse", init=False)
+
+
+@dataclass
+class OptionalNode:
+    """Optional operator (?)"""
+
+    expr: "AstNode"
+    type: str = field(default="Optional", init=False)
+
+
+@dataclass
+class StringInterpNode:
+    """String with interpolation"""
+
+    parts: list["str | AstNode"] = field(default_factory=list)
+    type: str = field(default="StringInterp", init=False)
+
+
+@dataclass
+class UpdateOpNode:
+    """Update operation (=, |=, +=, -=, *=, /=, %=, //=)"""
+
+    op: str  # "=", "|=", "+=", "-=", "*=", "/=", "%=", "//="
+    path: "AstNode"
+    value: "AstNode"
+    type: str = field(default="UpdateOp", init=False)
+
+
+@dataclass
+class ReduceNode:
+    """Reduce expression"""
+
+    expr: "AstNode"
+    var_name: str
+    init: "AstNode"
+    update: "AstNode"
+    type: str = field(default="Reduce", init=False)
+
+
+@dataclass
+class ForeachNode:
+    """Foreach expression"""
+
+    expr: "AstNode"
+    var_name: str
+    init: "AstNode"
+    update: "AstNode"
+    extract: "AstNode | None" = None
+    type: str = field(default="Foreach", init=False)
+
+
+# Union type for all AST nodes
+AstNode = Union[
+    IdentityNode,
+    FieldNode,
+    IndexNode,
+    SliceNode,
+    IterateNode,
+    PipeNode,
+    CommaNode,
+    LiteralNode,
+    ArrayNode,
+    ObjectNode,
+    ParenNode,
+    BinaryOpNode,
+    UnaryOpNode,
+    CondNode,
+    TryNode,
+    CallNode,
+    VarBindNode,
+    VarRefNode,
+    RecurseNode,
+    OptionalNode,
+    StringInterpNode,
+    UpdateOpNode,
+    ReduceNode,
+    ForeachNode,
+]