additory 0.1.0a3__py3-none-any.whl → 0.1.1a1__py3-none-any.whl

additory/expressions/parser.py

@@ -1,176 +1,372 @@
-# parser.py
+"""
+Expression parser for Additory.
 
-from dataclasses import dataclass
-from typing import Dict, Optional, List, Any
-
-import yaml
-
-from .logging import log_info, log_warning
-from .ast_builder import build_ast_from_expression   # <-- NEW: your AST builder
+Parses expression strings into Abstract Syntax Trees (AST) for compilation.
+"""
 
+import re
+from dataclasses import dataclass
+from typing import List, Optional, Union
 
-# ------------------------------------------------------------
-# Parsed Expression Structure
-# ------------------------------------------------------------
 
 @dataclass
-class ParsedExpression:
-    name: str
-    metadata: Dict[str, Any]
-    expression: str
-    raw_text: str
-    ast: Optional[Dict[str, Any]] = None          # <-- NEW
-    sample_clean: Optional[Dict[str, List[Any]]] = None
-    sample_unclean: Optional[Dict[str, List[Any]]] = None
-
-
-# ------------------------------------------------------------
-# Public API
-# ------------------------------------------------------------
-
-def parse_expression(text: str) -> ParsedExpression:
+class Token:
     """
-    Parses a .add expression file.
-    Supports two formats:
-      1. YAML-style (new)
-      2. Legacy metadata + expression block (old)
+    Token from expression string.
+    
+    Attributes:
+        type: Token type
+        value: Token value
+        position: Position in original string
     """
+    type: str
+    value: Union[str, int, float]
+    position: int
+    
+    def __repr__(self) -> str:
+        return f"Token({self.type}, {self.value!r}, pos={self.position})"
 
-    if not text.strip():
-        log_warning("[parser] Empty expression file")
-        return ParsedExpression(
-            name="unknown",
-            metadata={},
-            expression="",
-            raw_text=text,
-            ast=None,
-        )
-
-    # Detect YAML-style format
-    if _looks_like_yaml(text):
-        parsed = _parse_yaml_style(text)
-    else:
-        parsed = _parse_legacy_style(text)
-
-    # --------------------------------------------------------
-    # NEW: Build AST from parsed.expression
-    # --------------------------------------------------------
-    try:
-        parsed.ast = build_ast_from_expression(parsed.expression)
-    except Exception as e:
-        log_warning(f"[parser] Failed to build AST: {e}")
-        parsed.ast = None
-
-    return parsed
-
-
-# ------------------------------------------------------------
-# YAML-STYLE PARSER
-# ------------------------------------------------------------
-
-def _looks_like_yaml(text: str) -> bool:
-    lowered = text.lower()
-    return ("formula:" in lowered) or ("sample:" in lowered)
-
-
-def _parse_yaml_style(text: str) -> ParsedExpression:
-    try:
-        parsed = yaml.safe_load(text)
-    except Exception as e:
-        log_warning(f"[parser] YAML parse failed, falling back to legacy: {e}")
-        return _parse_legacy_style(text)
-
-    formula = parsed.get("formula", {})
-    sample = parsed.get("sample", {})
-    expression_block = formula.get("expression")
-
-    if not expression_block:
-        log_warning("[parser] YAML file missing 'formula.expression' block")
-        expression_block = ""
-
-    name = formula.get("name", "unknown")
-
-    return ParsedExpression(
-        name=name,
-        metadata=formula,
-        expression=_normalize_expression(expression_block),
-        raw_text=text,
-        sample_clean=sample.get("clean"),
-        sample_unclean=sample.get("unclean"),
-    )
-
-
-def _normalize_expression(expr):
-    if isinstance(expr, list):
-        return "\n".join(expr)
-    return str(expr).strip()
-
-
-# ------------------------------------------------------------
-# LEGACY PARSER
-# ------------------------------------------------------------
 
-def _parse_legacy_style(text: str) -> ParsedExpression:
-    lines = _preprocess(text)
-    metadata = _parse_metadata(lines)
-    expression = _parse_expression_block(lines)
-
-    name = metadata.get("name", "unknown")
-
-    return ParsedExpression(
-        name=name,
-        metadata=metadata,
-        expression=expression,
-        raw_text=text,
-    )
-
-
-# ------------------------------------------------------------
-# Internal Helpers
-# ------------------------------------------------------------
-
-def _preprocess(text: str) -> List[str]:
-    cleaned = []
-    for line in text.splitlines():
-        stripped = line.strip()
-        if not stripped:
-            continue
-        if stripped.startswith("#"):
-            continue
-        cleaned.append(stripped)
-    return cleaned
-
-
-def _parse_metadata(lines: List[str]) -> Dict[str, str]:
-    metadata = {}
-
-    for line in lines:
-        if line.lower().startswith("expression:"):
-            break
-
-        if ":" not in line:
-            log_warning(f"[parser] Invalid metadata line: {line}")
-            continue
-
-        key, value = line.split(":", 1)
-        metadata[key.strip()] = value.strip()
-
-    return metadata
-
-
-def _parse_expression_block(lines: List[str]) -> str:
-    expr_lines = []
-    in_expr = False
-
-    for line in lines:
-        if line.lower().startswith("expression:"):
-            in_expr = True
-            continue
-
-        if in_expr:
-            expr_lines.append(line)
-
-    if not expr_lines:
-        log_warning("[parser] No expression block found")
-
-    return "\n".join(expr_lines).strip()
+@dataclass
+class ASTNode:
+    """
+    Node in Abstract Syntax Tree.
+    
+    Attributes:
+        type: Node type ('binary_op', 'unary_op', 'function', 'identifier', 'literal')
+        value: Node value (operator, function name, column name, or literal value)
+        left: Left child node (for binary operators)
+        right: Right child node (for binary operators)
+        children: Child nodes (for functions)
+    """
+    type: str
+    value: Union[str, int, float, None]
+    left: Optional['ASTNode'] = None
+    right: Optional['ASTNode'] = None
+    children: Optional[List['ASTNode']] = None
+    
+    def __repr__(self) -> str:
+        if self.type == 'binary_op':
+            return f"BinaryOp({self.value}, {self.left}, {self.right})"
+        elif self.type == 'unary_op':
+            return f"UnaryOp({self.value}, {self.right})"
+        elif self.type == 'function':
+            return f"Function({self.value}, {self.children})"
+        elif self.type == 'identifier':
+            return f"Identifier({self.value})"
+        elif self.type == 'literal':
+            return f"Literal({self.value})"
+        else:
+            return f"ASTNode({self.type}, {self.value})"
+
+
+class ExpressionParser:
+    """
+    Parse expression strings into Abstract Syntax Tree (AST).
+    
+    Supports:
+    - Arithmetic operators: +, -, *, /, **, %
+    - Comparison operators: ==, !=, >, <, >=, <=
+    - Logical operators: AND, OR, NOT
+    - Functions: sqrt, abs, log, if_else, etc.
+    - Parentheses for grouping
+    """
+    
+    # Supported functions (case-insensitive)
+    FUNCTIONS = {
+        # Mathematical
+        'sqrt', 'abs', 'log', 'log10', 'exp', 'pow', 'round', 'floor', 'ceil',
+        # String
+        'lower', 'upper', 'trim', 'length', 'substring', 'replace', 'contains', 'matches',
+        # Date/Time
+        'year', 'month', 'day', 'hour', 'minute', 'second', 'day_of_week', 'time_of_day',
+        # Aggregation
+        'sum', 'mean', 'median', 'min', 'max', 'count', 'std',
+        # Conditional
+        'if_else', 'coalesce', 'is_null', 'is_not_null'
+    }
+    
+    # Operator precedence (higher = higher precedence)
+    PRECEDENCE = {
+        'OR': 1,
+        'AND': 2,
+        '==': 3, '!=': 3, '>': 3, '<': 3, '>=': 3, '<=': 3,
+        '+': 4, '-': 4,
+        '*': 5, '/': 5, '%': 5,
+        '**': 6,
+        'UNARY': 7  # Unary minus and NOT
+    }
+    
+    def __init__(self):
+        self.tokens = []
+        self.position = 0
+    
+    def parse(self, expression: str) -> ASTNode:
+        """
+        Parse expression string into AST.
+        
+        Args:
+            expression: Expression string to parse
+            
+        Returns:
+            Root AST node
+            
+        Example:
+            parser = ExpressionParser()
+            ast = parser.parse('weight / (height ** 2)')
+        """
+        # Tokenize
+        self.tokens = self.tokenize(expression)
+        self.position = 0
+        
+        # Build AST
+        if not self.tokens:
+            raise ValueError("Empty expression")
+        
+        ast = self.build_ast(self.tokens)
+        
+        # Check for unconsumed tokens
+        if self.position < len(self.tokens):
+            token = self.tokens[self.position]
+            raise ValueError(
+                f"Unexpected token '{token.value}' at position {token.position}"
+            )
+        
+        return ast
+    
+    def tokenize(self, expression: str) -> List[Token]:
+        """
+        Tokenize expression string.
+        
+        Args:
+            expression: Expression string
+            
+        Returns:
+            List of tokens
+        """
+        tokens = []
+        i = 0
+        
+        while i < len(expression):
+            # Skip whitespace
+            if expression[i].isspace():
+                i += 1
+                continue
+            
+            # Numbers (integers and floats)
+            if expression[i].isdigit() or (expression[i] == '.' and i + 1 < len(expression) and expression[i + 1].isdigit()):
+                start = i
+                has_dot = False
+                while i < len(expression) and (expression[i].isdigit() or (expression[i] == '.' and not has_dot)):
+                    if expression[i] == '.':
+                        has_dot = True
+                    i += 1
+                value_str = expression[start:i]
+                value = float(value_str) if has_dot else int(value_str)
+                tokens.append(Token('NUMBER', value, start))
+                continue
+            
+            # String literals (single or double quotes)
+            if expression[i] in ('"', "'"):
+                quote = expression[i]
+                start = i
+                i += 1
+                string_value = ''
+                while i < len(expression) and expression[i] != quote:
+                    if expression[i] == '\\' and i + 1 < len(expression):
+                        # Handle escape sequences
+                        i += 1
+                        if expression[i] == 'n':
+                            string_value += '\n'
+                        elif expression[i] == 't':
+                            string_value += '\t'
+                        elif expression[i] in ('"', "'", '\\'):
+                            string_value += expression[i]
+                        else:
+                            string_value += expression[i]
+                    else:
+                        string_value += expression[i]
+                    i += 1
+                
+                if i >= len(expression):
+                    raise ValueError(f"Unterminated string at position {start}")
+                
+                i += 1  # Skip closing quote
+                tokens.append(Token('STRING', string_value, start))
+                continue
+            
+            # Identifiers and keywords
+            if expression[i].isalpha() or expression[i] == '_':
+                start = i
+                while i < len(expression) and (expression[i].isalnum() or expression[i] == '_'):
+                    i += 1
+                value = expression[start:i]
+                
+                # Check if it's a keyword
+                value_upper = value.upper()
+                if value_upper in ('AND', 'OR', 'NOT'):
+                    tokens.append(Token('OPERATOR', value_upper, start))
+                elif value.lower() in self.FUNCTIONS:
+                    tokens.append(Token('FUNCTION', value.lower(), start))
+                else:
+                    tokens.append(Token('IDENTIFIER', value, start))
+                continue
+            
+            # Two-character operators
+            if i + 1 < len(expression):
+                two_char = expression[i:i+2]
+                if two_char in ('**', '==', '!=', '>=', '<='):
+                    tokens.append(Token('OPERATOR', two_char, i))
+                    i += 2
+                    continue
+            
+            # Single-character operators and punctuation
+            if expression[i] in '+-*/%><!':
+                tokens.append(Token('OPERATOR', expression[i], i))
+                i += 1
+                continue
+            
+            if expression[i] == '(':
+                tokens.append(Token('LPAREN', '(', i))
+                i += 1
+                continue
+            
+            if expression[i] == ')':
+                tokens.append(Token('RPAREN', ')', i))
+                i += 1
+                continue
+            
+            if expression[i] == ',':
+                tokens.append(Token('COMMA', ',', i))
+                i += 1
+                continue
+            
+            # Unknown character
+            raise ValueError(f"Invalid character '{expression[i]}' at position {i}")
+        
+        return tokens
+    
+    def build_ast(self, tokens: List[Token]) -> ASTNode:
+        """
+        Build AST from tokens using recursive descent parsing.
+        
+        Args:
+            tokens: List of tokens
+            
+        Returns:
+            Root AST node
+        """
+        return self._parse_expression()
+    
+    def _parse_expression(self, min_precedence: int = 0) -> ASTNode:
+        """Parse expression with operator precedence."""
+        # Parse left side (primary expression)
+        left = self._parse_primary()
+        
+        # Parse operators with precedence
+        while self.position < len(self.tokens):
+            token = self.tokens[self.position]
+            
+            # Check if it's a binary operator
+            if token.type != 'OPERATOR' or token.value in ('NOT',):
+                break
+            
+            # Get operator precedence
+            precedence = self.PRECEDENCE.get(token.value, 0)
+            if precedence < min_precedence:
+                break
+            
+            # Consume operator
+            operator = token.value
+            self.position += 1
+            
+            # Parse right side with higher precedence for left-associative operators
+            # For right-associative operators like **, use same precedence
+            next_min_precedence = precedence + (1 if operator != '**' else 0)
+            right = self._parse_expression(next_min_precedence)
+            
+            # Create binary operation node
+            left = ASTNode('binary_op', operator, left=left, right=right)
+        
+        return left
+    
+    def _parse_primary(self) -> ASTNode:
+        """Parse primary expression (literals, identifiers, functions, parentheses, unary ops)."""
+        if self.position >= len(self.tokens):
+            raise ValueError("Unexpected end of expression")
+        
+        token = self.tokens[self.position]
+        
+        # Unary operators (-, NOT)
+        if token.type == 'OPERATOR' and token.value in ('-', 'NOT'):
+            self.position += 1
+            operand = self._parse_primary()
+            return ASTNode('unary_op', token.value, right=operand)
+        
+        # Parentheses
+        if token.type == 'LPAREN':
+            self.position += 1
+            expr = self._parse_expression()
+            
+            if self.position >= len(self.tokens) or self.tokens[self.position].type != 'RPAREN':
+                raise ValueError(f"Unmatched parenthesis at position {token.position}")
+            
+            self.position += 1
+            return expr
+        
+        # Numbers
+        if token.type == 'NUMBER':
+            self.position += 1
+            return ASTNode('literal', token.value)
+        
+        # Strings
+        if token.type == 'STRING':
+            self.position += 1
+            return ASTNode('literal', token.value)
+        
+        # Functions
+        if token.type == 'FUNCTION':
+            return self._parse_function()
+        
+        # Identifiers (column names)
+        if token.type == 'IDENTIFIER':
+            self.position += 1
+            return ASTNode('identifier', token.value)
+        
+        raise ValueError(f"Unexpected token '{token.value}' at position {token.position}")
+    
+    def _parse_function(self) -> ASTNode:
+        """Parse function call."""
+        func_token = self.tokens[self.position]
+        func_name = func_token.value
+        self.position += 1
+        
+        # Expect opening parenthesis
+        if self.position >= len(self.tokens) or self.tokens[self.position].type != 'LPAREN':
+            raise ValueError(f"Expected '(' after function '{func_name}' at position {func_token.position}")
+        
+        self.position += 1
+        
+        # Parse arguments
+        args = []
+        
+        # Check for empty argument list
+        if self.position < len(self.tokens) and self.tokens[self.position].type == 'RPAREN':
+            self.position += 1
+            return ASTNode('function', func_name, children=args)
+        
+        # Parse first argument
+        args.append(self._parse_expression())
+        
+        # Parse remaining arguments
+        while self.position < len(self.tokens) and self.tokens[self.position].type == 'COMMA':
+            self.position += 1
+            args.append(self._parse_expression())
+        
+        # Expect closing parenthesis
+        if self.position >= len(self.tokens) or self.tokens[self.position].type != 'RPAREN':
+            raise ValueError(f"Expected ')' after function arguments at position {func_token.position}")
+        
+        self.position += 1
+        
+        return ASTNode('function', func_name, children=args)