tellaro-query-language 0.2.0__py3-none-any.whl → 0.2.2__py3-none-any.whl

tql/parser.py

@@ -150,6 +150,17 @@ class TQLParser:
                             if field_mutators:
                                 result["field_mutators"] = field_mutators
                             return result
+                        else:
+                            # This is field | mutator without operator (e.g., field | lowercase)
+                            # Treat as field exists with mutator for output transformation
+                            result = {
+                                "type": "comparison",
+                                "field": field_name,
+                                "type_hint": type_hint,
+                                "operator": "exists",
+                                "field_mutators": field_mutators,
+                            }
+                            return result
                 # Single item, unwrap it
                 return self._build_ast(parsed[0])
             elif len(parsed) >= 2 and isinstance(parsed[0], str) and parsed[0].lower() == "stats":
@@ -196,31 +207,45 @@ class TQLParser:
 
                     return result
 
-                # Check for is_private/is_global without operator (defaults to eq true)
-                # This happens when we have a field with is_private/is_global as the last mutator
-                elif isinstance(first, str) and isinstance(second, list) and len(second) == 1:
+                # Check for NOT operator first (before field | mutator check)
+                elif isinstance(first, str) and (first.lower() == "not" or first == "!"):
+                    # Unary logical operator (NOT or !)
+                    return {"type": "unary_op", "operator": "not", "operand": self._build_ast(second)}
+
+                # Check for field | mutator without operator
+                # This happens when we have a field with mutator(s) as the last element
+                elif isinstance(first, str) and isinstance(second, list):
                     # This could be field | mutator structure
-                    mutator_name = second[0] if isinstance(second[0], str) else None
-                    if mutator_name and mutator_name.lower() in ["is_private", "is_global"]:
+                    # Check if second is a mutator structure (either ['mutator'] or ['mutator', [...params...]])
+                    if len(second) >= 1 and isinstance(second[0], str):
+                        mutator_name = second[0]
                         # Build a typed_field from these components
                         typed_field = [first, second]
                         field_name, type_hint, field_mutators = self.ast_builder.extract_field_info(typed_field)
-                        # This is field | is_private or field | is_global without operator
-                        # Default to eq true
-                        result = {
-                            "type": "comparison",
-                            "field": field_name,
-                            "type_hint": type_hint,
-                            "operator": "eq",
-                            "value": "true",
-                        }
+
+                        if mutator_name.lower() in ["is_private", "is_global"]:
+                            # This is field | is_private or field | is_global without operator
+                            # Default to eq true
+                            result = {
+                                "type": "comparison",
+                                "field": field_name,
+                                "type_hint": type_hint,
+                                "operator": "eq",
+                                "value": "true",
+                            }
+                        else:
+                            # This is field | mutator without operator (e.g., field | lowercase)
+                            # Treat as field exists with mutator for output transformation
+                            result = {
+                                "type": "comparison",
+                                "field": field_name,
+                                "type_hint": type_hint,
+                                "operator": "exists",
+                            }
+
                         if field_mutators:
                             result["field_mutators"] = field_mutators
                         return result
-
-                elif isinstance(first, str) and (first.lower() == "not" or first == "!"):
-                    # Unary logical operator (NOT or !)
-                    return {"type": "unary_op", "operator": "not", "operand": self._build_ast(second)}
                 elif isinstance(second, str) and (second.lower() == "exists" or second.lower() == "!exists"):
                     # Unary comparison operation (field exists or !exists)
                     field_name, type_hint, mutators = self.ast_builder.extract_field_info(first)
@@ -249,25 +274,37 @@ class TQLParser:
                     # Fallback to treating as unary logical operator
                     return {"type": "unary_op", "operator": first.lower(), "operand": self._build_ast(second)}
             elif len(parsed) >= 3:
-                # Check if this is a field with multiple mutators ending in is_private/is_global
-                if isinstance(parsed[0], str) and all(isinstance(item, list) and len(item) == 1 for item in parsed[1:]):
+                # Check if this is a field with multiple mutators
+                if isinstance(parsed[0], str) and all(
+                    isinstance(item, list) and len(item) >= 1 and isinstance(item[0], str) for item in parsed[1:]
+                ):
                     # This looks like field | mutator1 | mutator2 | ...
                     last_mutator_list = parsed[-1]
-                    if (
-                        len(last_mutator_list) == 1
-                        and isinstance(last_mutator_list[0], str)
-                        and last_mutator_list[0].lower() in ["is_private", "is_global"]
-                    ):
-                        # This is a field with mutators ending in is_private/is_global
-                        # Build the typed_field structure and default to eq true
+                    if len(last_mutator_list) >= 1 and isinstance(last_mutator_list[0], str):
+                        # This is a field with mutators
+                        # Build the typed_field structure
                         field_name, type_hint, field_mutators = self.ast_builder.extract_field_info(parsed)
-                        result = {
-                            "type": "comparison",
-                            "field": field_name,
-                            "type_hint": type_hint,
-                            "operator": "eq",
-                            "value": "true",
-                        }
+
+                        # Check if last mutator is is_private/is_global
+                        last_mutator_name = last_mutator_list[0].lower()
+                        if last_mutator_name in ["is_private", "is_global"]:
+                            # Default to eq true for these special mutators
+                            result = {
+                                "type": "comparison",
+                                "field": field_name,
+                                "type_hint": type_hint,
+                                "operator": "eq",
+                                "value": "true",
+                            }
+                        else:
+                            # For other mutators, treat as field exists
+                            result = {
+                                "type": "comparison",
+                                "field": field_name,
+                                "type_hint": type_hint,
+                                "operator": "exists",
+                            }
+
                         if field_mutators:
                             result["field_mutators"] = field_mutators
                         return result
@@ -1370,10 +1407,28 @@ class TQLParser:
             else:
                 i += 1
 
-        # Process group by fields
+        # Process group by fields and visualization hint
         while i < len(parsed):
-            if isinstance(parsed[i], str) and parsed[i] not in ["by", ","]:
-                result["group_by"].append(parsed[i])
+            if isinstance(parsed[i], str):
+                if parsed[i] == "=>":
+                    # Visualization hint found
+                    i += 1
+                    if i < len(parsed) and isinstance(parsed[i], str):
+                        result["viz_hint"] = parsed[i].lower()
+                    break
+                elif parsed[i] not in ["by", ","]:
+                    # This is a simple field name without bucket size - normalize to dict format
+                    result["group_by"].append({"field": parsed[i], "bucket_size": None})
+            elif isinstance(parsed[i], list):
+                # This is a group by field with optional bucket size
+                if len(parsed[i]) >= 1:
+                    # Check for "top N" specification
+                    if len(parsed[i]) >= 3 and parsed[i][1].lower() == "top":
+                        field_spec = {"field": parsed[i][0], "bucket_size": int(parsed[i][2])}
+                        result["group_by"].append(field_spec)
+                    else:
+                        # No bucket size specified - normalize to dict format
+                        result["group_by"].append({"field": parsed[i][0], "bucket_size": None})
             i += 1
 
         return result

tql/parser_components/ast_builder.py

@@ -6,7 +6,7 @@ from typing import Any, Dict, List, Tuple, Union
 class ASTBuilder:
     """Builds Abstract Syntax Tree from parsed TQL expressions."""
 
-    def extract_field_info(self, field_spec: Any) -> Tuple[str, Union[str, None], List[Dict[str, Any]]]:
+    def extract_field_info(self, field_spec: Any) -> Tuple[str, Union[str, None], List[Dict[str, Any]]]:  # noqa: C901
         """Extract field name, optional type hint, and mutators from field specification.
 
         Args:
@@ -48,7 +48,25 @@ class ASTBuilder:
                             params = []
                             for param in item[1]:
                                 if isinstance(param, list) and len(param) == 2:
+                                    # Named parameter: [name, value]
                                     params.append(param)
+                                elif isinstance(param, str):
+                                    # Positional parameter - handle based on mutator type
+                                    mutator_name = item[0].lower()
+                                    if mutator_name == "split":
+                                        params.append(["delimiter", param])
+                                    elif mutator_name == "replace":
+                                        # For replace, first positional is 'find', second is 'replace'
+                                        if not params or all(p[0] != "find" for p in params):
+                                            params.append(["find", param])
+                                        elif all(p[0] != "replace" for p in params):
+                                            params.append(["replace", param])
+                                        else:
+                                            # Too many positional params
+                                            params.append(["_positional", param])
+                                    else:
+                                        # For other mutators, use first positional as unnamed param
+                                        params.append(["_positional", param])
                             if params:
                                 mutator_dict["params"] = params
                         mutators.append(mutator_dict)
@@ -113,7 +131,25 @@ class ASTBuilder:
                                     params = []
                                     for param in item[1]:
                                         if isinstance(param, list) and len(param) == 2:
+                                            # Named parameter: [name, value]
                                             params.append(param)
+                                        elif isinstance(param, str):
+                                            # Positional parameter - handle based on mutator type
+                                            mutator_name = item[0].lower()
+                                            if mutator_name == "split":
+                                                params.append(["delimiter", param])
+                                            elif mutator_name == "replace":
+                                                # For replace, first positional is 'find', second is 'replace'
+                                                if not params or all(p[0] != "find" for p in params):
+                                                    params.append(["find", param])
+                                                elif all(p[0] != "replace" for p in params):
+                                                    params.append(["replace", param])
+                                                else:
+                                                    # Too many positional params
+                                                    params.append(["_positional", param])
+                                            else:
+                                                # For other mutators, use first positional as unnamed param
+                                                params.append(["_positional", param])
                                     if params:
                                         mutator_dict["params"] = params
                                 mutators.append(mutator_dict)

tql/parser_components/field_extractor.py

@@ -109,4 +109,12 @@ class FieldExtractor:
         # Collect fields from group by
         if "group_by" in stats_node:
             for field in stats_node["group_by"]:
-                fields.add(field)
+                if isinstance(field, dict) and "field" in field:
+                    # Normalized format: {"field": "name", "bucket_size": N|None}
+                    fields.add(field["field"])
+                elif isinstance(field, str):
+                    # Legacy format: just field name (for backward compatibility)
+                    fields.add(field)
+                else:
+                    # Handle any other format gracefully
+                    fields.add(str(field))

tql/parser_components/grammar.py

@@ -4,6 +4,7 @@ from pyparsing import (
     CaselessKeyword,
     Forward,
     Group,
+    Literal,
 )
 from pyparsing import Optional as PyparsingOptional
 from pyparsing import (
@@ -122,12 +123,20 @@ class TQLGrammar:
         """Set up mutator definitions."""
         # Define mutators
         self.mutator_name = oneOf(
-            "lowercase uppercase trim split nslookup geoip_lookup geo "
+            "lowercase uppercase trim split replace nslookup geoip_lookup geo "
             "length refang defang b64encode b64decode urldecode "
-            "any all avg average max min sum is_private is_global",
+            "any all none avg average max min sum is_private is_global "
+            "count unique first last",
             caseless=True,
         )
-        self.mutator_param = Group(self.identifier + Suppress("=") + (self.string_literal | self.list_literal))
+        # Mutator parameters can be either named (key=value) or positional (just value)
+        # Named parameters: key=value where value can be string literal, list, identifier, or number
+        self.mutator_named_param = Group(
+            self.identifier + Suppress("=") + (self.string_literal | self.list_literal | self.identifier | self.number)
+        )
+        # Positional parameters can be strings (quoted or unquoted), numbers, or identifiers
+        self.mutator_positional_param = self.string_literal | self.number | self.identifier
+        self.mutator_param = self.mutator_named_param | self.mutator_positional_param
         self.mutator_params = Group(Suppress("(") + delimitedList(self.mutator_param) + Suppress(")"))
         self.mutator = Group(Suppress("|") + self.mutator_name + PyparsingOptional(self.mutator_params))
         self.mutator_chain = ZeroOrMore(self.mutator)
@@ -406,17 +415,32 @@ class TQLGrammar:
         # Multiple aggregations separated by commas
         self.agg_list = delimitedList(self.agg_with_alias)
 
-        # Group by fields
-        self.group_by_fields = delimitedList(self.field_name)
+        # Group by fields with optional "top N" for each field
+        self.top_kw = CaselessKeyword("top")
+        self.group_by_field_with_bucket = Group(self.field_name + PyparsingOptional(self.top_kw + self.number))
+        self.group_by_fields = delimitedList(self.group_by_field_with_bucket)
+
+        # Visualization hint: => chart_type
+        self.viz_arrow = Literal("=>")
+        self.viz_types = oneOf(
+            "bar barchart line area pie donut scatter heatmap treemap sunburst "
+            "table number gauge map grouped_bar stacked_bar nested_pie nested_donut chord",
+            caseless=True,
+        )
+        self.viz_hint = PyparsingOptional(self.viz_arrow + self.viz_types)
 
-        # Complete stats expression: | stats agg_functions [by group_fields]
+        # Complete stats expression: | stats agg_functions [by group_fields] [=> viz_type]
         self.stats_expr_with_pipe = Group(
-            Suppress("|") + self.stats_kw + self.agg_list + PyparsingOptional(self.by_kw + self.group_by_fields)
+            Suppress("|")
+            + self.stats_kw
+            + self.agg_list
+            + PyparsingOptional(self.by_kw + self.group_by_fields)
+            + self.viz_hint
         )
 
         # Stats expression without pipe (for standalone use)
         self.stats_expr_no_pipe = Group(
-            self.stats_kw + self.agg_list + PyparsingOptional(self.by_kw + self.group_by_fields)
+            self.stats_kw + self.agg_list + PyparsingOptional(self.by_kw + self.group_by_fields) + self.viz_hint
         )
 
         # Combined stats expression (with or without pipe)