tellaro-query-language 0.2.17__tar.gz → 0.2.19__tar.gz

{tellaro_query_language-0.2.17 → tellaro_query_language-0.2.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tellaro-query-language
-Version: 0.2.17
+Version: 0.2.19
 Summary: A flexible, human-friendly query language for searching and filtering structured data
 License: Proprietary
 License-File: LICENSE

{tellaro_query_language-0.2.17 → tellaro_query_language-0.2.19}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "tellaro-query-language"
-version = "0.2.17"
+version = "0.2.19"
 description = "A flexible, human-friendly query language for searching and filtering structured data"
 authors = ["Justin Henderson <justin@tellaro.io>"]
 license = "Proprietary"

{tellaro_query_language-0.2.17 → tellaro_query_language-0.2.19}/src/tql/evaluator_components/value_comparison.py

@@ -56,7 +56,14 @@ class ValueComparator:
                 return False  # Missing fields return False for all "is not" comparisons
             # For negated string operators, missing fields should return True
             # (e.g., if field doesn't exist, it doesn't contain/start with/end with the value)
-            elif operator in ["not_contains", "not_startswith", "not_endswith", "not_regexp"]:
+            elif operator in [
+                "not_contains",
+                "not_startswith",
+                "not_endswith",
+                "not_regexp",
+                "not_matches",
+                "not_regex",
+            ]:
                 return True
             # For not_cidr, missing fields should return False (can't check CIDR on missing IP)
             elif operator in ["cidr", "not_cidr"]:
@@ -94,18 +101,23 @@ class ValueComparator:
         if isinstance(field_value, str) and field_value.lower() in ["true", "false"]:
             field_value = field_value.lower() == "true"
 
-        # Type compatibility check for numeric operators
-        # If operator requires numeric comparison, both values must be numeric
+        # Type compatibility check for comparison operators
+        # For >, >=, <, <= operators:
+        # - Numeric comparison is preferred if both values are numeric
+        # - String comparison is allowed (supports ISO 8601 timestamps which sort correctly as strings)
+        # - Mixed types (one numeric, one string) return False
         # Exception: Arrays are handled specially in the operator logic below
         if operator in ["gt", "gte", "lt", "lte", ">", ">=", "<", "<="]:
             # Skip check if field_value is an array - handled by array logic below
             if not isinstance(field_value, (list, tuple)):
                 field_is_numeric = isinstance(field_value, (int, float)) and not isinstance(field_value, bool)
                 expected_is_numeric = isinstance(expected_value, (int, float)) and not isinstance(expected_value, bool)
+                field_is_string = isinstance(field_value, str)
+                expected_is_string = isinstance(expected_value, str)
 
-                if not (field_is_numeric and expected_is_numeric):
-                    # At least one value failed numeric conversion
-                    # Cannot perform numeric comparison - return False
+                # Allow comparison if both are numeric OR both are strings
+                if not ((field_is_numeric and expected_is_numeric) or (field_is_string and expected_is_string)):
+                    # Mixed types or unsupported types - cannot compare
                     return False
 
         try:
@@ -184,7 +196,7 @@ class ValueComparator:
                         return field_value in converted_list
                 else:
                     return field_value == expected_value
-            elif operator == "regexp":
+            elif operator in ("regexp", "matches", "regex"):
                 # Unwrap single-element lists for string operators
                 if isinstance(expected_value, list) and len(expected_value) == 1:
                     expected_value = expected_value[0]
@@ -259,7 +271,7 @@ class ValueComparator:
                     expected_value = expected_value[0]
                 # Case-insensitive comparison to match post-processor behavior
                 return not str(field_value).lower().endswith(str(expected_value).lower())
-            elif operator == "not_regexp":
+            elif operator in ("not_regexp", "not_matches", "not_regex"):
                 # Unwrap single-element lists for string operators
                 if isinstance(expected_value, list) and len(expected_value) == 1:
                     expected_value = expected_value[0]

{tellaro_query_language-0.2.17 → tellaro_query_language-0.2.19}/src/tql/mutators/encoding.py

@@ -337,26 +337,27 @@ class Md5Mutator(BaseMutator):
         append_field = self.params.get("field")
 
         # Handle different input types
+        # Note: MD5 is used here for data fingerprinting/checksums, not security
         hashed_value: Any
         if value is None:
             hashed_value = None
         elif isinstance(value, str):
-            hashed_value = hashlib.md5(value.encode("utf-8")).hexdigest()
+            hashed_value = hashlib.md5(value.encode("utf-8"), usedforsecurity=False).hexdigest()
         elif isinstance(value, bytes):
-            hashed_value = hashlib.md5(value).hexdigest()
+            hashed_value = hashlib.md5(value, usedforsecurity=False).hexdigest()
         elif isinstance(value, list):
             hashed_value = []
             for item in value:
                 if isinstance(item, str):
-                    hashed_value.append(hashlib.md5(item.encode("utf-8")).hexdigest())
+                    hashed_value.append(hashlib.md5(item.encode("utf-8"), usedforsecurity=False).hexdigest())
                 elif isinstance(item, bytes):
-                    hashed_value.append(hashlib.md5(item).hexdigest())
+                    hashed_value.append(hashlib.md5(item, usedforsecurity=False).hexdigest())
                 elif item is None:
                     hashed_value.append(None)
                 else:
-                    hashed_value.append(hashlib.md5(str(item).encode("utf-8")).hexdigest())
+                    hashed_value.append(hashlib.md5(str(item).encode("utf-8"), usedforsecurity=False).hexdigest())
         else:
-            hashed_value = hashlib.md5(str(value).encode("utf-8")).hexdigest()
+            hashed_value = hashlib.md5(str(value).encode("utf-8"), usedforsecurity=False).hexdigest()
 
         # If field is specified, add to record and return original value
         if append_field:

{tellaro_query_language-0.2.17 → tellaro_query_language-0.2.19}/src/tql/opensearch_components/field_mapping.py

@@ -231,7 +231,7 @@ class FieldMapping:
         }
 
         # Operators that work best with text fields (full-text search)
-        text_operators = {"contains", "regexp", "not_regexp"}
+        text_operators = {"contains", "regexp", "matches", "regex", "not_regexp", "not_matches", "not_regex"}
 
         # Operators that require numeric/date fields
         range_operators = {">", ">=", "<", "<=", "gt", "gte", "lt", "lte", "between", "not_between"}

{tellaro_query_language-0.2.17 → tellaro_query_language-0.2.19}/src/tql/opensearch_components/lucene_converter.py

@@ -105,7 +105,7 @@ class LuceneConverter:
                 return f"{lucene_field}:({' OR '.join(escaped_values)})"
             else:
                 return f"{lucene_field}:{escaped_value}"
-        elif operator == "regexp":
+        elif operator in ("regexp", "matches", "regex"):
             return f"{lucene_field}:/{escaped_value}/"
         elif operator == "exists":
             return f"_exists_:{lucene_field}"

{tellaro_query_language-0.2.17 → tellaro_query_language-0.2.19}/src/tql/opensearch_components/query_converter.py

@@ -315,7 +315,7 @@ class QueryConverter:
                 return {"terms": {opensearch_field: value}}
             else:
                 return {"term": {opensearch_field: value}}
-        elif operator == "regexp":
+        elif operator in ("regexp", "matches", "regex"):
             # Unwrap single-element lists for string operators
             if isinstance(value, list) and len(value) == 1:
                 value = value[0]
@@ -390,7 +390,7 @@ class QueryConverter:
             if isinstance(value, list) and len(value) == 1:
                 value = value[0]
             return {"bool": {"must_not": {"wildcard": {opensearch_field: f"*{value}"}}}}
-        elif operator == "not_regexp":
+        elif operator in ("not_regexp", "not_matches", "not_regex"):
             # Unwrap single-element lists for string operators
             if isinstance(value, list) and len(value) == 1:
                 value = value[0]

{tellaro_query_language-0.2.17 → tellaro_query_language-0.2.19}/src/tql/parser.py

@@ -14,7 +14,7 @@ from .parser_components.error_analyzer import ErrorAnalyzer
 from .parser_components.field_extractor import FieldExtractor
 from .parser_components.grammar import TQLGrammar
 
-ParserElement.enablePackrat()
+ParserElement.enable_packrat()
 
 
 class TQLParser:
@@ -53,7 +53,7 @@ class TQLParser:
 
         try:
             # Parse the query
-            parsed_result = self.grammar.tql_expr.parseString(query, parseAll=True)
+            parsed_result = self.grammar.tql_expr.parse_string(query, parse_all=True)
 
             # Convert to our AST format
             # Start depth counting at 0 from parse() entry point

{tellaro_query_language-0.2.17 → tellaro_query_language-0.2.19}/src/tql/parser_components/grammar.py

@@ -2,6 +2,7 @@
 
 from pyparsing import (
     CaselessKeyword,
+    DelimitedList,
     Forward,
     Group,
     Literal,
@@ -15,10 +16,8 @@ from pyparsing import (
     ZeroOrMore,
     alphanums,
     alphas,
-    delimitedList,
-    infixNotation,
-    nums,
-    oneOf,
+    infix_notation,
+    one_of,
     opAssoc,
 )
 
@@ -45,27 +44,54 @@ class TQLGrammar:
         """Set up basic tokens and literals."""
         # Basic tokens
         self.identifier = Word(alphas, alphanums + "_.-")
-        self.number = Word(nums + ".-")
+        # Number pattern supports:
+        # - Integers: 123, -456
+        # - Floats: 1.5, -3.14
+        # - Scientific notation: 1.0e5, 1.5e-3, 2E+10
+        # Pattern matches Rust's float grammar for parity
+        self.scientific_number = Regex(r"-?\d+\.\d+[eE][+-]?\d+")
+        self.regular_number = Regex(r"-?\d+(\.\d+)?")
+        self.number = self.scientific_number | self.regular_number
         self.string_literal = QuotedString('"') | QuotedString("'")
         # CIDR notation for IP addresses (e.g., 192.168.1.0/24)
-        self.cidr_notation = Word(nums + "./")
+        self.cidr_notation = Regex(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/\d{1,2}")
+        # IP address (without mask) - matches 4 octets separated by dots
+        self.ip_address = Regex(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}")
+        # Dot-separated numeric values (like partial IPs: 10.0.0, version numbers: 1.2.3)
+        # This allows values like "10.0.0" to be matched as a single token
+        self.dotted_number = Regex(r"\d+(\.\d+){2,}")
         # Define list items as strings, numbers, or identifiers
         self.list_item = self.string_literal | self.number | self.identifier
-        self.list_literal = Group(Suppress("[") + delimitedList(self.list_item) + Suppress("]"))
-
-        # Define simple values – note order matters (try string literals first, then CIDR)
-        self.simple_value = self.string_literal | self.cidr_notation | self.number | self.identifier
+        self.list_literal = Group(Suppress("[") + DelimitedList(self.list_item) + Suppress("]"))
+
+        # Define simple values – order matters:
+        # 1. String literals (quoted)
+        # 2. CIDR notation (IP/mask format) - must come before IP address
+        # 3. IP address (4 octets without mask)
+        # 4. Dotted numbers (partial IPs like 10.0.0, versions like 1.2.3)
+        # 5. Scientific notation (must come before regular numbers to avoid partial match)
+        # 6. Regular numbers
+        # 7. Identifiers (unquoted strings)
+        self.simple_value = (
+            self.string_literal
+            | self.cidr_notation
+            | self.ip_address
+            | self.dotted_number
+            | self.scientific_number
+            | self.regular_number
+            | self.identifier
+        )
 
         # Define type hints
-        self.type_hint = oneOf("number int float decimal date array bool boolean geo object string", caseless=True)
+        self.type_hint = one_of("number int float decimal date array bool boolean geo object string", caseless=True)
 
     def _setup_operators(self):
         """Set up operator definitions."""
         # Define binary operators (require a value) - != must come before ! operators
-        self.binary_ops = oneOf(
+        self.binary_ops = one_of(
             "!= "  # != must be before ! operators
-            + "!contains !in !startswith !endswith !regexp !cidr !is !between "
-            + "regexp in contains = eq ne > gt >= gte < lt <= lte cidr is startswith endswith any all none",
+            + "!contains !in !startswith !endswith !regexp !matches !cidr !is !between "
+            + "regexp matches regex in contains = eq ne > gt >= gte < lt <= lte cidr is startswith endswith any all none",
             caseless=True,
         )
 
@@ -75,6 +101,7 @@ class TQLGrammar:
         self.not_startswith_op = (CaselessKeyword("not") | "!") + CaselessKeyword("startswith")
         self.not_endswith_op = (CaselessKeyword("not") | "!") + CaselessKeyword("endswith")
         self.not_regexp_op = (CaselessKeyword("not") | "!") + CaselessKeyword("regexp")
+        self.not_matches_op = (CaselessKeyword("not") | "!") + CaselessKeyword("matches")
         self.not_cidr_op = (CaselessKeyword("not") | "!") + CaselessKeyword("cidr")
         self.not_any_op = (CaselessKeyword("not") | "!") + CaselessKeyword("any")
         self.not_all_op = (CaselessKeyword("not") | "!") + CaselessKeyword("all")
@@ -86,6 +113,7 @@ class TQLGrammar:
         self.bang_startswith_op = Suppress("!") + CaselessKeyword("startswith")
         self.bang_endswith_op = Suppress("!") + CaselessKeyword("endswith")
         self.bang_regexp_op = Suppress("!") + CaselessKeyword("regexp")
+        self.bang_matches_op = Suppress("!") + CaselessKeyword("matches")
         self.bang_cidr_op = Suppress("!") + CaselessKeyword("cidr")
         self.bang_any_op = Suppress("!") + CaselessKeyword("any")
         self.bang_all_op = Suppress("!") + CaselessKeyword("all")
@@ -97,7 +125,7 @@ class TQLGrammar:
         self.bang_between_op = Suppress("!") + CaselessKeyword("between")
 
         # Define unary operators (no value required)
-        self.unary_ops = oneOf("exists !exists", caseless=True)
+        self.unary_ops = one_of("exists !exists", caseless=True)
         self.not_exists_op = (CaselessKeyword("not") | "!") + CaselessKeyword("exists")
         self.bang_exists_op = Suppress("!") + CaselessKeyword("exists")
 
@@ -115,16 +143,22 @@ class TQLGrammar:
 
     def _setup_fields_and_values(self):
         """Set up field and value definitions."""
-        # Field names can contain single colons but we need to handle :: for type hints
-        # We'll match the field name greedily but stop at ::
-        self.field_name = Regex(r"[@a-zA-Z][@a-zA-Z0-9_.:-]*?(?=::|[^@a-zA-Z0-9_.:-]|$)")
+        # Field names:
+        # - May start with @ (like @timestamp, @metadata)
+        # - First char after optional @ must be a letter or underscore
+        # - Can contain letters, numbers, underscores, dots, hyphens
+        # - Colon NOT allowed (conflicts with :: type hints)
+        # - @ only allowed at start (time@stamp is INVALID)
+        # - Stops at :: for type hints
+        self.field_name = Regex(r"@?[a-zA-Z_][a-zA-Z0-9_.-]*(?=::|[^a-zA-Z0-9_.-]|$)")
 
     def _setup_mutators(self):
         """Set up mutator definitions."""
         # Define mutators
-        self.mutator_name = oneOf(
+        self.mutator_name = one_of(
             "lowercase uppercase trim split replace nslookup geoip_lookup geo "
-            "length refang defang b64encode b64decode urldecode "
+            "length refang defang b64encode b64decode urldecode hexencode hexdecode "
+            "md5 sha256 "
             "any all none avg average max min sum is_private is_global "
             "count unique first last",
             caseless=True,
@@ -137,7 +171,7 @@ class TQLGrammar:
         # Positional parameters can be strings (quoted or unquoted), numbers, or identifiers
         self.mutator_positional_param = self.string_literal | self.number | self.identifier
         self.mutator_param = self.mutator_named_param | self.mutator_positional_param
-        self.mutator_params = Group(Suppress("(") + delimitedList(self.mutator_param) + Suppress(")"))
+        self.mutator_params = Group(Suppress("(") + DelimitedList(self.mutator_param) + Suppress(")"))
         self.mutator = Group(Suppress("|") + self.mutator_name + PyparsingOptional(self.mutator_params))
         self.mutator_chain = ZeroOrMore(self.mutator)
 
@@ -188,6 +222,7 @@ class TQLGrammar:
                 | self.not_startswith_op
                 | self.not_endswith_op
                 | self.not_regexp_op
+                | self.not_matches_op
                 | self.not_cidr_op
                 | self.not_any_op
                 | self.not_all_op
@@ -197,6 +232,7 @@ class TQLGrammar:
                 | self.bang_startswith_op
                 | self.bang_endswith_op
                 | self.bang_regexp_op
+                | self.bang_matches_op
                 | self.bang_cidr_op
                 | self.bang_any_op
                 | self.bang_all_op
@@ -223,7 +259,7 @@ class TQLGrammar:
 
         # Define field list for reversed 'in' operator
         self.field_list_item = self.typed_field
-        self.field_list = Group(Suppress("[") + delimitedList(self.field_list_item) + Suppress("]"))
+        self.field_list = Group(Suppress("[") + DelimitedList(self.field_list_item) + Suppress("]"))
 
         # Special case for 'in' operator - value in field(s)
         self.value_in_field = Group(self.value + CaselessKeyword("in") + self.typed_field)
@@ -235,14 +271,14 @@ class TQLGrammar:
             self.typed_field
             + CaselessKeyword("in")
             + self.list_literal
-            + Literal("").setParseAction(lambda: "__field_in_values__")
+            + Literal("").set_parse_action(lambda: "__field_in_values__")
         )
         self.field_not_in_values = Group(
             self.typed_field
             + (CaselessKeyword("not") | Literal("!"))
             + CaselessKeyword("in")
             + self.list_literal
-            + Literal("").setParseAction(lambda: "__field_not_in_values__")
+            + Literal("").set_parse_action(lambda: "__field_not_in_values__")
         )
 
     def _setup_special_expressions(self):
@@ -259,12 +295,12 @@ class TQLGrammar:
         self.geo_param_value = (
             CaselessKeyword("true")
             | CaselessKeyword("false")
-            | QuotedString('"', escChar="\\")
-            | QuotedString("'", escChar="\\")
+            | QuotedString('"', esc_char="\\")
+            | QuotedString("'", esc_char="\\")
             | Regex(r"\d+")
         )
         self.geo_param = Group(self.geo_param_name + Suppress("=") + self.geo_param_value)
-        self.geo_params = PyparsingOptional(Suppress(",") + delimitedList(self.geo_param))
+        self.geo_params = PyparsingOptional(Suppress(",") + DelimitedList(self.geo_param))
 
         # Support multiple geo syntax patterns
         self.geo_empty = Group(
@@ -276,7 +312,7 @@ class TQLGrammar:
             + Suppress("|")
             + self.geo_kw
             + Suppress("(")
-            + delimitedList(self.geo_param)
+            + DelimitedList(self.geo_param)
             + Suppress(")")
         )
 
@@ -296,7 +332,7 @@ class TQLGrammar:
             + Suppress("(")
             + self.geo_conditions
             + Suppress(",")
-            + delimitedList(self.geo_param)
+            + DelimitedList(self.geo_param)
             + Suppress(")")
         )
 
@@ -305,7 +341,7 @@ class TQLGrammar:
             + Suppress("|")
             + self.geo_kw
             + Suppress("(")
-            + delimitedList(self.geo_param)
+            + DelimitedList(self.geo_param)
             + Suppress(",")
             + self.geo_conditions
             + Suppress(")")
@@ -329,13 +365,13 @@ class TQLGrammar:
         self.nslookup_param_value = (
             CaselessKeyword("true")
             | CaselessKeyword("false")
-            | QuotedString('"', escChar="\\")
-            | QuotedString("'", escChar="\\")
+            | QuotedString('"', esc_char="\\")
+            | QuotedString("'", esc_char="\\")
             | self.list_literal
             | Regex(r"\d+")
         )
         self.nslookup_param = Group(self.nslookup_param_name + Suppress("=") + self.nslookup_param_value)
-        self.nslookup_params = PyparsingOptional(Suppress(",") + delimitedList(self.nslookup_param))
+        self.nslookup_params = PyparsingOptional(Suppress(",") + DelimitedList(self.nslookup_param))
 
         # Support multiple nslookup syntax patterns
         self.nslookup_empty = Group(
@@ -347,7 +383,7 @@ class TQLGrammar:
             + Suppress("|")
             + self.nslookup_kw
             + Suppress("(")
-            + delimitedList(self.nslookup_param)
+            + DelimitedList(self.nslookup_param)
             + Suppress(")")
         )
 
@@ -367,7 +403,7 @@ class TQLGrammar:
             + Suppress("(")
             + self.nslookup_conditions
             + Suppress(",")
-            + delimitedList(self.nslookup_param)
+            + DelimitedList(self.nslookup_param)
             + Suppress(")")
         )
 
@@ -376,7 +412,7 @@ class TQLGrammar:
             + Suppress("|")
             + self.nslookup_kw
             + Suppress("(")
-            + delimitedList(self.nslookup_param)
+            + DelimitedList(self.nslookup_param)
             + Suppress(",")
             + self.nslookup_conditions
             + Suppress(")")
@@ -398,7 +434,7 @@ class TQLGrammar:
         self.by_kw = CaselessKeyword("by")
 
         # Aggregation function names - including aliases
-        self.agg_function_name = oneOf(
+        self.agg_function_name = one_of(
             "count unique_count sum min max average avg median med std standard_deviation "
             "percentile percentiles p pct percentile_rank percentile_ranks pct_rank pct_ranks "
             "values unique cardinality",
@@ -416,7 +452,7 @@ class TQLGrammar:
                 + Suppress("(")
                 + self.field_name
                 + PyparsingOptional(
-                    Suppress(",") + (oneOf("top bottom", caseless=True) + self.number | delimitedList(self.number))
+                    Suppress(",") + (one_of("top bottom", caseless=True) + self.number | DelimitedList(self.number))
                 )
                 + Suppress(")")
             )
@@ -429,16 +465,16 @@ class TQLGrammar:
         self.agg_with_alias = Group(self.agg_function + PyparsingOptional(self.as_kw + self.identifier))
 
         # Multiple aggregations separated by commas
-        self.agg_list = delimitedList(self.agg_with_alias)
+        self.agg_list = DelimitedList(self.agg_with_alias)
 
         # Group by fields with optional "top N" for each field
         self.top_kw = CaselessKeyword("top")
         self.group_by_field_with_bucket = Group(self.field_name + PyparsingOptional(self.top_kw + self.number))
-        self.group_by_fields = delimitedList(self.group_by_field_with_bucket)
+        self.group_by_fields = DelimitedList(self.group_by_field_with_bucket)
 
         # Visualization hint: => chart_type
         self.viz_arrow = Literal("=>")
-        self.viz_types = oneOf(
+        self.viz_types = one_of(
             "bar barchart line area pie donut scatter heatmap treemap sunburst "
             "table number gauge map grouped_bar stacked_bar nested_pie nested_donut chord",
             caseless=True,
@@ -486,7 +522,7 @@ class TQLGrammar:
         self.base_expr = self.geo_mutator_expr | self.nslookup_mutator_expr | self.comparison_expr
 
         # Define filter expression with operator precedence
-        self.filter_expr = infixNotation(
+        self.filter_expr = infix_notation(
             self.base_expr,
             [
                 (self.not_kw, 1, opAssoc.RIGHT),
@@ -508,7 +544,7 @@ class TQLGrammar:
         )
 
         # Define geo_conditions and nslookup_conditions
-        self.geo_conditions << infixNotation(
+        self.geo_conditions << infix_notation(
             self.comparison_expr,
             [
                 (self.not_kw, 1, opAssoc.RIGHT),
@@ -517,7 +553,7 @@ class TQLGrammar:
             ],
         )
 
-        self.nslookup_conditions << infixNotation(
+        self.nslookup_conditions << infix_notation(
             self.comparison_expr,
             [
                 (self.not_kw, 1, opAssoc.RIGHT),