tellaro-query-language 0.2.7__tar.gz → 0.2.10__tar.gz

{tellaro_query_language-0.2.7 → tellaro_query_language-0.2.10}/PKG-INFO

@@ -1,9 +1,9 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: tellaro-query-language
-Version: 0.2.7
+Version: 0.2.10
 Summary: A flexible, human-friendly query language for searching and filtering structured data
-Home-page: https://github.com/tellaro/tellaro-query-language
 License: Proprietary
+License-File: LICENSE
 Keywords: query,language,opensearch,elasticsearch,search,filter,tql
 Author: Justin Henderson
 Author-email: justin@tellaro.io
@@ -18,14 +18,15 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: Text Processing :: Linguistic
 Provides-Extra: opensearch
-Requires-Dist: dnspython (>=2.7.0,<3.0.0)
-Requires-Dist: maxminddb (>=2.7.0,<3.0.0)
+Requires-Dist: dnspython (>=2.8.0,<3.0.0)
+Requires-Dist: maxminddb (>=3.0.0,<4.0.0)
 Requires-Dist: opensearch-dsl (>=2.1.0,<3.0.0) ; extra == "opensearch"
-Requires-Dist: opensearch-py (>=2.4.2,<3.0.0) ; extra == "opensearch"
-Requires-Dist: pyparsing (>=3.2.1,<4.0.0)
-Requires-Dist: setuptools (>=80.0.0,<81.0.0)
-Requires-Dist: urllib3 (>=2.5.0,<3.0.0)
+Requires-Dist: opensearch-py (>=2.8.0,<3.0.0) ; extra == "opensearch"
+Requires-Dist: pyparsing (>=3.3.0,<4.0.0)
+Requires-Dist: setuptools (>=80.9.0,<81.0.0)
+Requires-Dist: urllib3 (>=2.6.0,<3.0.0)
 Project-URL: Documentation, https://github.com/tellaro/tellaro-query-language/tree/main/docs
+Project-URL: Homepage, https://github.com/tellaro/tellaro-query-language
 Project-URL: Repository, https://github.com/tellaro/tellaro-query-language
 Description-Content-Type: text/markdown
 

{tellaro_query_language-0.2.7 → tellaro_query_language-0.2.10}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "tellaro-query-language"
-version = "0.2.7"
+version = "0.2.10"
 description = "A flexible, human-friendly query language for searching and filtering structured data"
 authors = ["Justin Henderson <justin@tellaro.io>"]
 license = "Proprietary"
@@ -34,13 +34,13 @@ tests = "tql.scripts:run_tests"
 
 [tool.poetry.dependencies]
 python = ">=3.11,<3.14"
-pyparsing = "^3.2.1"
-setuptools = "^80.0.0"
-dnspython = "^2.7.0"
-opensearch-py = {version = "^2.4.2", optional = true}
+pyparsing = "^3.3.0"
+setuptools = "^80.9.0"
+dnspython = "^2.8.0"
+opensearch-py = {version = "^2.8.0", optional = true}
 opensearch-dsl = {version = "^2.1.0", optional = true}
-maxminddb = "^2.7.0"
-urllib3 = "^2.5.0"
+maxminddb = "^3.0.0"
+urllib3 = "^2.6.0"
 
 [tool.poetry.extras]
 opensearch = ["opensearch-py", "opensearch-dsl"]
@@ -76,7 +76,8 @@ vulture = "^2.14"
 types-setuptools = "^75.8.2.20250305"
 opensearch-dsl = "^2.1.0"
 python-dotenv = "^1.1.0"
-jupyter = "^1.1.1"
+# jupyter removed due to CVE-2025-53000 (nbconvert vulnerability, no patch available)
+# Use ipykernel for IDE notebook support; reinstall jupyter manually if needed
 redis = "^7.1.0"
 
 [build-system]

{tellaro_query_language-0.2.7 → tellaro_query_language-0.2.10}/src/tql/core_components/opensearch_operations.py

@@ -813,10 +813,10 @@ class OpenSearchOperations:
             # scan_all mode with post-processing - process all results
             processor = QueryPostProcessor()
 
-            # Extract all documents from hits
+            # Extract all documents from initial_hits (which contains all scrolled results)
             documents = []
             hit_metadata = []
-            for hit in hits:
+            for hit in initial_hits:
                 documents.append(hit["_source"])
                 hit_metadata.append(
                     {

{tellaro_query_language-0.2.7 → tellaro_query_language-0.2.10}/src/tql/evaluator_components/value_comparison.py

@@ -170,15 +170,17 @@ class ValueComparator:
                 return str(field_value).lower().endswith(str(expected_value).lower())
             elif operator == "in":
                 if isinstance(expected_value, list):
-                    if len(expected_value) == 1 and isinstance(field_value, list):
-                        # This is likely a reversed 'in' case: 'value' in field_list
-                        # Check if the single expected value is in the field list
-                        converted_expected = self._convert_numeric(expected_value[0])
-                        return converted_expected in field_value
+                    # Convert list elements to appropriate types for comparison
+                    converted_list = [self._convert_numeric(val) for val in expected_value]
+
+                    if isinstance(field_value, (list, tuple)):
+                        # Field is an array - check if ANY element of field equals ANY value in list
+                        # This handles both:
+                        # - "value" in field (single expected value, array field)
+                        # - field in ["val1", "val2"] (multiple expected values, array field)
+                        return any(elem in converted_list for elem in field_value)
                     else:
-                        # Standard case: field_value in list
-                        # Convert list elements to appropriate types for comparison
-                        converted_list = [self._convert_numeric(val) for val in expected_value]
+                        # Scalar field - check if field_value is in the list
                         return field_value in converted_list
                 else:
                     return field_value == expected_value

{tellaro_query_language-0.2.7 → tellaro_query_language-0.2.10}/src/tql/mutators/dns.py

@@ -246,20 +246,53 @@ class NSLookupMutator(BaseMutator):
 
         # Save enrichment if requested
         if save_enrichment:
-            # Always store ECS data directly, never use IP addresses as field names
+            # Determine the DNS data field (for full ECS structure)
+            # If append_field is like "destination.domain", dns_field is "destination.dns"
+            if append_field.endswith(".domain"):
+                dns_field = append_field.rsplit(".domain", 1)[0] + ".dns"
+            elif append_field == "domain":
+                dns_field = "dns"
+            else:
+                dns_field = append_field + "_dns"
+
             if len(queries) == 1 and queries[0] in resolved_results:
-                # Single query: store the ECS data directly
-                append_to_result(record, append_field, resolved_results[queries[0]])
+                # Single query: extract domain names for the domain field
+                dns_data = resolved_results[queries[0]]
+                answers = dns_data.get("answers", [])
+
+                # Store domain name(s) in the domain field (string or list of strings)
+                if len(answers) == 1:
+                    append_to_result(record, append_field, answers[0])
+                elif len(answers) > 1:
+                    append_to_result(record, append_field, answers)
+                # If no answers, don't set the domain field (leave it unset)
+
+                # Store full ECS data in the dns field
+                append_to_result(record, dns_field, dns_data)
+
             elif len(queries) > 1:
-                # Multiple queries: store as array of ECS results
+                # Multiple queries: collect all domain names and ECS results
+                all_domains = []
                 results_array = []
                 for query in queries:
                     if query in resolved_results:
-                        results_array.append(resolved_results[query])
-                append_to_result(record, append_field, results_array)
-            else:
-                # No results
-                append_to_result(record, append_field, None)
+                        dns_data = resolved_results[query]
+                        results_array.append(dns_data)
+                        answers = dns_data.get("answers", [])
+                        all_domains.extend(answers)
+
+                # Store unique domain names in the domain field
+                if all_domains:
+                    unique_domains = list(dict.fromkeys(all_domains))  # Preserve order, remove dupes
+                    if len(unique_domains) == 1:
+                        append_to_result(record, append_field, unique_domains[0])
+                    else:
+                        append_to_result(record, append_field, unique_domains)
+
+                # Store full ECS data array in the dns field
+                if results_array:
+                    append_to_result(record, dns_field, results_array)
+            # If no results, don't set any fields
 
         # For enrichment mutators, return data for comparison
         # The full enrichment data is stored via append_to_result above

{tellaro_query_language-0.2.7 → tellaro_query_language-0.2.10}/src/tql/parser.py

@@ -332,6 +332,29 @@ class TQLParser:
                         return result
 
             if len(parsed) == 4:
+                # Check for field_in_values marker: field in [values] __field_in_values__
+                if isinstance(parsed[3], str) and parsed[3] == "__field_in_values__":
+                    # This is field in [values] syntax
+                    field_part, op, values_list, marker = parsed
+                    field_name, type_hint, field_mutators = self.ast_builder.extract_field_info(field_part)
+                    # Extract values from the list
+                    values = []
+                    for item in values_list:
+                        if isinstance(item, list) and len(item) >= 1:
+                            values.append(item[0] if len(item) == 1 else item)
+                        else:
+                            values.append(item)
+                    result = {
+                        "type": "comparison",
+                        "field": field_name,
+                        "type_hint": type_hint,
+                        "operator": "in",
+                        "value": values,
+                    }
+                    if field_mutators:
+                        result["field_mutators"] = field_mutators
+                    return result
+
                 # Check for ANY/ALL operators: ANY field op value
                 first, field, operator, value = parsed
 
@@ -368,12 +391,25 @@ class TQLParser:
                             normalized_operator = "any"
                         else:
                             normalized_operator = f"not_{third.lower()}"
+
+                        # Extract value properly - unwrap if it's a double-wrapped list
+                        # This happens for "field not in [values]" where the list is wrapped twice
+                        value = fourth
+                        if (
+                            third.lower() == "in"
+                            and isinstance(fourth, list)
+                            and len(fourth) == 1
+                            and isinstance(fourth[0], list)
+                        ):
+                            # Unwrap the double-wrapped list for "not in" with values
+                            value = fourth[0]
+
                         result = {
                             "type": "comparison",
                             "field": field_name,
                             "type_hint": type_hint,
                             "operator": normalized_operator,
-                            "value": fourth,
+                            "value": value,
                         }
                         if field_mutators:
                             result["field_mutators"] = field_mutators
@@ -577,6 +613,29 @@ class TQLParser:
                             result["value_mutators"] = value_mutators
                         return result
             elif len(parsed) == 5:
+                # Check for field_not_in_values marker: field not in [values] __field_not_in_values__
+                if isinstance(parsed[4], str) and parsed[4] == "__field_not_in_values__":
+                    # This is field not in [values] syntax
+                    field_part, not_kw, in_kw, values_list, marker = parsed
+                    field_name, type_hint, field_mutators = self.ast_builder.extract_field_info(field_part)
+                    # Extract values from the list
+                    values = []
+                    for item in values_list:
+                        if isinstance(item, list) and len(item) >= 1:
+                            values.append(item[0] if len(item) == 1 else item)
+                        else:
+                            values.append(item)
+                    result = {
+                        "type": "comparison",
+                        "field": field_name,
+                        "type_hint": type_hint,
+                        "operator": "not_in",
+                        "value": values,
+                    }
+                    if field_mutators:
+                        result["field_mutators"] = field_mutators
+                    return result
+
                 # Check for natural between syntax: field between value1 and value2
                 # Only process as between if the second element is "between"
                 if (
@@ -996,7 +1055,7 @@ class TQLParser:
                     return {"type": "unknown", "value": parsed}
                 else:
                     # Comparison operation
-                    # Handle 'in' operator - always value in field(s)
+                    # Handle 'in' operator - can be "value in field(s)" or "field in [values]"
                     if isinstance(operator, str) and operator.lower() == "in":
                         # Check for old syntax: [field1, field2] in value
                         # The parser wraps list literals, so check for wrapped lists too
@@ -1031,26 +1090,21 @@ class TQLParser:
                                     position=0,
                                 )
 
-                        # For 'in' operator, left is always the value, right is field(s)
-                        # Extract the value from left
-                        value_extracted, value_mutators = self.ast_builder.extract_value_info(left)
-
-                        # Check if right is a list of fields
+                        # Check if right is a list of fields (value in [fields] syntax)
+                        # Note: field in [values] is now handled by field_in_values grammar rule
+                        # with __field_in_values__ marker, so this is only for value in [fields]
                         if isinstance(right, list) and len(right) > 0:
-                            # Check if all elements are fields
-                            all_fields = True
-                            for item in right:
-                                if isinstance(item, list):
-                                    # This is a typed_field group
-                                    if not (len(item) >= 1 and isinstance(item[0], str)):
-                                        all_fields = False
-                                        break
-                                elif not isinstance(item, str):
-                                    all_fields = False
-                                    break
+                            # Check if all elements are fields (typed_fields wrapped in lists)
+                            # value in [field1, field2] produces right = [['field1'], ['field2']]
+                            # field in ["val1", "val2"] produces right = ['val1', 'val2']
+                            # (but field in [values] is now handled by __field_in_values__ marker)
+                            is_field_list = all(
+                                isinstance(item, list) and len(item) >= 1 and isinstance(item[0], str) for item in right
+                            )
 
-                            if all_fields:
+                            if is_field_list:
                                 # This is "value in [field1, field2, ...]" format
+                                value_extracted, value_mutators = self.ast_builder.extract_value_info(left)
                                 # Create an OR expression for all fields
                                 field_comparisons = []
                                 for field in right:
@@ -1083,7 +1137,11 @@ class TQLParser:
                                     }
                                 return result
 
-                        # Otherwise, treat as standard "value in field" (single field)
+                        # For 'in' operator with single field, left is the value, right is field
+                        # Extract the value from left
+                        value_extracted, value_mutators = self.ast_builder.extract_value_info(left)
+
+                        # Treat as standard "value in field" (single field)
                         field_name, type_hint, field_mutators = self.ast_builder.extract_field_info(right)
                         result = {
                             "type": "comparison",

{tellaro_query_language-0.2.7 → tellaro_query_language-0.2.10}/src/tql/parser_components/grammar.py

@@ -225,10 +225,26 @@ class TQLGrammar:
         self.field_list_item = self.typed_field
         self.field_list = Group(Suppress("[") + delimitedList(self.field_list_item) + Suppress("]"))
 
-        # Special case for 'in' operator - always value in field(s)
+        # Special case for 'in' operator - value in field(s)
         self.value_in_field = Group(self.value + CaselessKeyword("in") + self.typed_field)
         self.value_in_field_list = Group(self.value + CaselessKeyword("in") + self.field_list)
 
+        # Field-first 'in' operator: field in [value1, value2, ...] (checks if field equals any value)
+        # Add a marker "__field_in_values__" to distinguish from value_in_field
+        self.field_in_values = Group(
+            self.typed_field
+            + CaselessKeyword("in")
+            + self.list_literal
+            + Literal("").setParseAction(lambda: "__field_in_values__")
+        )
+        self.field_not_in_values = Group(
+            self.typed_field
+            + (CaselessKeyword("not") | Literal("!"))
+            + CaselessKeyword("in")
+            + self.list_literal
+            + Literal("").setParseAction(lambda: "__field_not_in_values__")
+        )
+
     def _setup_special_expressions(self):
         """Set up special expressions like geo() and nslookup()."""
         # Forward declare for recursive use
@@ -455,6 +471,8 @@ class TQLGrammar:
             | self.is_not_comparison
             | self.not_between_comparison_natural
             | self.not_between_comparison_list
+            | self.field_not_in_values  # field not in [values] - must come before std_comparison
+            | self.field_in_values  # field in [values] - must come before std_comparison
             | self.std_comparison
             | self.between_comparison_natural
             | self.between_comparison_list