tellaro-query-language 0.1.9__tar.gz → 0.2.1__tar.gz

{tellaro_query_language-0.1.9 → tellaro_query_language-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tellaro-query-language
-Version: 0.1.9
+Version: 0.2.1
 Summary: A flexible, human-friendly query language for searching and filtering structured data
 Home-page: https://github.com/tellaro/tellaro-query-language
 License: MIT
@@ -217,6 +217,28 @@ poetry run tests
 
 **Note**: The development setup uses `python-dotenv` to load OpenSearch credentials from `.env` files for integration testing. This is NOT required when using TQL as a package - see the [Package Usage Guide](docs/package-usage-guide.md) for production configuration patterns.
 
+### TQL Playground
+
+The repository includes an interactive web playground for testing TQL queries:
+
+```bash
+# Navigate to the playground directory
+cd playground
+
+# Start with Docker (recommended)
+docker-compose up
+
+# Or start with OpenSearch included
+docker-compose --profile opensearch up
+```
+
+Access the playground at:
+- Frontend: http://localhost:5173
+- API: http://localhost:8000
+- API Docs: http://localhost:8000/docs
+
+The playground uses your local TQL source code, so any changes you make are immediately reflected. See [playground/README.md](playground/README.md) for more details.
+
 ### File Operations
 
 ```python

{tellaro_query_language-0.1.9 → tellaro_query_language-0.2.1}/README.md

@@ -187,6 +187,28 @@ poetry run tests
 
 **Note**: The development setup uses `python-dotenv` to load OpenSearch credentials from `.env` files for integration testing. This is NOT required when using TQL as a package - see the [Package Usage Guide](docs/package-usage-guide.md) for production configuration patterns.
 
+### TQL Playground
+
+The repository includes an interactive web playground for testing TQL queries:
+
+```bash
+# Navigate to the playground directory
+cd playground
+
+# Start with Docker (recommended)
+docker-compose up
+
+# Or start with OpenSearch included
+docker-compose --profile opensearch up
+```
+
+Access the playground at:
+- Frontend: http://localhost:5173
+- API: http://localhost:8000
+- API Docs: http://localhost:8000/docs
+
+The playground uses your local TQL source code, so any changes you make are immediately reflected. See [playground/README.md](playground/README.md) for more details.
+
 ### File Operations
 
 ```python

{tellaro_query_language-0.1.9 → tellaro_query_language-0.2.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "tellaro-query-language"
-version = "0.1.9"
+version = "0.2.1"
 description = "A flexible, human-friendly query language for searching and filtering structured data"
 authors = ["Justin Henderson <justin@tellaro.io>"]
 license = "MIT"

{tellaro_query_language-0.1.9 → tellaro_query_language-0.2.1}/src/tql/core.py

@@ -212,8 +212,10 @@ class TQL:
         if query_type == "stats_expr":
             # This is a pure stats query like "| stats count()"
             stats_result = self.stats(data, query)
+            # Get viz_hint from the parsed AST
+            viz_hint = ast.get("viz_hint")
             # Convert to execute_opensearch format
-            result["stats"] = self._convert_stats_result(stats_result)
+            result["stats"] = self._convert_stats_result(stats_result, viz_hint)
             result["total"] = len(records)
             return result
 
@@ -232,7 +234,9 @@ class TQL:
 
             # Apply stats to filtered data
             stats_result = self.stats_evaluator.evaluate_stats(filtered_records, stats_ast)
-            result["stats"] = self._convert_stats_result(stats_result)
+            # Get viz_hint from the stats AST
+            viz_hint = stats_ast.get("viz_hint")
+            result["stats"] = self._convert_stats_result(stats_result, viz_hint)
             result["total"] = len(filtered_records)
 
             # Include filtered documents if size > 0
@@ -241,40 +245,63 @@ class TQL:
 
             return result
 
-        # Handle regular filter queries
-        matched_records = []
-        has_enrichments = False
+        # Handle regular filter queries with mutator analysis
+        # Analyze the query for mutators
+        from .mutator_analyzer import MutatorAnalyzer
+        from .post_processor import QueryPostProcessor
+
+        analyzer = MutatorAnalyzer(field_mappings=self.field_mappings)
+        analysis_result = analyzer.analyze_ast(ast, context="in_memory")
 
+        # Use the optimized AST for evaluation
+        optimized_ast = analysis_result.optimized_ast
+
+        # First pass: collect all matching records using optimized AST
+        matched_records = []
         for record in records:
-            # Check if record matches
-            if self.evaluator._evaluate_node(ast, record, self._simple_mappings):
-                # Apply any mutators to enrich the record
-                enriched_record = self._apply_mutators_to_record(ast, record)
-                matched_records.append(enriched_record)
-                # Check if enrichments were added
-                if not has_enrichments and enriched_record is not record:
-                    has_enrichments = True
+            # Check if record matches using the optimized AST (without array operators)
+            if self.evaluator._evaluate_node(optimized_ast, record, self._simple_mappings):
+                matched_records.append(record)
+
+        # Apply post-processing if needed
+        if analysis_result.post_processing_requirements:
+            processor = QueryPostProcessor()
+
+            # Apply mutators/enrichments
+            processed_records = processor.process_results(
+                matched_records, analysis_result.post_processing_requirements, track_enrichments=save_enrichment
+            )
+
+            # Apply filters (for array operators like any/all/none)
+            filtered_records = processor.filter_results(processed_records, analysis_result.post_processing_requirements)
+
+            matched_records = filtered_records
+            result["post_processing_applied"] = True
+
+            # Add post-processing stats
+            result["post_processing_stats"] = {
+                "documents_retrieved": len(processed_records),
+                "documents_returned": len(filtered_records),
+                "documents_filtered": len(processed_records) - len(filtered_records),
+            }
 
         # Set result data
         result["total"] = len(matched_records)
         if size > 0:
             result["results"] = matched_records[:size]
 
-        # Check if post-processing (mutators) were applied
-        if has_enrichments:
-            result["post_processing_applied"] = True
+        # Update health status based on analysis
+        result["health_status"] = analysis_result.health_status
+        result["health_reasons"] = [
+            reason.get("reason", reason.get("description", "")) for reason in analysis_result.health_reasons
+        ]
 
         # Save enrichments if requested
-        if save_enrichment and has_enrichments and source_file:
-            # For file sources, update all records (not just matches)
-            all_enriched = []
-            for record in records:
-                enriched_record = self._apply_mutators_to_record(ast, record)
-                all_enriched.append(enriched_record)
-
-            # Save based on file type
-            if source_file.lower().endswith(".json"):
-                self.file_ops.save_enrichments_to_json(source_file, all_enriched)
+        if save_enrichment and result.get("post_processing_applied") and source_file:
+            # For file sources, we would need to re-process all records with enrichments
+            # This is a complex operation that would require applying mutators to all records
+            # For now, we'll skip this functionality in the in-memory implementation
+            pass
 
         return result
 
@@ -370,6 +397,38 @@ class TQL:
                 elif node_type == "unary_op":
                     stats["logical_operators"].add("not")
                     traverse_ast(node.get("operand"), depth + 1)
+                elif node_type == "query_with_stats":
+                    # Traverse into the filter part to find mutators and fields
+                    filter_node = node.get("filter")
+                    if filter_node:
+                        traverse_ast(filter_node, depth + 1)
+                    # Also traverse the stats part
+                    stats_node = node.get("stats")
+                    if stats_node:
+                        traverse_ast(stats_node, depth + 1)
+                elif node_type == "stats_expr":
+                    # Check aggregations for any fields or mutators
+                    aggregations = node.get("aggregations", [])
+                    for agg in aggregations:
+                        if isinstance(agg, dict):
+                            field = agg.get("field")
+                            if field and field != "*":
+                                stats["fields"].add(field)
+                            # Check for field mutators in aggregations
+                            if agg.get("field_mutators"):
+                                stats["has_mutators"] = True
+                elif node_type == "geo_expr":
+                    # Geo expressions always have mutators
+                    field = node.get("field")
+                    if field:
+                        stats["fields"].add(field)
+                    stats["has_mutators"] = True
+                elif node_type == "nslookup_expr":
+                    # NSLookup expressions always have mutators
+                    field = node.get("field")
+                    if field:
+                        stats["fields"].add(field)
+                    stats["has_mutators"] = True
 
         traverse_ast(ast)
 
@@ -451,6 +510,7 @@ class TQL:
         query: Optional[str] = None,
         size: int = 500,
         from_: int = 0,
+        sort: Optional[List[Dict[str, Any]]] = None,
         timestamp_field: str = "@timestamp",
         time_range: Optional[Dict[str, str]] = None,
         scan_all: bool = False,
@@ -468,7 +528,8 @@ class TQL:
             index: Index name to search
             query: The TQL query string
             size: Number of results to return (default: 500)
-            from_: Starting offset for pagination (default: 0)
+            search_after: Values from previous result's sort field for pagination
+            sort: Sort order specification (e.g., [{"@timestamp": "desc"}, {"_id": "asc"}])
             timestamp_field: Field name for timestamp filtering (default: "@timestamp")
             time_range: Optional time range dict with 'gte' and/or 'lte' keys
             scan_all: If True, use scroll API to retrieve all matching documents
@@ -480,6 +541,7 @@ class TQL:
             Dictionary containing:
             - results: List of processed results
             - total: Total number of matching documents
+            - sort_values: Sort values of the last document (for search_after pagination)
             - post_processing_applied: Whether post-processing was applied
             - health_status: Query health status
             - health_reasons: List of health issues
@@ -511,6 +573,8 @@ class TQL:
                 "scan_all": scan_all,
                 "scroll_size": scroll_size,
                 "scroll_timeout": scroll_timeout,
+                "from_": from_,
+                "sort": sort,
             }
         )
 
@@ -519,7 +583,7 @@ class TQL:
             filtered_kwargs["client"] = opensearch_client
 
         # Execute using new implementation
-        results = self.opensearch_ops.execute_opensearch(query, index=index, size=size, from_=from_, **filtered_kwargs)
+        results = self.opensearch_ops.execute_opensearch(query, index=index, size=size, **filtered_kwargs)
 
         # Convert to old format if needed
         if isinstance(results, list):
@@ -978,22 +1042,140 @@ class TQL:
         Returns:
             Enriched record (may be same as input if no enrichments)
         """
-        # For now, return the original record
-        # TODO: Implement mutator application for enrichment  # noqa: W0511
-        return record
+        # Check if we need to apply mutators
+        if not self._has_output_mutators(ast):
+            return record
+
+        # Deep copy to avoid modifying original
+        import copy
+
+        enriched_record = copy.deepcopy(record)
+
+        # Apply mutators from AST nodes
+        self._apply_node_mutators(ast, enriched_record)
+
+        return enriched_record
+
+    def _has_output_mutators(self, ast: Dict[str, Any]) -> bool:
+        """Check if AST contains mutators that should transform output.
+
+        Args:
+            ast: Query AST
+
+        Returns:
+            True if output mutators are present
+        """
+        if isinstance(ast, dict):
+            node_type = ast.get("type")
+
+            # Check for field mutators with exists operator (output transformation)
+            if node_type == "comparison" and ast.get("operator") == "exists" and ast.get("field_mutators"):
+                return True
+
+            # Recursively check child nodes
+            if node_type == "logical_op":
+                left = ast.get("left", {})
+                right = ast.get("right", {})
+                return self._has_output_mutators(left) or self._has_output_mutators(right)
+            elif node_type == "unary_op":
+                operand = ast.get("operand", {})
+                return self._has_output_mutators(operand)
+
+        return False
+
+    def _apply_node_mutators(self, ast: Dict[str, Any], record: Dict[str, Any]) -> None:
+        """Apply mutators from AST nodes to the record.
 
-    def _convert_stats_result(self, stats_result: Dict[str, Any]) -> Dict[str, Any]:
+        Args:
+            ast: Query AST
+            record: Record to modify (in-place)
+        """
+        if not isinstance(ast, dict):
+            return
+
+        node_type = ast.get("type")
+
+        # Apply mutators for exists operator (output transformation)
+        if node_type == "comparison" and ast.get("operator") == "exists" and ast.get("field_mutators"):
+            field_name = ast["field"]
+            field_mutators = ast["field_mutators"]
+
+            # Get field value
+            field_value = self._get_nested_field(record, field_name)
+
+            if field_value is not None:
+                # Apply mutators
+                from .mutators import apply_mutators
+
+                mutated_value = apply_mutators(field_value, field_mutators, field_name, record)
+
+                # Update record with mutated value
+                self._set_nested_field(record, field_name, mutated_value)
+
+        # Recursively process child nodes
+        elif node_type == "logical_op":
+            self._apply_node_mutators(ast.get("left", {}), record)
+            self._apply_node_mutators(ast.get("right", {}), record)
+        elif node_type == "unary_op":
+            self._apply_node_mutators(ast.get("operand", {}), record)
+
+    def _get_nested_field(self, record: Dict[str, Any], field_path: str) -> Any:
+        """Get value from nested field path.
+
+        Args:
+            record: Record dictionary
+            field_path: Dot-separated field path
+
+        Returns:
+            Field value or None if not found
+        """
+        parts = field_path.split(".")
+        current = record
+
+        for part in parts:
+            if isinstance(current, dict) and part in current:
+                current = current[part]
+            else:
+                return None
+
+        return current
+
+    def _set_nested_field(self, record: Dict[str, Any], field_path: str, value: Any) -> None:
+        """Set value in nested field path.
+
+        Args:
+            record: Record dictionary to modify
+            field_path: Dot-separated field path
+            value: Value to set
+        """
+        parts = field_path.split(".")
+        current = record
+
+        # Navigate to parent of target field
+        for part in parts[:-1]:
+            if part not in current:
+                current[part] = {}
+            current = current[part]
+
+        # Set the value
+        if len(parts) > 0:
+            current[parts[-1]] = value
+
+    def _convert_stats_result(self, stats_result: Dict[str, Any], viz_hint: Optional[str] = None) -> Dict[str, Any]:
         """Convert stats result from query() format to execute_opensearch format.
 
         Args:
             stats_result: Result from stats() or query_stats() method
+            viz_hint: Optional visualization hint from the query
 
         Returns:
             Stats result in execute_opensearch format
         """
         # Map the stats evaluator format to execute_opensearch format
+        result = {}
+
         if stats_result.get("type") == "simple_aggregation":
-            return {
+            result = {
                 "type": "stats",
                 "operation": stats_result["function"],
                 "field": stats_result["field"],
@@ -1001,10 +1183,22 @@ class TQL:
             }
         elif stats_result.get("type") == "multiple_aggregations":
             # For multiple aggregations, return the results dict
-            return {"type": "stats_multiple", "results": stats_result["results"]}
+            result = {"type": "stats_multiple", "results": stats_result["results"]}
         elif stats_result.get("type") == "grouped_aggregation":
             # For grouped aggregations
-            return {"type": "stats_grouped", "group_by": stats_result["group_by"], "results": stats_result["results"]}
+            result = {
+                "type": "stats_grouped",
+                "group_by": stats_result["group_by"],
+                "results": stats_result["results"],
+                "operation": stats_result.get("function", "count"),
+                "field": stats_result.get("field", "*"),
+            }
         else:
             # Return as-is if format is unknown
-            return stats_result
+            result = stats_result
+
+        # Add visualization hint if provided
+        if viz_hint:
+            result["viz_hint"] = viz_hint
+
+        return result