tellaro-query-language 0.1.0__py3-none-any.whl

tql/mutators/string.py

@@ -0,0 +1,165 @@
+"""String manipulation mutators."""
+
+from typing import Any, Dict, Optional
+
+from .base import BaseMutator, PerformanceClass, append_to_result
+
+
+class LowercaseMutator(BaseMutator):
+    """Mutator that converts a string value to lowercase.
+
+    Performance Characteristics:
+    - In-memory: FAST - Simple string operation with minimal overhead
+    - OpenSearch: MODERATE - Requires post-processing of all results
+
+    Example:
+        field | lowercase eq 'hello'
+    """
+
+    def __init__(self, params: Optional[Dict[str, Any]] = None) -> None:
+        super().__init__(params)
+        # String operations are very fast in memory
+        self.performance_in_memory = PerformanceClass.FAST
+        # Post-processing in OpenSearch has moderate impact due to result set iteration
+        self.performance_opensearch = PerformanceClass.MODERATE
+
+    def apply(self, field_name: str, record: Dict[str, Any], value: Any) -> Any:
+        if isinstance(value, str):
+            return value.lower()
+        elif isinstance(value, (list, tuple)):
+            # Apply lowercase to each element in the array
+            return [self.apply(field_name, record, item) if isinstance(item, str) else item for item in value]
+        else:
+            # For non-string values, return as-is
+            return value
+
+
+class UppercaseMutator(BaseMutator):
+    """Mutator that converts a string value to uppercase.
+
+    Performance Characteristics:
+    - In-memory: FAST - Simple string operation with minimal overhead
+    - OpenSearch: MODERATE - Requires post-processing of all results
+
+    Example:
+        field | uppercase eq 'HELLO'
+    """
+
+    def __init__(self, params: Optional[Dict[str, Any]] = None) -> None:
+        super().__init__(params)
+        self.performance_in_memory = PerformanceClass.FAST
+        self.performance_opensearch = PerformanceClass.MODERATE
+
+    def apply(self, field_name: str, record: Dict[str, Any], value: Any) -> Any:
+        if isinstance(value, str):
+            return value.upper()
+        elif isinstance(value, (list, tuple)):
+            # Apply uppercase to each element in the array
+            return [self.apply(field_name, record, item) if isinstance(item, str) else item for item in value]
+        else:
+            # For non-string values, return as-is
+            return value
+
+
+class TrimMutator(BaseMutator):
+    """Mutator that trims whitespace from a string value.
+
+    Performance Characteristics:
+    - In-memory: FAST - Simple string operation with minimal overhead
+    - OpenSearch: MODERATE - Requires post-processing of all results
+
+    Example:
+        field | trim eq 'hello world'
+    """
+
+    def __init__(self, params: Optional[Dict[str, Any]] = None) -> None:
+        super().__init__(params)
+        self.performance_in_memory = PerformanceClass.FAST
+        self.performance_opensearch = PerformanceClass.MODERATE
+
+    def apply(self, field_name: str, record: Dict[str, Any], value: Any) -> Any:
+        if isinstance(value, str):
+            return value.strip()
+        elif isinstance(value, (list, tuple)):
+            # Apply trim to each element in the array
+            return [self.apply(field_name, record, item) if isinstance(item, str) else item for item in value]
+        else:
+            # For non-string values, return as-is
+            return value
+
+
+class SplitMutator(BaseMutator):
+    """Mutator that splits a string value on a delimiter."""
+
+    def apply(self, field_name: str, record: Dict[str, Any], value: Any) -> Any:
+        """Apply the split transformation."""
+        delimiter = self.params.get("delimiter", " ")
+        append_field = self.params.get("field")
+
+        # Perform the split operation
+        if value is None:
+            # Handle None - return empty list
+            split_result = []
+        elif isinstance(value, str):
+            split_result = value.split(delimiter)
+        elif isinstance(value, list):
+            # Split each string in the list
+            split_result = []
+            for item in value:
+                if isinstance(item, str):
+                    split_result.extend(item.split(delimiter))
+                else:
+                    # Keep non-string items as-is
+                    split_result.append(item)
+        elif isinstance(value, (int, float, bool)):
+            # Convert to string first, then split
+            split_result = str(value).split(delimiter)
+        else:
+            # For other types (dict, etc), return as single-item list
+            # This maintains list type consistency for the split operation
+            split_result = [value]
+
+        # If append_field is specified, add to record and return original value
+        if append_field:
+            append_to_result(record, append_field, split_result)
+            return value
+        else:
+            # Return the split result directly
+            return split_result
+
+
+class LengthMutator(BaseMutator):
+    """Mutator that returns the length of a string or list."""
+
+    def apply(self, field_name: str, record: Dict[str, Any], value: Any) -> Any:
+        """Apply the length transformation."""
+        append_field = self.params.get("field")
+
+        # Calculate length
+        if value is None:
+            # Handle None gracefully - treat as empty
+            length_value = 0
+        elif isinstance(value, (str, list, dict, tuple, set)):
+            length_value = len(value)
+        elif isinstance(value, (int, float)):
+            # For numbers, convert to string and get length
+            # This allows checking number of digits
+            length_value = len(str(value))
+        elif isinstance(value, bool):
+            # For booleans, return length of string representation
+            length_value = len(str(value))
+        else:
+            # For other types, try to get length or return 0
+            try:
+                length_value = len(value)
+            except TypeError:
+                # If object doesn't support len(), return 0
+                length_value = 0
+
+        # If append_field is specified, add to record and return original value
+        if append_field:
+            append_to_result(record, append_field, length_value)
+            return value
+        else:
+            # Return the length value directly
+            return length_value

tql/opensearch.py

@@ -0,0 +1,78 @@
+"""OpenSearch backend for TQL.
+
+This module provides OpenSearch integration for TQL, converting TQL queries
+to OpenSearch Query DSL with intelligent field selection based on operators.
+"""
+
+from typing import Any, Dict, Optional, Union
+
+from .opensearch_components import FieldMapping, LuceneConverter, QueryConverter
+
+
+class OpenSearchBackend:
+    """OpenSearch backend for TQL query conversion with intelligent field selection."""
+
+    def __init__(self, field_mappings: Optional[Dict[str, Union[str, Dict[str, Any]]]] = None):
+        """Initialize the OpenSearch backend.
+
+        Args:
+            field_mappings: Field mappings in two formats:
+                          1. Simple: {"field_name": "target_field_name"}
+                          2. Intelligent: {"field_name": {"field_name": "keyword", "field_name.text": "text"}}
+        """
+        self.field_mappings = field_mappings or {}
+        self.intelligent_mappings = {}
+        self.simple_mappings = {}
+
+        # Parse field mappings
+        for field_name, mapping in self.field_mappings.items():
+            if isinstance(mapping, dict):
+                # Check if this is an OpenSearch-style mapping with "type" and "fields"
+                if "type" in mapping and not any(k for k in mapping.keys() if k not in ["type", "fields", "analyzer"]):
+                    # This is an OpenSearch-style mapping for a single field
+                    field_mapping = FieldMapping(mapping)
+                    field_mapping.set_base_field_name(field_name)
+                    self.intelligent_mappings[field_name] = field_mapping
+                else:
+                    # Traditional intelligent mapping with multiple field variants
+                    self.intelligent_mappings[field_name] = FieldMapping(mapping)
+            elif isinstance(mapping, str):
+                # Check if this looks like a type specification (common OpenSearch types)
+                if mapping in [
+                    "keyword",
+                    "text",
+                    "long",
+                    "integer",
+                    "short",
+                    "byte",
+                    "double",
+                    "float",
+                    "boolean",
+                    "date",
+                    "ip",
+                ]:
+                    # This is a type specification, create an intelligent mapping
+                    self.intelligent_mappings[field_name] = FieldMapping({field_name: mapping})
+                else:
+                    # Simple field name mapping (backward compatibility)
+                    self.simple_mappings[field_name] = mapping
+
+        # Initialize converters
+        self.query_converter = QueryConverter(self.intelligent_mappings, self.simple_mappings)
+        self.lucene_converter = LuceneConverter(self.intelligent_mappings, self.simple_mappings)
+
+    def convert(self, ast: Dict[str, Any]) -> Dict[str, Any]:
+        """Convert a TQL AST to OpenSearch Query DSL.
+
+        Args:
+            ast: The parsed TQL query AST
+
+        Returns:
+            OpenSearch Query DSL as a dictionary
+        """
+        query_dsl = {"query": self.query_converter.convert_node(ast)}
+        return query_dsl
+
+    def convert_lucene(self, ast: Dict[str, Any]) -> str:
+        """Convert a TQL AST to Lucene query string."""
+        return self.lucene_converter.convert_lucene(ast)

tql/opensearch_components/README.md

@@ -0,0 +1,130 @@
+# OpenSearch Components
+
+This package contains the modular components for OpenSearch backend integration.
+
+## Overview
+
+The OpenSearch components package provides intelligent query conversion from TQL to OpenSearch Query DSL with field-aware optimizations.
+
+### Components
+
+#### `field_mapping.py` - Field Mapping Logic
+Manages intelligent field selection based on field types and operators:
+- Supports multiple mapping formats (simple, intelligent, OpenSearch-style)
+- Automatic field variant selection (keyword vs text fields)
+- Operator compatibility validation
+- Analyzer-aware field selection
+
+**Key Classes:**
+- `FieldMapping` - Represents field mapping configuration
+
+**Key Methods:**
+- `get_field_for_operator()` - Select optimal field variant for operator
+- `validate_operator_for_field_type()` - Check operator/field compatibility
+- `needs_wildcard_conversion()` - Determine if wildcard conversion needed
+
+#### `query_converter.py` - Query Conversion
+Converts TQL AST to OpenSearch Query DSL:
+- Handles all TQL operators and expressions
+- Intelligent query generation based on field types
+- Post-processing detection for complex operations
+- Special handling for array operators (ANY, ALL)
+
+**Key Classes:**
+- `QueryConverter` - Main conversion engine
+
+**Key Methods:**
+- `convert_node()` - Convert AST node to OpenSearch query
+- `_convert_comparison()` - Handle comparison operations
+- `_convert_logical_op()` - Handle AND/OR operations
+- `_convert_geo_expr()` - Handle geo expressions
+
+#### `lucene_converter.py` - Lucene Query Conversion
+Converts TQL AST to Lucene query strings:
+- Alternative query format for Lucene-based systems
+- Proper escaping of special characters
+- Support for all TQL operators
+- Field name resolution
+
+**Key Classes:**
+- `LuceneConverter` - Lucene string converter
+
+**Key Methods:**
+- `convert_lucene()` - Convert AST to Lucene query string
+- `_escape_lucene_value()` - Escape special characters
+
+## Field Mapping Formats
+
+### Simple Mapping
+```python
+field_mappings = {
+    "src_ip": "source.ip",  # Simple rename
+    "status": "event.status"
+}
+```
+
+### Type Specification
+```python
+field_mappings = {
+    "ip_address": "ip",      # Field type
+    "user_id": "keyword",
+    "score": "float"
+}
+```
+
+### Intelligent Mapping
+```python
+field_mappings = {
+    "message": {
+        "message": "keyword",
+        "message.text": {"type": "text", "analyzer": "standard"},
+        "message.english": {"type": "text", "analyzer": "english"}
+    }
+}
+```
+
+### OpenSearch-Style Mapping
+```python
+field_mappings = {
+    "title": {
+        "type": "text",
+        "fields": {
+            "keyword": {"type": "keyword"},
+            "autocomplete": {"type": "text", "analyzer": "autocomplete"}
+        }
+    }
+}
+```
+
+## Operator Field Selection
+
+The system automatically selects the appropriate field variant based on the operator:
+
+| Operator Type | Preferred Field | Example |
+|--------------|-----------------|---------|
+| Exact match (=, !=) | keyword | `title.keyword` |
+| Text search (contains) | text | `title` or `title.text` |
+| Wildcard (startswith) | keyword | `title.keyword` |
+| Range (>, <) | numeric/keyword | `price` or `timestamp` |
+| CIDR | ip/keyword | `source.ip` |
+
+## Usage
+
+Used internally by `OpenSearchBackend`:
+
+```python
+from tql import TQL
+
+tql = TQL(field_mappings={
+    "message": {
+        "message": "keyword",
+        "message.text": "text"
+    }
+})
+
+# Automatically uses message.text for text search
+query = tql.to_opensearch("message contains 'error'")
+
+# Automatically uses message (keyword) for exact match
+query = tql.to_opensearch("message = 'ERROR: Failed'")
+```

tql/opensearch_components/__init__.py

@@ -0,0 +1,17 @@
+"""OpenSearch backend components for TQL.
+
+This package contains modular components for OpenSearch integration:
+- field_mapping: Field mapping and intelligent field selection
+- query_converter: TQL AST to OpenSearch Query DSL conversion
+- lucene_converter: TQL AST to Lucene query string conversion
+"""
+
+from .field_mapping import FieldMapping
+from .lucene_converter import LuceneConverter
+from .query_converter import QueryConverter
+
+__all__ = [
+    "FieldMapping",
+    "QueryConverter",
+    "LuceneConverter",
+]