tellaro-query-language 0.1.0__py3-none-any.whl

tql/mutators/list.py

@@ -0,0 +1,212 @@
+"""List evaluation mutators for aggregate operations."""
+
+from typing import Any, Dict
+
+from .base import BaseMutator
+
+
+class AnyMutator(BaseMutator):
+    """
+    Mutator that evaluates if any element in a list is truthy.
+
+    For lists: Returns True if any element is truthy.
+    For single values: Returns the truthiness of the value.
+    For None/empty: Returns False.
+    """
+
+    def apply(self, field_name: str, record: Dict[str, Any], value: Any) -> Any:  # noqa: C901
+        """Apply the any transformation."""
+        if value is None:
+            return False
+        elif isinstance(value, list):
+            return any(value)
+        else:
+            # For single values, return truthiness
+            return bool(value)
+
+
+class AllMutator(BaseMutator):
+    """
+    Mutator that evaluates if all elements in a list are truthy.
+
+    For lists: Returns True if all elements are truthy.
+    For single values: Returns the truthiness of the value.
+    For None: Returns False.
+    For empty lists: Returns True (following Python's all() behavior).
+    """
+
+    def apply(self, field_name: str, record: Dict[str, Any], value: Any) -> Any:  # noqa: C901
+        """Apply the all transformation."""
+        if value is None:
+            return False
+        elif isinstance(value, list):
+            return all(value)
+        else:
+            # For single values, return truthiness
+            return bool(value)
+
+
+class AvgMutator(BaseMutator):
+    """
+    Mutator that calculates the average of numeric values.
+
+    For lists: Returns the average of numeric elements.
+    For single numeric values: Returns the value itself.
+    For non-numeric or empty: Returns None.
+    """
+
+    def apply(self, field_name: str, record: Dict[str, Any], value: Any) -> Any:  # noqa: C901
+        """Apply the average transformation."""
+        if value is None:
+            return None
+        elif isinstance(value, list):
+            numeric_values = []
+            for item in value:
+                if isinstance(item, (int, float)) and not isinstance(item, bool):
+                    numeric_values.append(item)
+                elif isinstance(item, str):
+                    # Try to convert string to number
+                    try:
+                        numeric_values.append(float(item))
+                    except ValueError:
+                        pass
+
+            if numeric_values:
+                return sum(numeric_values) / len(numeric_values)
+            else:
+                return None
+        elif isinstance(value, (int, float)) and not isinstance(value, bool):
+            return value
+        elif isinstance(value, str):
+            # Try to convert string to number
+            try:
+                return float(value)
+            except ValueError:
+                return None
+        else:
+            return None
+
+
+# Alias for AvgMutator
+class AverageMutator(AvgMutator):
+    """Alias for AvgMutator."""
+
+
+class SumMutator(BaseMutator):
+    """
+    Mutator that calculates the sum of numeric values.
+
+    For lists: Returns the sum of numeric elements.
+    For single numeric values: Returns the value itself.
+    For non-numeric or empty: Returns 0.
+    """
+
+    def apply(self, field_name: str, record: Dict[str, Any], value: Any) -> Any:  # noqa: C901
+        """Apply the sum transformation."""
+        if value is None:
+            return 0
+        elif isinstance(value, list):
+            numeric_values = []
+            for item in value:
+                if isinstance(item, (int, float)) and not isinstance(item, bool):
+                    numeric_values.append(item)
+                elif isinstance(item, str):
+                    # Try to convert string to number
+                    try:
+                        numeric_values.append(float(item))
+                    except ValueError:
+                        pass
+
+            return sum(numeric_values) if numeric_values else 0
+        elif isinstance(value, (int, float)) and not isinstance(value, bool):
+            return value
+        elif isinstance(value, str):
+            # Try to convert string to number
+            try:
+                return float(value)
+            except ValueError:
+                return 0
+        else:
+            return 0
+
+
+class MaxMutator(BaseMutator):
+    """
+    Mutator that finds the maximum value.
+
+    For lists: Returns the maximum of comparable elements.
+    For single values: Returns the value itself.
+    For empty or incomparable: Returns None.
+    """
+
+    def apply(self, field_name: str, record: Dict[str, Any], value: Any) -> Any:  # noqa: C901
+        """Apply the max transformation."""
+        if value is None:
+            return None
+        elif isinstance(value, list):
+            if not value:
+                return None
+
+            # Filter out None values
+            filtered_values = [v for v in value if v is not None]
+            if not filtered_values:
+                return None
+
+            try:
+                return max(filtered_values)
+            except (TypeError, ValueError):
+                # If values aren't comparable, try numeric conversion
+                numeric_values = []
+                for item in filtered_values:
+                    if isinstance(item, (int, float)) and not isinstance(item, bool):
+                        numeric_values.append(item)
+                    elif isinstance(item, str):
+                        try:
+                            numeric_values.append(float(item))
+                        except ValueError:
+                            pass
+
+                return max(numeric_values) if numeric_values else None
+        else:
+            return value
+
+
+class MinMutator(BaseMutator):
+    """
+    Mutator that finds the minimum value.
+
+    For lists: Returns the minimum of comparable elements.
+    For single values: Returns the value itself.
+    For empty or incomparable: Returns None.
+    """
+
+    def apply(self, field_name: str, record: Dict[str, Any], value: Any) -> Any:  # noqa: C901
+        """Apply the min transformation."""
+        if value is None:
+            return None
+        elif isinstance(value, list):
+            if not value:
+                return None
+
+            # Filter out None values
+            filtered_values = [v for v in value if v is not None]
+            if not filtered_values:
+                return None
+
+            try:
+                return min(filtered_values)
+            except (TypeError, ValueError):
+                # If values aren't comparable, try numeric conversion
+                numeric_values = []
+                for item in filtered_values:
+                    if isinstance(item, (int, float)) and not isinstance(item, bool):
+                        numeric_values.append(item)
+                    elif isinstance(item, str):
+                        try:
+                            numeric_values.append(float(item))
+                        except ValueError:
+                            pass
+
+                return min(numeric_values) if numeric_values else None
+        else:
+            return value

tql/mutators/network.py

@@ -0,0 +1,163 @@
+"""Network-related mutators for IP address operations."""
+
+import ipaddress
+from typing import Any, Dict, Optional
+
+from .base import BaseMutator, PerformanceClass
+
+
+class IsPrivateMutator(BaseMutator):
+    """
+    Mutator that checks if an IP address is in a private range.
+
+    Performance Characteristics:
+    - In-memory: FAST - Simple IP range calculations
+    - OpenSearch: MODERATE - Requires post-processing of all results
+
+    This mutator returns True if the IP is in one of the RFC 1918 private ranges:
+    - 10.0.0.0/8 (10.0.0.0 - 10.255.255.255)
+    - 172.16.0.0/12 (172.16.0.0 - 172.31.255.255)
+    - 192.168.0.0/16 (192.168.0.0 - 192.168.255.255)
+
+    Also includes other private/special ranges:
+    - 127.0.0.0/8 (loopback)
+    - 169.254.0.0/16 (link-local)
+    - fc00::/7 (IPv6 unique local)
+    - fe80::/10 (IPv6 link-local)
+
+    Used as a filter in queries like: ip | is_private()
+    Returns True/False for filtering purposes.
+
+    Example:
+        source_ip | is_private()
+    """
+
+    def __init__(self, params: Optional[Dict[str, Any]] = None) -> None:
+        super().__init__(params)
+        self.performance_in_memory = PerformanceClass.FAST
+        self.performance_opensearch = PerformanceClass.MODERATE
+
+    def apply(self, field_name: str, record: Dict[str, Any], value: Any) -> Any:
+        """Check if the value is a private IP address."""
+        if value is None:
+            return False
+
+        # Handle lists - return True if any IP is private
+        if isinstance(value, list):
+            return any(self._is_private_ip(str(item)) for item in value if item is not None)
+
+        # Single value
+        return self._is_private_ip(str(value))
+
+    def _is_private_ip(self, ip_str: str) -> bool:
+        """Check if a single IP address is private."""
+        try:
+            ip_obj = ipaddress.ip_address(ip_str)
+
+            # Check if it's a private address
+            # This includes RFC 1918 for IPv4 and fc00::/7 for IPv6
+            if ip_obj.is_private:
+                return True
+
+            # Also check other special-use addresses
+            if ip_obj.is_loopback:  # 127.0.0.0/8 or ::1
+                return True
+            if ip_obj.is_link_local:  # 169.254.0.0/16 or fe80::/10
+                return True
+            if hasattr(ip_obj, "is_reserved") and ip_obj.is_reserved:  # Reserved addresses
+                return True
+
+            return False
+
+        except (ValueError, AttributeError):
+            # Not a valid IP address
+            return False
+
+
+class IsGlobalMutator(BaseMutator):
+    """
+    Mutator that checks if an IP address is globally routable.
+
+    Performance Characteristics:
+    - In-memory: FAST - Simple IP range calculations
+    - OpenSearch: MODERATE - Requires post-processing of all results
+
+    This mutator returns True if the IP is a globally routable address,
+    meaning it's not:
+    - Private (RFC 1918)
+    - Loopback
+    - Link-local
+    - Multicast
+    - Reserved
+    - Unspecified
+
+    Used as a filter in queries like: ip | is_global()
+    Returns True/False for filtering purposes.
+
+    Example:
+        destination_ip | is_global()
+    """
+
+    def __init__(self, params: Optional[Dict[str, Any]] = None) -> None:
+        super().__init__(params)
+        self.performance_in_memory = PerformanceClass.FAST
+        self.performance_opensearch = PerformanceClass.MODERATE
+
+    def apply(self, field_name: str, record: Dict[str, Any], value: Any) -> Any:
+        """Check if the value is a global IP address."""
+        if value is None:
+            return False
+
+        # Handle lists - return True if any IP is global
+        if isinstance(value, list):
+            return any(self._is_global_ip(str(item)) for item in value if item is not None)
+
+        # Single value
+        return self._is_global_ip(str(value))
+
+    def _is_global_ip(self, ip_str: str) -> bool:  # noqa: C901
+        """Check if a single IP address is globally routable."""
+        try:
+            ip_obj = ipaddress.ip_address(ip_str)
+
+            # Check if it's NOT any of the special-use addresses
+            if ip_obj.is_private:
+                return False
+            if ip_obj.is_loopback:
+                return False
+            if ip_obj.is_link_local:
+                return False
+            if ip_obj.is_multicast:
+                return False
+            if ip_obj.is_unspecified:  # 0.0.0.0 or ::
+                return False
+            if hasattr(ip_obj, "is_reserved") and ip_obj.is_reserved:
+                return False
+
+            # For IPv4, also check some additional ranges
+            if isinstance(ip_obj, ipaddress.IPv4Address):
+                # Check for special ranges not covered by is_private
+                ip_int = int(ip_obj)
+
+                # 0.0.0.0/8 - "This" network
+                if ip_int >> 24 == 0:
+                    return False
+
+                # 100.64.0.0/10 - Shared address space (CGN)
+                if ip_int >> 22 == 0x191:  # 100.64/10
+                    return False
+
+                # 198.18.0.0/15 - Benchmarking
+                if ip_int >> 17 == 0x18C9:  # 198.18/15
+                    return False
+
+                # 240.0.0.0/4 - Reserved (Class E)
+                if ip_int >> 28 == 0xF:
+                    return False
+
+            # If it passed all checks, it's global
+            return True
+
+        except (ValueError, AttributeError):
+            # Not a valid IP address
+            return False

tql/mutators/security.py

@@ -0,0 +1,225 @@
+"""Security-related mutators for defanging and refanging URLs and indicators."""
+
+from typing import Any, Dict
+
+from .base import BaseMutator, append_to_result
+
+
+class RefangMutator(BaseMutator):
+    """
+    Mutator that refangs (un-defangs) URLs and indicators.
+
+    This mutator reverses common defanging patterns to make URLs and
+    indicators clickable/active again. It handles various defanging patterns:
+    - hXXp:// -> http://
+    - hXXps:// -> https://
+    - [.]  -> .
+    - [.] -> .
+    - [:]  -> :
+    - [:] -> :
+    - fXp:// -> ftp://
+    - [at] -> @
+    - [@] -> @
+
+    Parameters:
+        field: Optional field to store the refanged value
+    """
+
+    def apply(self, field_name: str, record: Dict[str, Any], value: Any) -> Any:
+        """Apply the refang transformation."""
+        append_field = self.params.get("field")
+
+        # Handle different input types
+        refanged_value: Any
+        if value is None:
+            refanged_value = None
+        elif isinstance(value, str):
+            refanged_value = self._refang_string(value)
+        elif isinstance(value, list):
+            # Refang each string in the list
+            refanged_value = []
+            for item in value:
+                if isinstance(item, str):
+                    refanged_value.append(self._refang_string(item))
+                else:
+                    refanged_value.append(item)
+        elif isinstance(value, (int, float, bool)):
+            # Convert to string, refang, then return
+            refanged_value = self._refang_string(str(value))
+        else:
+            # For other types, return as-is
+            refanged_value = value
+
+        # If append_field is specified, add to record and return original value
+        if append_field:
+            append_to_result(record, append_field, refanged_value)
+            return value
+        else:
+            # Return the refanged value directly
+            return refanged_value
+
+    def _refang_string(self, s: str) -> str:
+        """Refang a single string."""
+        result = s
+
+        # Apply replacements in specific order to handle spaces properly
+        # First handle patterns with spaces
+        result = result.replace(" [.] ", ".")
+        result = result.replace(" [dot] ", ".")
+        result = result.replace(" [at] ", "@")
+        result = result.replace(" [:] ", ":")
+
+        # Protocol defanging (various cases)
+        result = result.replace("hxxp://", "http://")
+        result = result.replace("hXXp://", "http://")
+        result = result.replace("HxXp://", "http://")
+        result = result.replace("HxxP://", "http://")
+        result = result.replace("HXXP://", "http://")
+        result = result.replace("hxxps://", "https://")
+        result = result.replace("hXXps://", "https://")
+        result = result.replace("HXXPS://", "https://")
+        result = result.replace("fxp://", "ftp://")
+        result = result.replace("fXp://", "ftp://")
+        result = result.replace("FXP://", "ftp://")
+
+        # Dot defanging
+        result = result.replace("[.]", ".")
+        result = result.replace("(.)", ".")
+        result = result.replace("{.}", ".")
+        result = result.replace("[dot]", ".")
+        result = result.replace("(dot)", ".")
+        result = result.replace("{dot}", ".")
+
+        # Colon defanging
+        result = result.replace("[:]", ":")
+        result = result.replace("(:)", ":")
+        result = result.replace("{:}", ":")
+
+        # At symbol defanging
+        result = result.replace("[at]", "@")
+        result = result.replace("(at)", "@")
+        result = result.replace("{at}", "@")
+        result = result.replace("[@]", "@")
+        result = result.replace("(@)", "@")
+        result = result.replace("{@}", "@")
+
+        # Slash defanging
+        result = result.replace("[/]", "/")
+        result = result.replace("(/)", "/")
+        result = result.replace("{/}", "/")
+
+        return result
+
+
+class DefangMutator(BaseMutator):
+    """
+    Mutator that defangs URLs and indicators to make them unclickable.
+
+    This mutator applies common defanging patterns to URLs and indicators
+    to prevent accidental clicks or automatic processing:
+    - http:// -> hXXp://
+    - https:// -> hXXps://
+    - . -> [.]
+    - : -> [:]
+    - @ -> [at]
+    - ftp:// -> fXp://
+
+    Parameters:
+        field: Optional field to store the defanged value
+    """
+
+    def apply(self, field_name: str, record: Dict[str, Any], value: Any) -> Any:
+        """Apply the defang transformation."""
+        append_field = self.params.get("field")
+
+        # Handle different input types
+        defanged_value: Any
+        if value is None:
+            defanged_value = None
+        elif isinstance(value, str):
+            defanged_value = self._defang_string(value)
+        elif isinstance(value, list):
+            # Defang each string in the list
+            defanged_value = []
+            for item in value:
+                if isinstance(item, str):
+                    defanged_value.append(self._defang_string(item))
+                else:
+                    defanged_value.append(item)
+        elif isinstance(value, (int, float, bool)):
+            # Convert to string, defang, then return
+            defanged_value = self._defang_string(str(value))
+        else:
+            # For other types, return as-is
+            defanged_value = value
+
+        # If append_field is specified, add to record and return original value
+        if append_field:
+            append_to_result(record, append_field, defanged_value)
+            return value
+        else:
+            # Return the defanged value directly
+            return defanged_value
+
+    def _defang_string(self, s: str) -> str:
+        """Defang a single string."""
+        # Apply defanging patterns
+        result = s
+
+        # Protocol defanging (do these first to avoid double-defanging)
+        result = result.replace("https://", "hXXps://")
+        result = result.replace("http://", "hXXp://")
+        result = result.replace("ftp://", "fXp://")
+        result = result.replace("HTTPS://", "HXXPS://")
+        result = result.replace("HTTP://", "HXXP://")
+        result = result.replace("FTP://", "FXP://")
+
+        # Now defang dots, but not in the protocol part we just defanged
+        # Split by whitespace to handle individual tokens
+        tokens = result.split()
+        defanged_tokens = []
+
+        for token in tokens:
+            # Check if this is a URL (has protocol)
+            has_protocol = any(
+                token.startswith(p)
+                for p in [
+                    "hXXp://",
+                    "hXXps://",
+                    "fXp://",
+                    "HXXP://",
+                    "HXXPS://",
+                    "FXP://",
+                    "hxxp://",
+                    "hxxps://",
+                    "fxp://",  # Already defanged variations
+                ]
+            )
+
+            if has_protocol and "://" in token:
+                # For URLs, defang only the domain part
+                protocol, rest = token.split("://", 1)
+                # Only defang if not already defanged
+                if "[.]" not in rest and "[at]" not in rest:
+                    # Defang dots in domain/path
+                    rest = rest.replace(".", "[.]")
+                    # Defang @ if present (for URLs with auth)
+                    rest = rest.replace("@", "[at]")
+                    # Defang colons in port numbers
+                    # Only defang colon if it's followed by numbers (port)
+                    import re
+
+                    rest = re.sub(r":(\d+)", r"[:]\1", rest)
+                defanged_tokens.append(f"{protocol}://{rest}")
+            else:
+                # For non-URL tokens, defang dots and @ symbols
+                # But avoid double-defanging
+                if "[.]" not in token and "[at]" not in token:
+                    defanged = token.replace(".", "[.]")
+                    defanged = defanged.replace("@", "[at]")
+                    defanged_tokens.append(defanged)
+                else:
+                    # Already defanged, leave as-is
+                    defanged_tokens.append(token)
+
+        return " ".join(defanged_tokens)