tellaro-query-language 0.2.2__py3-none-any.whl → 0.2.3__py3-none-any.whl

{tellaro_query_language-0.2.2.dist-info → tellaro_query_language-0.2.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tellaro-query-language
-Version: 0.2.2
+Version: 0.2.3
 Summary: A flexible, human-friendly query language for searching and filtering structured data
 Home-page: https://github.com/tellaro/tellaro-query-language
 License: MIT

{tellaro_query_language-0.2.2.dist-info → tellaro_query_language-0.2.3.dist-info}/RECORD

@@ -1,25 +1,25 @@
-tql/__init__.py,sha256=mRrrtun-1Xx9k8g0aaiqxYhhNAwWEyRC4zrVMO49Kkg,1260
+tql/__init__.py,sha256=eqti5Fmu8EjD-NbCwqb1UKGJQ0OpWJLREsVMqqn6Hs4,1260
 tql/analyzer.py,sha256=Sfzj6f7YzqylT8HIL9hDbXdhl0lf8q8DNoafrxkD-F8,15456
 tql/cache/__init__.py,sha256=GIzIEMZUZEYJj72sAhuVLEG-OJEKUG2srUWNM3Ix-T8,213
-tql/cache/base.py,sha256=0b-8uyh3JltayGmXQI45snTqsM5sQu9u0KcNvZIRa-I,687
-tql/cache/memory.py,sha256=ibcmQSAxNvqCy6DksbU7gLu6UArYp1u3fW-oLubxtV0,2056
+tql/cache/base.py,sha256=CwLpobv4WR7WSz99JVWAHNn-XQTtqr38Yg5UiWNXPiA,3192
+tql/cache/memory.py,sha256=1kDfyODBCDi5UvSP_f_7UhHVmug-sLnrsMYvm4C4cxo,5978
 tql/cache/redis.py,sha256=ZU_IsVDvpSYpNvPfnZ4iulJDODpEGx3c4dkXLzPzPVc,2309
-tql/core.py,sha256=bMPrcuutY-1yvC-4M7w2y1JxNitMyBSpxPfg8ohjO60,48406
+tql/core.py,sha256=ZNtxNJSZqotJ-3z9kg8Nc2lKDoeR9Bspe8k7VrXp0Ow,49363
 tql/core_components/README.md,sha256=Rm7w4UHdQ0vPBEFybE5b62IOvSA5Nzq2GRvtBHOapmc,3068
 tql/core_components/__init__.py,sha256=v8BBybPlqV7dkVY9mw1mblvqyAFJZ7Pf_bEc-jAL7FI,643
 tql/core_components/file_operations.py,sha256=Jr0kkxz_OP2KHOAsIr7KMtYe_lbu8LuBUySt2LQbjJw,3925
-tql/core_components/opensearch_operations.py,sha256=KvmK1FnkGZFBjBysH_sDjzIRnyUcNn7wzLzuRr1rBlg,54264
+tql/core_components/opensearch_operations.py,sha256=zgxGiDpXyPW0ZUX-StpZXxf84s8eLxSymAGM5UUJimk,55253
 tql/core_components/stats_operations.py,sha256=aqTGAqIFvR6EkSbJEd0qft8Ldy8uiTrK2XI9o5bZUOs,8014
 tql/core_components/validation_operations.py,sha256=_VPXh0HABBjsXF99jFT7B6-5QAPsADOCy6poinGrxeE,22454
-tql/evaluator.py,sha256=_JYr-wK3F1wvBoNGIBiAEaP6Ot1g2qxZ4lOjPdOqvDk,17698
+tql/evaluator.py,sha256=W2PbD0umxKORlb9npPgg985I7eYxc9QsXUtiZsgOPCk,17889
 tql/evaluator_components/README.md,sha256=c59yf2au34yPhrru7JWgGop_ORteB6w5vfMhsac8j3k,3882
 tql/evaluator_components/__init__.py,sha256=DourRUSYXWPnCghBFj7W0YfMeymT3X8YTDCwnLIyP1c,535
 tql/evaluator_components/field_access.py,sha256=BuXvL9jlv4H77neT70Vh7_qokmzs-d4EbSDA2FB1IT0,6435
 tql/evaluator_components/special_expressions.py,sha256=K6M5pW4Re2kEqxfxj9sc7I_M1tU3pn6LKJ2AfjHeciA,12917
-tql/evaluator_components/value_comparison.py,sha256=pL7-hxdNbzJ53DrTSiDdd7KYbVLChuNwFRLjG7P_1KM,17939
+tql/evaluator_components/value_comparison.py,sha256=Woo8bNwIARdfe7F7ApYD5xi4NrrPwhdem_5ZKSik-t4,21369
 tql/exceptions.py,sha256=hatIixXci6p57J9RrkfdvmKM_2i-JKb8ViL2kU4z7a8,5550
 tql/geoip_normalizer.py,sha256=tvie-5xevJEeLp2KmjoXDjYdND8AvyVE7lCO8qgUzGY,10486
-tql/mutator_analyzer.py,sha256=OzI7t3C4H0IJOonpywE5LWz2cm5Dco5xnp2RTQOiSWg,55638
+tql/mutator_analyzer.py,sha256=OWx3k5lK5aFHWU9Ez6DaIhenEZDxj9CbB0vM71xqUTw,55670
 tql/mutators/__init__.py,sha256=eTK8sRw4KXXnTZTn5ETIqwcaIek5rSUIVyZsxTwNNHA,6966
 tql/mutators/base.py,sha256=4Ze_x1sTO11OILXfcF2XN7ttyHcZ4gwn96UXFMMaC6M,2523
 tql/mutators/dns.py,sha256=1IKgHolFLRMR4TOgK0AiLjz5vDtFiqO328mVF4Vzk3s,14428
@@ -36,21 +36,21 @@ tql/opensearch_components/field_mapping.py,sha256=fj388cKVyDXLJKi8giSiGHL9zg4cFR
 tql/opensearch_components/lucene_converter.py,sha256=OvYTZHNBktPGow1fsVm4TMlvxHSmWrnqo42lFZNxXTo,13175
 tql/opensearch_components/query_converter.py,sha256=vLoBqv7W3ntqUH6hcuT4PDJkGkAGSQCxMvAWC482c0g,41971
 tql/opensearch_mappings.py,sha256=sVLlQlE3eGD7iNNZ_m4F4j5GVzQAJhZyCqDKYRhLRh8,11531
-tql/opensearch_stats.py,sha256=aMV__jtlfogGBnFucsNPazORro2mYTz_C_w9uxOqsMI,24384
-tql/parser.py,sha256=9kewX4IbBL3W5hbq9Xhi4BGrQ4QaoWqz9AJV0Yuf9YA,78665
+tql/opensearch_stats.py,sha256=l1VsHp1hFzsz8VFFD42M4xwlVgKkjptLs8TFB0wqqRw,24478
+tql/parser.py,sha256=t1bpL1hrHVpLZKeEP_DxMszlYUbh7QiudFDYxZtlc5s,80286
 tql/parser_components/README.md,sha256=lvQX72ckq2zyotGs8QIHHCIFqaA7bOHwkP44wU8Zoiw,2322
 tql/parser_components/__init__.py,sha256=zBwHBMPJyHSBbaOojf6qTrJYjJg5A6tPUE8nHFdRiQs,521
 tql/parser_components/ast_builder.py,sha256=erHoeKAMzobswoRIXB9xcsZbzQ5-2ZwaYfQgRWoUAa8,9653
 tql/parser_components/error_analyzer.py,sha256=qlCD9vKyW73aeKQYI33P1OjIWSJ3LPd08wuN9cis2fU,4012
 tql/parser_components/field_extractor.py,sha256=eUEkmiYWX2OexanFqhHeX8hcIkRlfIcgMB667e0HRYs,4629
 tql/parser_components/grammar.py,sha256=h58RBshZHXgbP1EmNwmf7dny-fgVloNg-qN4Rivross,20599
-tql/post_processor.py,sha256=MZOJzuWTL2qdvu-AUNMryYF2D-piv8rYH5vCcrLt5-A,50069
-tql/scripts.py,sha256=VOr5vCjIvKlW36kwvJx7JGFIRM16IJZlbJcWlBexBtk,3728
-tql/stats_evaluator.py,sha256=OQZuNLwLHAtWrwAh3utdtr1fQt3tftCs6L-1G1NQCGQ,22318
+tql/post_processor.py,sha256=hItSj1VaQwCf2rbRQzLaMfAWR5RqMTpvDP-_X0HVhVo,51697
+tql/scripts.py,sha256=2iryuAWqIxYoFA7R5hIzC6NE72ihTH9kIZd1-thKPTM,4331
+tql/stats_evaluator.py,sha256=xJoTaBCBiKzQ0HuITGhTA41dVpeDhIMU9EMguCh_VG0,22427
 tql/stats_transformer.py,sha256=MT-4rDWZSySgn4Fuq9H0c-mvwFYLM6FqWpPv2rHX-rE,7588
 tql/validators.py,sha256=e9MlX-zQ_O3M8YP8vXyMjKU8iiJMTh6mMK0iv0_4gTY,3771
-tellaro_query_language-0.2.2.dist-info/LICENSE,sha256=zRhQ85LnW55fWgAjQctckwQ67DX5Jmt64lq343ThZFU,1063
-tellaro_query_language-0.2.2.dist-info/METADATA,sha256=QsCXKY_0aHeMorc4PepJ84ViZbTK53suxILmr868Lkk,15740
-tellaro_query_language-0.2.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-tellaro_query_language-0.2.2.dist-info/entry_points.txt,sha256=H43APfGBMsZkKsUCnFTaqprQPW-Kce2yz2qsBL3dZrw,164
-tellaro_query_language-0.2.2.dist-info/RECORD,,
+tellaro_query_language-0.2.3.dist-info/LICENSE,sha256=zRhQ85LnW55fWgAjQctckwQ67DX5Jmt64lq343ThZFU,1063
+tellaro_query_language-0.2.3.dist-info/METADATA,sha256=jRsW9n1nhyQb7m0qZd4F7h-q_ZoH_WzfVcCJMzC9P3c,15740
+tellaro_query_language-0.2.3.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+tellaro_query_language-0.2.3.dist-info/entry_points.txt,sha256=H43APfGBMsZkKsUCnFTaqprQPW-Kce2yz2qsBL3dZrw,164
+tellaro_query_language-0.2.3.dist-info/RECORD,,

tql/__init__.py

@@ -28,7 +28,7 @@ from .opensearch_mappings import (
     get_sample_data_from_index,
 )
 
-__version__ = "0.1.0"
+__version__ = "0.2.2"
 __all__ = [
     "TQL",
     "TQLParseError",

tql/cache/base.py

@@ -1,25 +1,97 @@
-"""Base cache infrastructure."""
+"""Base cache infrastructure for TQL.
+
+This module provides the base CacheManager class that defines the caching
+interface used throughout TQL. Concrete implementations include LocalCacheManager
+for in-memory caching and RedisCacheManager for distributed caching.
+"""
 
 from typing import Any, Dict, Optional
 
 
 class CacheManager:
-    """Base class for cache management."""
+    """Base class for cache management.
+
+    This class defines the interface for all cache implementations in TQL.
+    Subclasses should override these methods to provide actual caching functionality.
+
+    The base implementation provides no-op defaults that can be safely used when
+    caching is disabled or not needed.
+
+    Example:
+        >>> cache = LocalCacheManager()
+        >>> cache.set("user:123", {"name": "Alice", "age": 30}, ttl=3600)
+        >>> user = cache.get("user:123")
+        >>> cache.delete("user:123")
+    """
 
     def get(self, key: str) -> Optional[Any]:
-        """Retrieve value from cache."""
+        """Retrieve a value from the cache.
+
+        Args:
+            key: The cache key to look up. Should be a string identifier.
+
+        Returns:
+            The cached value if it exists and hasn't expired, None otherwise.
+
+        Example:
+            >>> value = cache.get("my_key")
+            >>> if value is not None:
+            ...     print(f"Found: {value}")
+        """
         return None
 
     def set(self, key: str, value: Any, ttl: Optional[int] = None) -> None:
-        """Store value in cache."""
+        """Store a value in the cache.
+
+        Args:
+            key: The cache key under which to store the value.
+            value: The value to cache. Can be any Python object.
+            ttl: Time-to-live in seconds. If None or 0, the value never expires.
+
+        Example:
+            >>> cache.set("config", {"debug": True}, ttl=300)  # Cache for 5 minutes
+            >>> cache.set("permanent", {"version": "1.0"})  # Never expires
+        """
 
     def delete(self, key: str) -> None:
-        """Remove value from cache."""
+        """Remove a value from the cache.
+
+        Args:
+            key: The cache key to delete.
+
+        Example:
+            >>> cache.delete("expired_key")
+        """
 
     def clear_pattern(self, pattern: str) -> int:  # pylint: disable=unused-argument
-        """Clear all keys matching pattern."""
+        """Clear all keys matching a pattern.
+
+        Args:
+            pattern: A pattern string to match keys. Format depends on implementation.
+                    For Redis: supports wildcards like "user:*" or "session:?123"
+                    For Local: basic string matching
+
+        Returns:
+            The number of keys that were deleted.
+
+        Example:
+            >>> count = cache.clear_pattern("temp:*")
+            >>> print(f"Cleared {count} temporary keys")
+        """
         return 0
 
     def get_stats(self) -> Dict[str, Any]:
-        """Get cache statistics."""
+        """Get cache statistics and metrics.
+
+        Returns:
+            Dictionary containing cache statistics such as:
+            - hit_rate: Cache hit rate percentage
+            - miss_rate: Cache miss rate percentage
+            - size: Number of items in cache
+            - memory_usage: Memory used by cache (if available)
+
+        Example:
+            >>> stats = cache.get_stats()
+            >>> print(f"Hit rate: {stats.get('hit_rate', 0)}%")
+        """
         return {}

tql/cache/memory.py

@@ -1,63 +1,171 @@
-"""In-memory cache implementation."""
+"""In-memory cache implementation for TQL.
+
+This module provides a simple in-memory cache with TTL (time-to-live) support
+and basic LRU (Least Recently Used) eviction when the cache reaches its size limit.
+"""
 
 import time
-from typing import Any, Dict, Optional, Tuple
+from typing import Any, Dict, Optional
 
 from .base import CacheManager
 
 
 class LocalCacheManager(CacheManager):
-    """Local in-memory cache using LRU."""
+    """Local in-memory cache with TTL and LRU eviction.
+
+    This implementation provides thread-safe in-memory caching suitable for
+    single-process applications. For distributed caching across multiple
+    processes or servers, use RedisCacheManager instead.
+
+    Features:
+        - TTL-based expiration
+        - LRU eviction when cache is full
+        - Hit/miss statistics tracking
+        - Pattern-based key clearing
+
+    Args:
+        max_size: Maximum number of items to store (default: 10000)
+        default_ttl: Default time-to-live in seconds (default: 3600 = 1 hour)
+
+    Example:
+        >>> cache = LocalCacheManager(max_size=1000, default_ttl=600)
+        >>> cache.set("user:123", {"name": "Alice"}, ttl=300)
+        >>> user = cache.get("user:123")
+        >>> stats = cache.get_stats()
+        >>> print(f"Hit rate: {stats['hit_rate']:.2%}")
+
+    Attributes:
+        max_size: Maximum cache size
+        default_ttl: Default TTL for cached items
+    """
 
     def __init__(self, max_size: int = 10000, default_ttl: int = 3600):
+        """Initialize the local cache.
+
+        Args:
+            max_size: Maximum number of items to cache before eviction starts.
+            default_ttl: Default expiration time in seconds for cached items.
+        """
         self.max_size = max_size
         self.default_ttl = default_ttl
-        self._cache: Dict[str, Tuple[Any, float]] = {}
+        self._cache: Dict[str, Any] = {}
+        self._expiry: Dict[str, float] = {}
         self._hits = 0
         self._misses = 0
 
     def get(self, key: str) -> Optional[Any]:
-        """Retrieve value from cache if not expired."""
+        """Retrieve value from cache if not expired.
+
+        Args:
+            key: The cache key to retrieve.
+
+        Returns:
+            The cached value if present and not expired, None otherwise.
+
+        Note:
+            This method automatically removes expired keys when accessed.
+            Hit/miss statistics are updated on each call.
+        """
         if key in self._cache:
-            value, expiry = self._cache[key]
-            if expiry > time.time():
+            expiry = self._expiry.get(key, float('inf'))
+            if expiry == 0 or expiry > time.time():
                 self._hits += 1
-                return value
+                return self._cache[key]
             else:
-                # Expired
+                # Expired - clean up
                 del self._cache[key]
+                del self._expiry[key]
         self._misses += 1
         return None
 
     def set(self, key: str, value: Any, ttl: Optional[int] = None) -> None:
-        """Store value in cache with TTL."""
-        if len(self._cache) >= self.max_size:
-            # Simple eviction: remove oldest
+        """Store value in cache with optional TTL.
+
+        Args:
+            key: The cache key under which to store the value.
+            value: The value to cache (any Python object).
+            ttl: Time-to-live in seconds. If None, uses default_ttl.
+                 If 0, the item never expires.
+
+        Note:
+            When the cache is full (reaches max_size), the oldest item
+            is evicted to make room for the new one (LRU eviction).
+        """
+        if len(self._cache) >= self.max_size and key not in self._cache:
+            # Simple eviction: remove oldest (first in dict)
             oldest_key = next(iter(self._cache))
             del self._cache[oldest_key]
+            self._expiry.pop(oldest_key, None)
 
-        expiry = time.time() + (ttl or self.default_ttl)
-        self._cache[key] = (value, expiry)
+        self._cache[key] = value
+        if ttl == 0:
+            # Never expires
+            self._expiry[key] = 0
+        else:
+            expiry_time = time.time() + (ttl if ttl is not None else self.default_ttl)
+            self._expiry[key] = expiry_time
 
     def delete(self, key: str) -> None:
-        """Remove value from cache."""
+        """Remove value from cache.
+
+        Args:
+            key: The cache key to delete.
+
+        Note:
+            If the key doesn't exist, this method does nothing (no error raised).
+        """
         self._cache.pop(key, None)
+        self._expiry.pop(key, None)
 
     def clear_pattern(self, pattern: str) -> int:
-        """Clear all keys matching pattern."""
+        """Clear all keys matching a glob pattern.
+
+        Args:
+            pattern: A glob pattern to match keys. Supports wildcards:
+                    - '*' matches any sequence of characters
+                    - '?' matches any single character
+                    - '[seq]' matches any character in seq
+                    - '[!seq]' matches any character not in seq
+
+        Returns:
+            The number of keys that were deleted.
+
+        Example:
+            >>> cache.set("user:123", data1)
+            >>> cache.set("user:456", data2)
+            >>> cache.set("session:789", data3)
+            >>> count = cache.clear_pattern("user:*")  # Deletes user:123 and user:456
+            >>> print(count)  # 2
+        """
         import fnmatch
 
         keys_to_delete = [k for k in self._cache.keys() if fnmatch.fnmatch(k, pattern)]
         for key in keys_to_delete:
             del self._cache[key]
+            self._expiry.pop(key, None)
         return len(keys_to_delete)
 
     def get_stats(self) -> Dict[str, Any]:
-        """Get cache statistics."""
+        """Get cache performance statistics.
+
+        Returns:
+            Dictionary containing:
+            - hits: Number of successful cache retrievals
+            - misses: Number of cache misses
+            - hit_rate: Ratio of hits to total requests (0.0 to 1.0)
+            - size: Current number of items in cache
+            - max_size: Maximum cache capacity
+
+        Example:
+            >>> stats = cache.get_stats()
+            >>> print(f"Cache is {stats['hit_rate']:.2%} effective")
+            >>> print(f"Using {stats['size']}/{stats['max_size']} slots")
+        """
+        total_requests = self._hits + self._misses
         return {
             "hits": self._hits,
             "misses": self._misses,
-            "hit_rate": self._hits / (self._hits + self._misses) if (self._hits + self._misses) > 0 else 0,
+            "hit_rate": self._hits / total_requests if total_requests > 0 else 0.0,
             "size": len(self._cache),
             "max_size": self.max_size,
         }

tql/core.py

@@ -100,8 +100,25 @@ class TQL:
                     # This is an OpenSearch-style mapping, map field to itself
                     self._simple_mappings[k] = k
                 else:
-                    # Extract the first key as the simple mapping
-                    self._simple_mappings[k] = next(iter(v.keys()))
+                    # Intelligent field mapping extraction for complex mappings
+                    # Priority: 1) Key matching field name, 2) Key without dots (primary field), 3) First key
+
+                    if k in v:
+                        # Field name exists as key in mapping (e.g., {"username": {"username": "keyword", ...}})
+                        self._simple_mappings[k] = k
+                    else:
+                        # Find primary field (keys without dots, not starting with underscore)
+                        primary_fields = [
+                            field_key for field_key in v.keys()
+                            if '.' not in field_key and not field_key.startswith('_')
+                        ]
+
+                        if primary_fields:
+                            # Use first primary field
+                            self._simple_mappings[k] = primary_fields[0]
+                        else:
+                            # Fallback to first key (maintain backward compatibility)
+                            self._simple_mappings[k] = next(iter(v.keys()))
             else:
                 # Default to mapping field to itself
                 self._simple_mappings[k] = k

tql/core_components/opensearch_operations.py

@@ -239,7 +239,7 @@ class OpenSearchOperations:
             analysis_result = self.analyze_opensearch_query(query)
             has_mutators = isinstance(analysis_result, MutatorAnalysisResult)
             needs_post_processing_for_stats = (
-                has_mutators and bool(analysis_result.post_processing_requirements) if has_mutators else False
+                has_mutators and bool(analysis_result.post_processing_requirements) if has_mutators else False  # type: ignore[union-attr]
             )
 
             # Handle stats queries differently
@@ -258,7 +258,7 @@ class OpenSearchOperations:
                 if filter_ast:
                     # Use the optimized AST if we have mutators
                     if has_mutators and needs_post_processing_for_stats:
-                        filter_query = backend.convert(analysis_result.optimized_ast.get("filter", filter_ast))["query"]
+                        filter_query = backend.convert(analysis_result.optimized_ast.get("filter", filter_ast))["query"]  # type: ignore[union-attr]
                     else:
                         filter_query = backend.convert(filter_ast)["query"]
                 else:
@@ -529,6 +529,8 @@ class OpenSearchOperations:
                 stats_evaluator = TQLStatsEvaluator()
 
                 # Execute the stats aggregation in memory
+                if stats_ast_for_post_processing is None:
+                    raise ValueError("Stats AST is None but phase2 processing was requested")
                 stats_results = stats_evaluator.evaluate_stats(filtered_docs, stats_ast_for_post_processing, {})
 
                 # Format response for stats-only (no documents)
@@ -547,7 +549,7 @@ class OpenSearchOperations:
                     "performance_impact": {
                         "overhead_ms": 0,  # Would need timing to calculate
                         "documents_processed": len(all_documents),
-                        "mutators_applied": len(analysis_result.post_processing_requirements) if has_mutators else 0,
+                        "mutators_applied": len(analysis_result.post_processing_requirements) if has_mutators else 0,  # type: ignore[union-attr]
                     },
                     "opensearch_query": complete_opensearch_query,
                 }
@@ -580,6 +582,8 @@ class OpenSearchOperations:
                 translator = OpenSearchStatsTranslator()
 
                 # Transform the response using the translator
+                if stats_ast is None:
+                    raise ValueError("Stats AST is None but grouping was detected")
                 transformed_response = translator.transform_response(response, stats_ast)
 
                 # The transformed response already has the correct structure
@@ -925,6 +929,21 @@ class OpenSearchOperations:
         # Get opensearch total before filtering
         opensearch_total = total_hits
 
+        # Track optimization features used in this query
+        optimizations_applied = []
+        if scan_all:
+            optimizations_applied.append("scroll_api")
+        if needs_phase2 and pagination_stats and pagination_stats.get("pages_checked", 0) > 1:
+            optimizations_applied.append("auto_pagination")
+        if request_cache:
+            optimizations_applied.append("request_cache")
+        if preference:
+            optimizations_applied.append("preference_routing")
+        if routing:
+            optimizations_applied.append("custom_routing")
+        if terminate_after:
+            optimizations_applied.append("early_termination")
+
         result = {
             "results": results,
             "total": len(results),
@@ -934,7 +953,7 @@ class OpenSearchOperations:
             "health_status": health_status,
             "health_reasons": health_reasons,
             "performance_impact": performance_impact,
-            "optimizations_applied": [],  # TODO: Track actual optimizations  # noqa: W0511
+            "optimizations_applied": optimizations_applied,
             "opensearch_query": (
                 complete_opensearch_query if "complete_opensearch_query" in locals() else {}
             ),  # Include the full query body

tql/evaluator.py

@@ -350,6 +350,8 @@ class TQLEvaluator:
                 return left_missing or right_missing
             elif node_type == "unary_op":
                 # Don't recurse through NOT operators - they handle missing fields themselves
+                # The NOT operator has special logic at lines 213-254 that handles missing fields correctly
+                # Recursing here would cause double-handling and incorrect results
                 return False
             elif node_type == "collection_op":
                 field_name = node["field"]

tql/evaluator_components/value_comparison.py

@@ -6,6 +6,7 @@ operator implementations, and special cases like CIDR matching.
 
 import ipaddress
 import re
+from functools import lru_cache
 from typing import Any
 
 
@@ -15,6 +16,23 @@ class ValueComparator:
     # Sentinel value to distinguish missing fields from None values
     _MISSING_FIELD = object()
 
+    @staticmethod
+    @lru_cache(maxsize=256)
+    def _compile_regex(pattern: str) -> re.Pattern:
+        """Compile and cache regex patterns for performance.
+
+        Args:
+            pattern: Regex pattern string
+
+        Returns:
+            Compiled regex pattern
+
+        Note:
+            Uses LRU cache with max 256 patterns. This significantly improves
+            performance when the same regex patterns are used repeatedly in queries.
+        """
+        return re.compile(pattern)
+
     def compare_values(self, field_value: Any, operator: str, expected_value: Any) -> bool:  # noqa: C901
         """Compare a field value against an expected value using the given operator.
 
@@ -49,9 +67,17 @@ class ValueComparator:
                 return False
 
         # Handle None field values (field exists but is None)
+        # IMPORTANT: None is a valid value, distinct from missing fields.
+        # For 'exists' operator: This code path should NOT be reached because 'exists'
+        # checks field presence in the record, not the value. The evaluator handles
+        # 'exists' before calling compare_values. If we reach here with None, it means
+        # the field exists but has None value, which should NOT match 'exists'.
         if field_value is None:
             if operator in ["exists"]:
-                return True  # Field exists, even if value is None
+                # Field key exists in record but value is None
+                # Semantics: 'exists' means "field has a non-null value"
+                # This matches database behavior where NULL != EXISTS
+                return False  # None value does not satisfy 'exists'
             elif operator in ["is"]:
                 # Check for null comparison - expected_value can be None or "null"
                 return expected_value is None or (isinstance(expected_value, str) and expected_value.lower() == "null")
@@ -59,6 +85,10 @@ class ValueComparator:
                 return False
 
         # Convert numeric strings to numbers for comparison
+        # IMPORTANT: Store original values to check if conversion succeeded
+        field_value_original = field_value
+        expected_value_original = expected_value
+
         field_value = self._convert_numeric(field_value)
         expected_value = self._convert_numeric(expected_value)
 
@@ -68,6 +98,20 @@ class ValueComparator:
         if isinstance(field_value, str) and field_value.lower() in ["true", "false"]:
             field_value = field_value.lower() == "true"
 
+        # Type compatibility check for numeric operators
+        # If operator requires numeric comparison, both values must be numeric
+        # Exception: Arrays are handled specially in the operator logic below
+        if operator in ["gt", "gte", "lt", "lte", ">", ">=", "<", "<="]:
+            # Skip check if field_value is an array - handled by array logic below
+            if not isinstance(field_value, (list, tuple)):
+                field_is_numeric = isinstance(field_value, (int, float)) and not isinstance(field_value, bool)
+                expected_is_numeric = isinstance(expected_value, (int, float)) and not isinstance(expected_value, bool)
+
+                if not (field_is_numeric and expected_is_numeric):
+                    # At least one value failed numeric conversion
+                    # Cannot perform numeric comparison - return False
+                    return False
+
         try:
             if operator in ["eq", "="]:
                 # Handle array fields - check if ANY element equals expected value
@@ -104,27 +148,30 @@ class ValueComparator:
                 if isinstance(expected_value, list) and len(expected_value) == 1:
                     expected_value = expected_value[0]
                 # Handle list fields by checking if ANY element contains the expected value
+                # Case-insensitive comparison to match post-processor behavior
                 if isinstance(field_value, list):
                     # For arrays, check if ANY element contains the expected value
-                    return any(str(expected_value) in str(elem) for elem in field_value)
+                    return any(str(expected_value).lower() in str(elem).lower() for elem in field_value)
                 else:
-                    return str(expected_value) in str(field_value)
+                    return str(expected_value).lower() in str(field_value).lower()
             elif operator == "startswith":
                 # Unwrap single-element lists for string operators
                 if isinstance(expected_value, list) and len(expected_value) == 1:
                     expected_value = expected_value[0]
                 # Handle array fields - check if ANY element starts with expected value
+                # Case-insensitive comparison to match post-processor behavior
                 if isinstance(field_value, (list, tuple)):
-                    return any(str(elem).startswith(str(expected_value)) for elem in field_value)
-                return str(field_value).startswith(str(expected_value))
+                    return any(str(elem).lower().startswith(str(expected_value).lower()) for elem in field_value)
+                return str(field_value).lower().startswith(str(expected_value).lower())
             elif operator == "endswith":
                 # Unwrap single-element lists for string operators
                 if isinstance(expected_value, list) and len(expected_value) == 1:
                     expected_value = expected_value[0]
                 # Handle array fields - check if ANY element ends with expected value
+                # Case-insensitive comparison to match post-processor behavior
                 if isinstance(field_value, (list, tuple)):
-                    return any(str(elem).endswith(str(expected_value)) for elem in field_value)
-                return str(field_value).endswith(str(expected_value))
+                    return any(str(elem).lower().endswith(str(expected_value).lower()) for elem in field_value)
+                return str(field_value).lower().endswith(str(expected_value).lower())
             elif operator == "in":
                 if isinstance(expected_value, list):
                     if len(expected_value) == 1 and isinstance(field_value, list):
@@ -143,7 +190,13 @@ class ValueComparator:
                 # Unwrap single-element lists for string operators
                 if isinstance(expected_value, list) and len(expected_value) == 1:
                     expected_value = expected_value[0]
-                return bool(re.search(str(expected_value), str(field_value)))
+                # Use cached regex compilation for performance
+                try:
+                    pattern = self._compile_regex(str(expected_value))
+                    return bool(pattern.search(str(field_value)))
+                except (re.error, TypeError):
+                    # Invalid regex pattern, fall back to no match
+                    return False
             elif operator == "cidr":
                 # Unwrap single-element lists for CIDR
                 if isinstance(expected_value, list) and len(expected_value) == 1:
@@ -194,22 +247,31 @@ class ValueComparator:
                 # Unwrap single-element lists for string operators
                 if isinstance(expected_value, list) and len(expected_value) == 1:
                     expected_value = expected_value[0]
-                return str(expected_value) not in str(field_value)
+                # Case-insensitive comparison to match post-processor behavior
+                return str(expected_value).lower() not in str(field_value).lower()
             elif operator == "not_startswith":
                 # Unwrap single-element lists for string operators
                 if isinstance(expected_value, list) and len(expected_value) == 1:
                     expected_value = expected_value[0]
-                return not str(field_value).startswith(str(expected_value))
+                # Case-insensitive comparison to match post-processor behavior
+                return not str(field_value).lower().startswith(str(expected_value).lower())
             elif operator == "not_endswith":
                 # Unwrap single-element lists for string operators
                 if isinstance(expected_value, list) and len(expected_value) == 1:
                     expected_value = expected_value[0]
-                return not str(field_value).endswith(str(expected_value))
+                # Case-insensitive comparison to match post-processor behavior
+                return not str(field_value).lower().endswith(str(expected_value).lower())
             elif operator == "not_regexp":
                 # Unwrap single-element lists for string operators
                 if isinstance(expected_value, list) and len(expected_value) == 1:
                     expected_value = expected_value[0]
-                return not bool(re.search(str(expected_value), str(field_value)))
+                # Use cached regex compilation for performance
+                try:
+                    pattern = self._compile_regex(str(expected_value))
+                    return not bool(pattern.search(str(field_value)))
+                except (re.error, TypeError):
+                    # Invalid regex pattern, fall back to match (not regexp succeeds)
+                    return True
             elif operator == "not_cidr":
                 # Unwrap single-element lists for CIDR
                 if isinstance(expected_value, list) and len(expected_value) == 1:

tql/mutator_analyzer.py

@@ -491,7 +491,7 @@ class MutatorAnalyzer:
                     for param_name, param_value in geo_params.items():
                         mutator_params.append([param_name, param_value])
 
-                    geo_mutator = {"name": "geoip_lookup"}
+                    geo_mutator: Dict[str, Any] = {"name": "geoip_lookup"}
                     if mutator_params:
                         geo_mutator["params"] = mutator_params
 
@@ -539,7 +539,7 @@ class MutatorAnalyzer:
                     for param_name, param_value in nslookup_params.items():
                         mutator_params.append([param_name, param_value])
 
-                    nslookup_mutator = {"name": "nslookup"}
+                    nslookup_mutator: Dict[str, Any] = {"name": "nslookup"}
                     if mutator_params:
                         nslookup_mutator["params"] = mutator_params
 

tql/opensearch_stats.py

@@ -449,7 +449,7 @@ class OpenSearchStatsTranslator:
         aggregations: List[Dict[str, Any]],
         normalized_fields: List[Dict[str, Any]],
         level: int,
-    ) -> Optional[Dict[str, Any]]:
+    ) -> Union[Dict[str, Any], List[Dict[str, Any]], None]:
         """Transform a bucket recursively for multi-level grouping.
 
         Args:
@@ -502,6 +502,8 @@ class OpenSearchStatsTranslator:
                     result["aggregations"][agg_key] = value
         else:
             # Handle nested buckets
+            if next_level_field is None:
+                return None
             sub_buckets = bucket[next_level_field].get("buckets", [])
             sub_results = []
             print(

tql/parser.py

@@ -24,6 +24,9 @@ class TQLParser:
     evaluated against data or converted to backend-specific query formats.
     """
 
+    # Maximum query depth to prevent stack overflow and DoS attacks
+    MAX_QUERY_DEPTH = 50
+
     def __init__(self):
         """Initialize the parser with TQL grammar."""
         self.grammar = TQLGrammar()
@@ -53,7 +56,8 @@ class TQLParser:
             parsed_result = self.grammar.tql_expr.parseString(query, parseAll=True)
 
             # Convert to our AST format
-            return self._build_ast(parsed_result.asList()[0])
+            # Start depth counting at 0 from parse() entry point
+            return self._build_ast(parsed_result.asList()[0], depth=0)
 
         except ParseException as e:
             # Extract position and context from pyparsing exception
@@ -114,15 +118,29 @@ class TQLParser:
         # Extract fields using the field extractor
         return self.field_extractor.extract_fields(ast)
 
-    def _build_ast(self, parsed: Any) -> Dict[str, Any]:  # noqa: C901
+    def _build_ast(self, parsed: Any, depth: int = 0) -> Dict[str, Any]:  # noqa: C901
         """Build AST from parsed pyparsing result.
 
         Args:
             parsed: The parsed result from pyparsing
+            depth: Current recursion depth (for DoS prevention)
 
         Returns:
             Dictionary representing the AST node
+
+        Raises:
+            TQLSyntaxError: If query depth exceeds maximum allowed depth
         """
+        # Check depth limit to prevent stack overflow and DoS attacks
+        if depth > self.MAX_QUERY_DEPTH:
+            raise TQLSyntaxError(
+                f"Query depth exceeds maximum allowed depth of {self.MAX_QUERY_DEPTH}. "
+                "Please simplify your query to reduce nesting.",
+                position=0,
+                query="",
+                suggestions=["Reduce query nesting depth", "Split into multiple simpler queries"]
+            )
+
         if isinstance(parsed, list):
             if len(parsed) == 1:
                 # Single item, check if it's a field with is_private/is_global mutator
@@ -162,7 +180,7 @@ class TQLParser:
                             }
                             return result
                 # Single item, unwrap it
-                return self._build_ast(parsed[0])
+                return self._build_ast(parsed[0], depth + 1)
             elif len(parsed) >= 2 and isinstance(parsed[0], str) and parsed[0].lower() == "stats":
                 # This is a stats expression without filter (applies to all records)
                 return self._build_stats_ast(parsed)
@@ -210,7 +228,7 @@ class TQLParser:
                 # Check for NOT operator first (before field | mutator check)
                 elif isinstance(first, str) and (first.lower() == "not" or first == "!"):
                     # Unary logical operator (NOT or !)
-                    return {"type": "unary_op", "operator": "not", "operand": self._build_ast(second)}
+                    return {"type": "unary_op", "operator": "not", "operand": self._build_ast(second, depth + 1)}
 
                 # Check for field | mutator without operator
                 # This happens when we have a field with mutator(s) as the last element
@@ -267,12 +285,12 @@ class TQLParser:
                         # This is filter | stats
                         return {
                             "type": "query_with_stats",
-                            "filter": self._build_ast(first),
+                            "filter": self._build_ast(first, depth + 1),
                             "stats": self._build_stats_ast(second),
                         }
                 else:
                     # Fallback to treating as unary logical operator
-                    return {"type": "unary_op", "operator": first.lower(), "operand": self._build_ast(second)}
+                    return {"type": "unary_op", "operator": first.lower(), "operand": self._build_ast(second, depth + 1)}
             elif len(parsed) >= 3:
                 # Check if this is a field with multiple mutators
                 if isinstance(parsed[0], str) and all(
@@ -419,7 +437,7 @@ class TQLParser:
                             "field": field_name,
                             "type_hint": type_hint,
                             "field_mutators": field_mutators,
-                            "conditions": self._build_ast(conditions) if conditions else None,
+                            "conditions": self._build_ast(conditions, depth + 1) if conditions else None,
                         }
 
                         # Add geo parameters if any
@@ -497,7 +515,7 @@ class TQLParser:
                             "field": field_name,
                             "type_hint": type_hint,
                             "field_mutators": field_mutators,
-                            "conditions": self._build_ast(conditions) if conditions else None,
+                            "conditions": self._build_ast(conditions, depth + 1) if conditions else None,
                         }
 
                         # Add nslookup parameters if any
@@ -638,7 +656,7 @@ class TQLParser:
                             "field": field_name,
                             "type_hint": type_hint,
                             "field_mutators": field_mutators,
-                            "conditions": self._build_ast(conditions) if conditions else None,
+                            "conditions": self._build_ast(conditions, depth + 1) if conditions else None,
                         }
 
                         # Add geo parameters if any
@@ -715,7 +733,7 @@ class TQLParser:
                             "field": field_name,
                             "type_hint": type_hint,
                             "field_mutators": field_mutators,
-                            "conditions": self._build_ast(conditions) if conditions else None,
+                            "conditions": self._build_ast(conditions, depth + 1) if conditions else None,
                         }
 
                         # Add nslookup parameters if any
@@ -725,7 +743,7 @@ class TQLParser:
                         return result
                     else:
                         # This is a chained operation, not a between operation
-                        return self._build_chained_ast(parsed)
+                        return self._build_chained_ast(parsed, depth + 1)
 
             elif len(parsed) == 6:
                 # Check for "field not between value1 and value2" or "field ! between value1 and value2"
@@ -814,7 +832,7 @@ class TQLParser:
                             "field": field_name,
                             "type_hint": type_hint,
                             "field_mutators": field_mutators,
-                            "conditions": self._build_ast(conditions) if conditions else None,
+                            "conditions": self._build_ast(conditions, depth + 1) if conditions else None,
                         }
 
                         # Add geo parameters if any
@@ -824,7 +842,7 @@ class TQLParser:
                         return result
                     else:
                         # This is a chained operation, not a not_between operation
-                        return self._build_chained_ast(parsed)
+                        return self._build_chained_ast(parsed, depth + 1)
 
             elif len(parsed) == 3:
                 # Binary operation or comparison (including negated unary operators like "field not exists")
@@ -869,7 +887,7 @@ class TQLParser:
                         "field": field_name,
                         "type_hint": type_hint,
                         "field_mutators": field_mutators,
-                        "conditions": self._build_ast(conditions) if conditions else None,
+                        "conditions": self._build_ast(conditions, depth + 1) if conditions else None,
                     }
 
                     # Add geo parameters if any
@@ -923,7 +941,7 @@ class TQLParser:
                         "field": field_name,
                         "type_hint": type_hint,
                         "field_mutators": field_mutators,
-                        "conditions": self._build_ast(conditions) if conditions else None,
+                        "conditions": self._build_ast(conditions, depth + 1) if conditions else None,
                     }
 
                     # Add nslookup parameters if any
@@ -937,8 +955,8 @@ class TQLParser:
                     return {
                         "type": "logical_op",
                         "operator": operator.lower(),
-                        "left": self._build_ast(left),
-                        "right": self._build_ast(right),
+                        "left": self._build_ast(left, depth + 1),
+                        "right": self._build_ast(right, depth + 1),
                     }
                 elif (
                     isinstance(operator, str)
@@ -1189,7 +1207,7 @@ class TQLParser:
                 # Handle longer lists (chained operations)
                 # This happens with infixNotation for multiple AND/OR operations
                 # The structure will be flattened, so we need to reconstruct the tree
-                return self._build_chained_ast(parsed)
+                return self._build_chained_ast(parsed, depth + 1)
         else:
             # Single value - should already be a proper AST node
             if isinstance(parsed, dict):
@@ -1201,21 +1219,34 @@ class TQLParser:
         # This should be unreachable, but helps mypy understand all paths return
         raise AssertionError("Unreachable code in _build_ast")
 
-    def _build_chained_ast(self, parsed_list: List[Any]) -> Dict[str, Any]:
+    def _build_chained_ast(self, parsed_list: List[Any], depth: int = 0) -> Dict[str, Any]:
         """Build AST from chained operations (e.g., A AND B AND C).
 
         Args:
             parsed_list: List of alternating operands and operators
+            depth: Current recursion depth (for DoS prevention)
 
         Returns:
             Dictionary representing the AST node
+
+        Raises:
+            TQLSyntaxError: If query depth exceeds maximum allowed depth
         """
+        # Check depth limit to prevent stack overflow
+        if depth > self.MAX_QUERY_DEPTH:
+            raise TQLSyntaxError(
+                f"Query depth exceeds maximum allowed depth of {self.MAX_QUERY_DEPTH}. "
+                "Please simplify your query to reduce nesting.",
+                position=0,
+                query="",
+                suggestions=["Reduce query nesting depth", "Split into multiple simpler queries"]
+            )
         if len(parsed_list) < 3:
             # Not enough elements for a chained operation
             return {"type": "unknown", "value": parsed_list}
 
         # Start with the first operand
-        result = self._build_ast(parsed_list[0])
+        result = self._build_ast(parsed_list[0], depth + 1)
 
         # Process pairs of (operator, operand)
         i = 1
@@ -1228,7 +1259,7 @@ class TQLParser:
                     "type": "logical_op",
                     "operator": operator.lower(),
                     "left": result,
-                    "right": self._build_ast(operand),
+                    "right": self._build_ast(operand, depth + 1),
                 }
             else:
                 # This shouldn't happen in a well-formed chained expression

tql/post_processor.py

@@ -73,12 +73,17 @@ class QueryPostProcessor:
                             field_accessor.get_field_value, evaluator._evaluate_node
                         )
 
+                        # Safe access with validation
+                        # conditions is guaranteed to exist by the if check above
+                        conditions = requirement.metadata["conditions"]
+                        nslookup_params = requirement.metadata.get("nslookup_params", {})
+
                         # Build node for evaluation
                         node = {
                             "type": "nslookup_expr",
                             "field": requirement.field_name,
-                            "conditions": requirement.metadata["conditions"],
-                            "nslookup_params": requirement.metadata.get("nslookup_params", {}),
+                            "conditions": conditions,
+                            "nslookup_params": nslookup_params,
                         }
 
                         # Evaluate the nslookup expression
@@ -91,6 +96,7 @@ class QueryPostProcessor:
                         and requirement.metadata
                         and "conditions" in requirement.metadata
                     ):
+                        # Safe access - conditions is guaranteed to exist by the if check
                         conditions = requirement.metadata["conditions"]
                         if conditions:
                             # Get the geo data that was enriched
@@ -129,12 +135,13 @@ class QueryPostProcessor:
                         # Check if this is an array operator with comparison
                         if "comparison_operator" in requirement.metadata:
                             # This is a special case: field | any/all/none eq value
-                            array_operator = requirement.metadata["operator"]
-                            comparison_operator = requirement.metadata["comparison_operator"]
+                            # Safe access - both keys are guaranteed to exist by the if checks
+                            array_operator = requirement.metadata["operator"]  # exists from line 128 check
+                            comparison_operator = requirement.metadata["comparison_operator"]  # exists from line 135 check
                             value = requirement.metadata.get("value")
 
-                            # Get the field value
-                            temp_field_name = f"__{requirement.field_name}_mutated__"
+                            # Get the field value with proper nested field handling
+                            temp_field_name = self._get_mutated_field_name(requirement.field_name)
                             field_value = self._get_field_value(result, temp_field_name)
                             if field_value is None:
                                 # No mutated value, get original
@@ -148,18 +155,21 @@ class QueryPostProcessor:
                                 break
                         else:
                             # Regular operator check
+                            # Safe access - operator is guaranteed to exist by the if check at line 134
                             operator = requirement.metadata["operator"]
                             value = requirement.metadata.get("value")
 
                             # Check if this was originally a different operator (for type-changing mutators)
                             if requirement.metadata.get("_original_comparison"):
+                                # Safe access - validated by .get() check above
                                 original = requirement.metadata["_original_comparison"]
-                                operator = original["operator"]
+                                # Validate that operator exists in original
+                                operator = original.get("operator", operator)
                                 value = original.get("value", value)
 
                             # Get the field value - either mutated or original
                             # First check for mutated value in temp field
-                            temp_field_name = f"__{requirement.field_name}_mutated__"
+                            temp_field_name = self._get_mutated_field_name(requirement.field_name)
                             field_value = self._get_field_value(result, temp_field_name)
                             if field_value is None:
                                 # No mutated value, get original
@@ -373,7 +383,7 @@ class QueryPostProcessor:
                 return False
 
             # Get the field value
-            temp_field_name = f"__{field_name}_mutated__"
+            temp_field_name = self._get_mutated_field_name(field_name)
             field_value = self._get_field_value(result, temp_field_name)
             if field_value is None:
                 # No mutated value, get original
@@ -703,7 +713,7 @@ class QueryPostProcessor:
                 self._set_field_value(result, requirement.field_name, mutated_value)
             elif not is_geo_enrichment:
                 # For type-changing mutators with filtering operations, store in temp field
-                temp_field_name = f"__{requirement.field_name}_mutated__"
+                temp_field_name = self._get_mutated_field_name(requirement.field_name)
                 self._set_field_value(result, temp_field_name, mutated_value)
 
             # Check if we have any enrichment mutators
@@ -994,6 +1004,25 @@ class QueryPostProcessor:
 
         return current
 
+    def _get_mutated_field_name(self, field_name: str) -> str:
+        """Generate the correct mutated field name for nested or flat fields.
+
+        Args:
+            field_name: The original field name (e.g., "user.address.zip" or "status")
+
+        Returns:
+            Mutated field name with proper nesting:
+            - "user.address.zip" -> "user.address.__zip_mutated__"
+            - "status" -> "__status_mutated__"
+        """
+        field_parts = field_name.split('.')
+        if len(field_parts) > 1:
+            # For nested fields, only mutate the leaf field name
+            return '.'.join(field_parts[:-1] + [f"__{field_parts[-1]}_mutated__"])
+        else:
+            # For flat fields, mutate the entire name
+            return f"__{field_name}_mutated__"
+
     def _get_field_value(self, record: Dict[str, Any], field_path: str) -> Any:
         """Get a field value from a record, supporting nested fields.
 

tql/scripts.py

@@ -1,5 +1,6 @@
 """ Runs pytest, coverage, linters, and security checks. """
 
+import os
 import subprocess  # nosec
 
 
@@ -29,8 +30,13 @@ def run_coverage():
     Run coverage against all files in the `src` directory
     and output an XML report to `reports/coverage.xml`.
     """
+    # Set environment to skip integration tests by default
+    env = os.environ.copy()
+    if "INTEGRATION_TEST_ENABLE" not in env:
+        env["INTEGRATION_TEST_ENABLE"] = "false"
+    
     # 1. Run pytest with coverage, using `src` as the source
-    subprocess.run(["coverage", "run", "--source=src", "-m", "pytest"], check=True)  # nosec
+    subprocess.run(["coverage", "run", "--source=src", "-m", "pytest"], check=True, env=env)  # nosec
 
     # 2. Generate an XML coverage report in `reports/coverage.xml`
     subprocess.run(["coverage", "xml", "-o", "reports/coverage/coverage.xml"], check=True)  # nosec
@@ -40,7 +46,12 @@ def run_coverage():
 
 def run_tests():
     """Runs pytests against tests in the `tests` directory."""
-    subprocess.run(["pytest", "tests"], check=True)  # nosec
+    # Set environment to skip integration tests by default
+    env = os.environ.copy()
+    if "INTEGRATION_TEST_ENABLE" not in env:
+        env["INTEGRATION_TEST_ENABLE"] = "false"
+    
+    subprocess.run(["pytest", "tests"], check=True, env=env)  # nosec
 
 
 def run_lint_all():
@@ -84,6 +95,11 @@ def run_lint():
 
 def run_badge():
     """Generate a badge using genbadge."""
+    # Set environment to skip integration tests by default
+    env = os.environ.copy()
+    if "INTEGRATION_TEST_ENABLE" not in env:
+        env["INTEGRATION_TEST_ENABLE"] = "false"
+    
     subprocess.run(  # nosec
         [
             "coverage",
@@ -94,6 +110,7 @@ def run_badge():
             "--junit-xml=reports/junit/junit.xml",
         ],
         check=True,
+        env=env,
     )
 
     # 2. Generate an XML coverage report in `reports/coverage.xml`

tql/stats_evaluator.py

@@ -6,7 +6,7 @@ aggregation queries against data records in memory.
 
 import statistics
 from collections import defaultdict
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Set, Union
 
 from .exceptions import TQLError
 
@@ -180,7 +180,9 @@ class TQLStatsEvaluator:
             # Build group key
             key_parts = []
             for field_spec in normalized_fields:
-                field_name = field_spec["field"]
+                field_name = field_spec.get("field")
+                if field_name is None:
+                    continue
                 value = self._get_field_value(record, field_name)
                 key_parts.append((field_name, value))
 
@@ -407,7 +409,7 @@ class TQLStatsEvaluator:
         filtered_results = []
 
         # Track unique values at each level
-        level_values = {}
+        level_values: Dict[int, Dict[Any, Set[Any]]] = {}
         for level, field_spec in enumerate(normalized_fields):
             level_values[level] = {}