elastalert2 2.30.0__tar.gz → 2.31.0__tar.gz

{elastalert2-2.30.0/elastalert2.egg-info → elastalert2-2.31.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: elastalert2
-Version: 2.30.0
+Version: 2.31.0
 Summary: Automated rule-based alerting for Elasticsearch
 Home-page: https://github.com/jertel/elastalert2
 License: Apache 2.0

{elastalert2-2.30.0 → elastalert2-2.31.0}/elastalert/__init__.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 import copy
 import elastalert.eql as eql
+import elastalert.esql as esql
 
 from elasticsearch import Elasticsearch
 from elasticsearch import RequestsHttpConnection
@@ -134,6 +135,7 @@ class ElasticSearchClient(Elasticsearch):
 
         path = _make_path(index, doc_type, "_search")
         eql_body = eql.format_request(body)
+        esql_body = esql.format_request(body)
         if eql_body is not None:
             path = path.replace('/_search', '/_eql/search')
             body = eql_body
@@ -143,6 +145,10 @@ class ElasticSearchClient(Elasticsearch):
                 params.pop('scroll')
             if '_source_includes' in params:
                 params.pop('_source_includes')
+        elif esql_body is not None:
+            path = "/_query"
+            body = esql_body
+            params = {'format': 'json'}
 
         results = self.transport.perform_request(
             "POST",
@@ -152,6 +158,9 @@ class ElasticSearchClient(Elasticsearch):
             body=body,
         )
 
-        eql.format_results(results);
+        if esql_body is not None:
+            esql.format_results(results, default_index=index)
+        else:
+            eql.format_results(results)
 
         return results

{elastalert2-2.30.0 → elastalert2-2.31.0}/elastalert/elastalert.py

@@ -337,9 +337,10 @@ class ElastAlerter(object):
 
             # Convert the timestamp to a datetime
             ts = lookup_es_key(hit['_source'], rule['timestamp_field'])
-            if not ts and not rule["_source_enabled"]:
+            if ts is None:
                 raise EAException(
-                    "Error: No timestamp was found for hit. '_source_enabled' is set to false, check your mappings for stored fields"
+                    f"Error: The configured timestamp_field '{rule['timestamp_field']}' was not found in the query results. "
+                    f"Available fields: {list(hit['_source'].keys())}"
                 )
 
             set_es_key(hit['_source'], rule['timestamp_field'], rule['ts_to_dt'](ts))

elastalert2-2.31.0/elastalert/esql.py

@@ -0,0 +1,76 @@
+import hashlib
+import json
+
+
+def format_request(body):
+    query = body.get('query')
+    if not query:
+        return None
+    
+    query_bool = query.get('bool')
+    if not query_bool:
+        return None
+    
+    filt = query_bool.get('filter')
+    if not filt:
+        return None
+    
+    filter_bool = filt.get('bool')
+    if not filter_bool:
+        return None
+    
+    filter_bool_must = filter_bool.get('must')
+    if not filter_bool_must:
+        return None
+    
+    other_filters = []
+    esql = None
+    for f in filter_bool_must:
+        if f.get('esql'):
+            esql = f['esql']
+        else:
+            other_filters.append(f)
+
+    if esql:
+        new_body = {'filter': {'bool': {'must': other_filters}}, 'query': esql}
+        return new_body
+
+    return None
+
+
+def format_results(results, default_index=None):
+    columns = results.pop('columns', None)
+    values = results.pop('values', None)
+    if columns is None or values is None:
+        return results
+    
+    col_names = [col['name'] for col in columns]
+    hits = []
+    for val_row in values:
+        doc = dict(zip(col_names, val_row))
+        
+        # Pull metadata fields if present
+        doc_id = doc.get('_id', None)
+        doc_index = doc.get('_index', default_index)
+        
+        if not doc_id:
+            # Generate deterministic hash of the document to use as unique ID
+            doc_hash = hashlib.sha256(json.dumps(doc, sort_keys=True, default=str).encode('utf-8')).hexdigest()
+            doc_id = doc_hash
+            
+        hit = {
+            '_index': doc_index,
+            '_id': doc_id,
+            '_source': doc
+        }
+        hits.append(hit)
+        
+    results['hits'] = {
+        'total': {
+            'value': len(values),
+            'relation': 'eq'
+        },
+        'hits': hits
+    }
+    results['esql'] = True
+    return results

{elastalert2-2.30.0 → elastalert2-2.31.0}/elastalert/kibana_discover.py

@@ -74,6 +74,36 @@ def kibana7_disover_global_state(from_time, to_time):
     } )
 
 
+def _normalize_query_value(value):
+    """Collapse empty/single-element query_key values before filter emission.
+
+    Elasticsearch returns multi-valued fields as arrays even when they hold a
+    single item, and rules using such fields as ``query_key`` otherwise render
+    Kibana filters with ``params.query: ['']`` — a phrase filter that matches
+    no documents, producing an empty Discover view for the reviewer.
+
+    Returns:
+      - ``None`` when the value is absent, an empty string, or a list that
+        collapses to nothing after dropping ``None``/empty-string entries;
+        callers emit the existing ``negate exists`` filter for this case.
+      - The sole surviving element when the input is a single-item list.
+      - The original value otherwise (scalars pass through; multi-value lists
+        are returned unchanged so callers may extend to a ``phrases`` filter).
+    """
+    if value is None:
+        return None
+    if isinstance(value, str):
+        return None if value == '' else value
+    if isinstance(value, list):
+        cleaned = [v for v in value if v not in (None, '')]
+        if not cleaned:
+            return None
+        if len(cleaned) == 1:
+            return cleaned[0]
+        return cleaned
+    return value
+
+
 def kibana_discover_app_state(index, columns, filters, query_keys, match):
     app_filters = []
 
@@ -105,7 +135,7 @@ def kibana_discover_app_state(index, columns, filters, query_keys, match):
         } )
 
     for query_key in query_keys:
-        query_value = lookup_es_key(match, query_key)
+        query_value = _normalize_query_value(lookup_es_key(match, query_key))
 
         if query_value is None:
             app_filters.append( {

{elastalert2-2.30.0 → elastalert2-2.31.0}/elastalert/test_rule.py

@@ -211,7 +211,7 @@ class MockElastAlerter(object):
 
         terms = res['hits']['hits'][0]['_source']
 
-        is_eql = res.get('eql') == True
+        is_eql = res.get('eql') == True or res.get('esql') == True
 
         if not is_eql:
             # Get a count of all docs

{elastalert2-2.30.0 → elastalert2-2.31.0/elastalert2.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: elastalert2
-Version: 2.30.0
+Version: 2.31.0
 Summary: Automated rule-based alerting for Elasticsearch
 Home-page: https://github.com/jertel/elastalert2
 License: Apache 2.0

{elastalert2-2.30.0 → elastalert2-2.31.0}/elastalert2.egg-info/SOURCES.txt

@@ -10,6 +10,7 @@ elastalert/create_index.py
 elastalert/elastalert.py
 elastalert/enhancements.py
 elastalert/eql.py
+elastalert/esql.py
 elastalert/kibana_discover.py
 elastalert/kibana_external_url_formatter.py
 elastalert/loaders.py

{elastalert2-2.30.0 → elastalert2-2.31.0}/setup.py

@@ -8,7 +8,7 @@ from setuptools import setup
 base_dir = os.path.dirname(__file__)
 setup(
     name='elastalert2',
-    version='2.30.0',
+    version='2.31.0',
     description='Automated rule-based alerting for Elasticsearch',
     long_description=open('README.md').read(),
     long_description_content_type="text/markdown",