elasticsearch9 9.1.1__py3-none-any.whl → 9.2.0__py3-none-any.whl

elasticsearch9/esql/functions.py

@@ -38,6 +38,20 @@ def abs(number: ExpressionType) -> InstrumentedExpression:
     return InstrumentedExpression(f"ABS({_render(number)})")
 
 
+def absent(field: ExpressionType) -> InstrumentedExpression:
+    """Returns true if the input expression yields no non-null values within the
+    current aggregation context.
+
+    :param field: Expression that outputs values to be checked for absence.
+    """
+    return InstrumentedExpression(f"ABSENT({_render(field)})")
+
+
+def absent_over_time(field: ExpressionType) -> InstrumentedExpression:
+    """Calculates the absence of a field in the output result over time range."""
+    return InstrumentedExpression(f"ABSENT_OVER_TIME({_render(field)})")
+
+
 def acos(number: ExpressionType) -> InstrumentedExpression:
     """Returns the arccosine of `n` as an angle, expressed in radians.
 
@@ -364,6 +378,11 @@ def exp(number: ExpressionType) -> InstrumentedExpression:
     return InstrumentedExpression(f"EXP({_render(number)})")
 
 
+def first(value: ExpressionType, sort: ExpressionType) -> InstrumentedExpression:
+    """Calculates the earliest value of a field."""
+    return InstrumentedExpression(f"FIRST({_render(value)}, {_render(sort)})")
+
+
 def first_over_time(field: ExpressionType) -> InstrumentedExpression:
     """The earliest value of a field, where recency determined by the
     `@timestamp` field.
@@ -463,6 +482,11 @@ def kql(query: ExpressionType) -> InstrumentedExpression:
     return InstrumentedExpression(f"KQL({_render(query)})")
 
 
+def last(value: ExpressionType, sort: ExpressionType) -> InstrumentedExpression:
+    """Calculates the latest value of a field."""
+    return InstrumentedExpression(f"LAST({_render(value)}, {_render(sort)})")
+
+
 def last_over_time(field: ExpressionType) -> InstrumentedExpression:
     """The latest value of a field, where recency determined by the
     `@timestamp` field.
@@ -697,6 +721,18 @@ def mv_concat(string: ExpressionType, delim: ExpressionType) -> InstrumentedExpr
     return InstrumentedExpression(f"MV_CONCAT({_render(string)}, {_render(delim)})")
 
 
+def mv_contains(
+    superset: ExpressionType, subset: ExpressionType
+) -> InstrumentedExpression:
+    """Checks if all values yielded by the second multivalue expression are present in the
+    values yielded by the first multivalue expression. Returns a boolean. Null values are
+    treated as an empty set.
+    """
+    return InstrumentedExpression(
+        f"MV_CONTAINS({_render(superset)}, {_render(subset)})"
+    )
+
+
 def mv_count(field: ExpressionType) -> InstrumentedExpression:
     """Converts a multivalued expression into a single valued column containing
     a count of the number of values.
@@ -894,6 +930,18 @@ def pow(base: ExpressionType, exponent: ExpressionType) -> InstrumentedExpressio
     return InstrumentedExpression(f"POW({_render(base)}, {_render(exponent)})")
 
 
+def present(field: ExpressionType) -> InstrumentedExpression:
+    """Returns true if the input expression yields any non-null values within the current
+    aggregation context. Otherwise it returns false.
+    """
+    return InstrumentedExpression(f"PRESENT({_render(field)})")
+
+
+def present_over_time(field: ExpressionType) -> InstrumentedExpression:
+    """Calculates the presence of a field in the output result over time range."""
+    return InstrumentedExpression(f"PRESENT_OVER_TIME({_render(field)})")
+
+
 def qstr(
     query: ExpressionType, options: ExpressionType = None
 ) -> InstrumentedExpression:
@@ -1452,6 +1500,11 @@ def sum(number: ExpressionType) -> InstrumentedExpression:
     return InstrumentedExpression(f"SUM({_render(number)})")
 
 
+def sum_over_time(field: ExpressionType) -> InstrumentedExpression:
+    """Calculates the sum over time value of a field."""
+    return InstrumentedExpression(f"SUM({_render(field)})")
+
+
 def tan(angle: ExpressionType) -> InstrumentedExpression:
     """Returns the tangent of an angle.
 
@@ -1483,6 +1536,17 @@ def term(field: ExpressionType, query: ExpressionType) -> InstrumentedExpression
     return InstrumentedExpression(f"TERM({_render(field)}, {_render(query)})")
 
 
+def text_embedding(
+    text: ExpressionType, inference_id: ExpressionType
+) -> InstrumentedExpression:
+    """Generates dense vector embeddings from text input using a specified inference endpoint.
+    Use this function to generate query vectors for KNN searches against your vectorized data
+    or others dense vector based operations."""
+    return InstrumentedExpression(
+        f"TEXT_EMBEDDING({_render(text)}, {_render(inference_id)})"
+    )
+
+
 def top(
     field: ExpressionType, limit: ExpressionType, order: ExpressionType
 ) -> InstrumentedExpression:
@@ -1596,6 +1660,22 @@ def to_double(field: ExpressionType) -> InstrumentedExpression:
     return InstrumentedExpression(f"TO_DOUBLE({_render(field)})")
 
 
+def to_geohash(field: ExpressionType) -> InstrumentedExpression:
+    """Converts an input value to a geohash value. A string will only be successfully
+    converted if it respects the geohash format, as described for the geohash grid
+    aggregation.
+    """
+    return InstrumentedExpression(f"TO_GEOHASH({_render(field)})")
+
+
+def to_geohex(field: ExpressionType) -> InstrumentedExpression:
+    """Converts an input value to a geohex value. A string will only be successfully
+    converted if it respects the geohex format, as described for the geohex grid
+    aggregation.
+    """
+    return InstrumentedExpression(f"TO_GEOHEX({_render(field)})")
+
+
 def to_geopoint(field: ExpressionType) -> InstrumentedExpression:
     """Converts an input value to a `geo_point` value. A string will only be
     successfully converted if it respects the WKT Point format.
@@ -1616,6 +1696,14 @@ def to_geoshape(field: ExpressionType) -> InstrumentedExpression:
     return InstrumentedExpression(f"TO_GEOSHAPE({_render(field)})")
 
 
+def to_geotile(field: ExpressionType) -> InstrumentedExpression:
+    """Converts an input value to a geotile value. A string will only be successfully
+    converted if it respects the geotile format, as described for the geotile grid
+    aggregation.
+    """
+    return InstrumentedExpression(f"TO_GEOTILE({_render(field)})")
+
+
 def to_integer(field: ExpressionType) -> InstrumentedExpression:
     """Converts an input value to an integer value. If the input parameter is
     of a date type, its value will be interpreted as milliseconds since the

elasticsearch9/helpers/__init__.py

@@ -19,12 +19,21 @@ from .._async.helpers import async_bulk, async_reindex, async_scan, async_stream
 from .._utils import fixup_module_metadata
 from .actions import _chunk_actions  # noqa: F401
 from .actions import _process_bulk_chunk  # noqa: F401
-from .actions import bulk, expand_action, parallel_bulk, reindex, scan, streaming_bulk
+from .actions import (
+    BULK_FLUSH,
+    bulk,
+    expand_action,
+    parallel_bulk,
+    reindex,
+    scan,
+    streaming_bulk,
+)
 from .errors import BulkIndexError, ScanError
 
 __all__ = [
     "BulkIndexError",
     "ScanError",
+    "BULK_FLUSH",
     "expand_action",
     "streaming_bulk",
     "bulk",

elasticsearch9/helpers/actions.py

@@ -16,9 +16,10 @@
 #  under the License.
 
 import logging
+import queue
 import time
+from enum import Enum
 from operator import methodcaller
-from queue import Queue
 from typing import (
     Any,
     Callable,
@@ -37,13 +38,21 @@ from typing import (
 from elastic_transport import OpenTelemetrySpan
 
 from .. import Elasticsearch
-from ..compat import to_bytes
+from ..compat import safe_thread, to_bytes
 from ..exceptions import ApiError, NotFoundError, TransportError
 from ..serializer import Serializer
 from .errors import BulkIndexError, ScanError
 
 logger = logging.getLogger("elasticsearch.helpers")
 
+
+class BulkMeta(Enum):
+    flush = 1
+    done = 2
+
+
+BULK_FLUSH = BulkMeta.flush
+
 _TYPE_BULK_ACTION = Union[bytes, str, Dict[str, Any]]
 _TYPE_BULK_ACTION_HEADER = Dict[str, Any]
 _TYPE_BULK_ACTION_BODY = Union[None, bytes, Dict[str, Any]]
@@ -51,6 +60,13 @@ _TYPE_BULK_ACTION_HEADER_AND_BODY = Tuple[
     _TYPE_BULK_ACTION_HEADER, _TYPE_BULK_ACTION_BODY
 ]
 
+_TYPE_BULK_ACTION_WITH_META = Union[bytes, str, Dict[str, Any], BulkMeta]
+_TYPE_BULK_ACTION_HEADER_WITH_META = Union[Dict[str, Any], BulkMeta]
+_TYPE_BULK_ACTION_HEADER_WITH_META_AND_BODY = Union[
+    Tuple[_TYPE_BULK_ACTION_HEADER, _TYPE_BULK_ACTION_BODY],
+    Tuple[BulkMeta, Any],
+]
+
 
 def expand_action(data: _TYPE_BULK_ACTION) -> _TYPE_BULK_ACTION_HEADER_AND_BODY:
     """
@@ -139,7 +155,9 @@ class _ActionChunker:
         ] = []
 
     def feed(
-        self, action: _TYPE_BULK_ACTION_HEADER, data: _TYPE_BULK_ACTION_BODY
+        self,
+        action: _TYPE_BULK_ACTION_HEADER_WITH_META,
+        data: _TYPE_BULK_ACTION_BODY,
     ) -> Optional[
         Tuple[
             List[
@@ -152,23 +170,25 @@ class _ActionChunker:
         ]
     ]:
         ret = None
-        raw_action = action
-        raw_data = data
-        action_bytes = to_bytes(self.serializer.dumps(action), "utf-8")
-        # +1 to account for the trailing new line character
-        cur_size = len(action_bytes) + 1
-
-        data_bytes: Optional[bytes]
-        if data is not None:
-            data_bytes = to_bytes(self.serializer.dumps(data), "utf-8")
-            cur_size += len(data_bytes) + 1
-        else:
-            data_bytes = None
+        action_bytes = b""
+        data_bytes: Optional[bytes] = None
+        cur_size = 0
+        if not isinstance(action, BulkMeta):
+            action_bytes = to_bytes(self.serializer.dumps(action), "utf-8")
+            # +1 to account for the trailing new line character
+            cur_size = len(action_bytes) + 1
+
+            if data is not None:
+                data_bytes = to_bytes(self.serializer.dumps(data), "utf-8")
+                cur_size += len(data_bytes) + 1
+            else:
+                data_bytes = None
 
         # full chunk, send it and start a new one
         if self.bulk_actions and (
             self.size + cur_size > self.max_chunk_bytes
             or self.action_count == self.chunk_size
+            or (action == BulkMeta.flush and self.bulk_actions)
         ):
             ret = (self.bulk_data, self.bulk_actions)
             self.bulk_actions = []
@@ -176,15 +196,16 @@ class _ActionChunker:
             self.size = 0
             self.action_count = 0
 
-        self.bulk_actions.append(action_bytes)
-        if data_bytes is not None:
-            self.bulk_actions.append(data_bytes)
-            self.bulk_data.append((raw_action, raw_data))
-        else:
-            self.bulk_data.append((raw_action,))
+        if not isinstance(action, BulkMeta):
+            self.bulk_actions.append(action_bytes)
+            if data_bytes is not None:
+                self.bulk_actions.append(data_bytes)
+                self.bulk_data.append((action, data))
+            else:
+                self.bulk_data.append((action,))
 
-        self.size += cur_size
-        self.action_count += 1
+            self.size += cur_size
+            self.action_count += 1
         return ret
 
     def flush(
@@ -209,9 +230,10 @@ class _ActionChunker:
 
 
 def _chunk_actions(
-    actions: Iterable[_TYPE_BULK_ACTION_HEADER_AND_BODY],
+    actions: Iterable[_TYPE_BULK_ACTION_HEADER_WITH_META_AND_BODY],
     chunk_size: int,
     max_chunk_bytes: int,
+    flush_after_seconds: Optional[float],
     serializer: Serializer,
 ) -> Iterable[
     Tuple[
@@ -231,10 +253,41 @@ def _chunk_actions(
     chunker = _ActionChunker(
         chunk_size=chunk_size, max_chunk_bytes=max_chunk_bytes, serializer=serializer
     )
-    for action, data in actions:
-        ret = chunker.feed(action, data)
-        if ret:
-            yield ret
+
+    if not flush_after_seconds:
+        for action, data in actions:
+            ret = chunker.feed(action, data)
+            if ret:
+                yield ret
+    else:
+        item_queue: queue.Queue[_TYPE_BULK_ACTION_HEADER_WITH_META_AND_BODY] = (
+            queue.Queue(maxsize=1)
+        )
+
+        def get_items() -> None:
+            try:
+                for item in actions:
+                    item_queue.put(item)
+            finally:
+                # make sure we signal the end even if there is an exception
+                item_queue.put((BulkMeta.done, None))
+
+        with safe_thread(get_items):
+            timeout: Optional[float] = flush_after_seconds
+            while True:
+                try:
+                    action, data = item_queue.get(timeout=timeout)
+                    timeout = flush_after_seconds
+                except queue.Empty:
+                    action, data = BulkMeta.flush, None
+                    timeout = None
+
+                if action is BulkMeta.done:
+                    break
+                ret = chunker.feed(action, data)
+                if ret:
+                    yield ret
+
     ret = chunker.flush()
     if ret:
         yield ret
@@ -361,9 +414,10 @@ def _process_bulk_chunk(
 
 def streaming_bulk(
     client: Elasticsearch,
-    actions: Iterable[_TYPE_BULK_ACTION],
+    actions: Iterable[_TYPE_BULK_ACTION_WITH_META],
     chunk_size: int = 500,
     max_chunk_bytes: int = 100 * 1024 * 1024,
+    flush_after_seconds: Optional[float] = None,
     raise_on_error: bool = True,
     expand_action_callback: Callable[
         [_TYPE_BULK_ACTION], _TYPE_BULK_ACTION_HEADER_AND_BODY
@@ -397,6 +451,9 @@ def streaming_bulk(
     :arg actions: iterable containing the actions to be executed
     :arg chunk_size: number of docs in one chunk sent to es (default: 500)
     :arg max_chunk_bytes: the maximum size of the request in bytes (default: 100MB)
+    :arg flush_after_seconds: time in seconds after which a chunk is written even
+        if hasn't reached `chunk_size` or `max_chunk_bytes`. Set to 0 to not use a
+        timeout-based flush. (default: 0)
     :arg raise_on_error: raise ``BulkIndexError`` containing errors (as `.errors`)
         from the execution of the last chunk when some occur. By default we raise.
     :arg raise_on_exception: if ``False`` then don't propagate exceptions from
@@ -425,6 +482,13 @@ def streaming_bulk(
 
         serializer = client.transport.serializers.get_serializer("application/json")
 
+        def expand_action_with_meta(
+            data: _TYPE_BULK_ACTION_WITH_META,
+        ) -> _TYPE_BULK_ACTION_HEADER_WITH_META_AND_BODY:
+            if isinstance(data, BulkMeta):
+                return data, None
+            return expand_action_callback(data)
+
         bulk_data: List[
             Union[
                 Tuple[_TYPE_BULK_ACTION_HEADER],
@@ -433,9 +497,10 @@ def streaming_bulk(
         ]
         bulk_actions: List[bytes]
         for bulk_data, bulk_actions in _chunk_actions(
-            map(expand_action_callback, actions),
+            map(expand_action_with_meta, actions),
             chunk_size,
             max_chunk_bytes,
+            flush_after_seconds,
             serializer,
         ):
             for attempt in range(max_retries + 1):
@@ -557,6 +622,7 @@ def parallel_bulk(
     thread_count: int = 4,
     chunk_size: int = 500,
     max_chunk_bytes: int = 100 * 1024 * 1024,
+    flush_after_seconds: Optional[float] = None,
     queue_size: int = 4,
     expand_action_callback: Callable[
         [_TYPE_BULK_ACTION], _TYPE_BULK_ACTION_HEADER_AND_BODY
@@ -573,6 +639,9 @@ def parallel_bulk(
     :arg thread_count: size of the threadpool to use for the bulk requests
     :arg chunk_size: number of docs in one chunk sent to es (default: 500)
     :arg max_chunk_bytes: the maximum size of the request in bytes (default: 100MB)
+    :arg flush_after_seconds: time in seconds after which a chunk is written even
+        if hasn't reached `chunk_size` or `max_chunk_bytes`. Set to 0 to not use a
+        timeout-based flush. (default: 0)
     :arg raise_on_error: raise ``BulkIndexError`` containing errors (as `.errors`)
         from the execution of the last chunk when some occur. By default we raise.
     :arg raise_on_exception: if ``False`` then don't propagate exceptions from
@@ -596,7 +665,7 @@ def parallel_bulk(
             super()._setup_queues()  # type: ignore[misc]
             # The queue must be at least the size of the number of threads to
             # prevent hanging when inserting sentinel values during teardown.
-            self._inqueue: Queue[
+            self._inqueue: queue.Queue[
                 Tuple[
                     List[
                         Union[
@@ -605,7 +674,7 @@ def parallel_bulk(
                     ],
                     List[bytes],
                 ]
-            ] = Queue(max(queue_size, thread_count))
+            ] = queue.Queue(max(queue_size, thread_count))
             self._quick_put = self._inqueue.put
 
     with client._otel.helpers_span("helpers.parallel_bulk") as otel_span:
@@ -625,7 +694,11 @@ def parallel_bulk(
                     )
                 ),
                 _chunk_actions(
-                    expanded_actions, chunk_size, max_chunk_bytes, serializer
+                    expanded_actions,
+                    chunk_size,
+                    max_chunk_bytes,
+                    flush_after_seconds,
+                    serializer,
                 ),
             ):
                 yield from result

{elasticsearch9-9.1.1.dist-info → elasticsearch9-9.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: elasticsearch9
-Version: 9.1.1
+Version: 9.2.0
 Summary: Python client for Elasticsearch
 Project-URL: Documentation, https://elasticsearch9-py.readthedocs.io/
 Project-URL: Homepage, https://github.com/elastic/elasticsearch9-py
@@ -18,16 +18,17 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
-Requires-Python: >=3.9
-Requires-Dist: elastic-transport<10,>=9.1.0
+Requires-Python: >=3.10
+Requires-Dist: anyio
+Requires-Dist: elastic-transport<10,>=9.2.0
 Requires-Dist: python-dateutil
+Requires-Dist: sniffio
 Requires-Dist: typing-extensions
 Provides-Extra: async
 Requires-Dist: aiohttp<4,>=3; extra == 'async'
@@ -36,6 +37,7 @@ Requires-Dist: aiohttp; extra == 'dev'
 Requires-Dist: black; extra == 'dev'
 Requires-Dist: build; extra == 'dev'
 Requires-Dist: coverage; extra == 'dev'
+Requires-Dist: httpx; extra == 'dev'
 Requires-Dist: isort; extra == 'dev'
 Requires-Dist: jinja2; extra == 'dev'
 Requires-Dist: mapbox-vector-tile; extra == 'dev'
@@ -44,7 +46,8 @@ Requires-Dist: nox; extra == 'dev'
 Requires-Dist: numpy; extra == 'dev'
 Requires-Dist: orjson; extra == 'dev'
 Requires-Dist: pandas; extra == 'dev'
-Requires-Dist: pyarrow; extra == 'dev'
+Requires-Dist: pyarrow; (python_version < '3.14') and extra == 'dev'
+Requires-Dist: pydantic; extra == 'dev'
 Requires-Dist: pyright; extra == 'dev'
 Requires-Dist: pytest; extra == 'dev'
 Requires-Dist: pytest-asyncio; extra == 'dev'
@@ -55,6 +58,7 @@ Requires-Dist: pyyaml>=5.4; extra == 'dev'
 Requires-Dist: requests<3,>=2; extra == 'dev'
 Requires-Dist: simsimd; extra == 'dev'
 Requires-Dist: tqdm; extra == 'dev'
+Requires-Dist: trio; extra == 'dev'
 Requires-Dist: twine; extra == 'dev'
 Requires-Dist: types-python-dateutil; extra == 'dev'
 Requires-Dist: types-tqdm; extra == 'dev'
@@ -166,6 +170,24 @@ Documentation for the client is [available on elastic.co] and [Read the Docs].
 [Read the Docs]: https://elasticsearch-py.readthedocs.io
 
 
+## Try Elasticsearch and Kibana locally
+
+If you want to try Elasticsearch and Kibana locally, you can run the following command:
+
+```bash
+curl -fsSL https://elastic.co/start-local | sh
+```
+
+This will run Elasticsearch at [http://localhost:9200](http://localhost:9200) and Kibana at [http://localhost:5601](http://localhost:5601).
+
+More information is available [here](https://www.elastic.co/guide/en/elasticsearch/reference/current/run-elasticsearch-locally.html).
+
+
+## Contributing
+
+See [CONTRIBUTING.md](./CONTRIBUTING.md)
+
+
 ## License
 
 This software is licensed under the [Apache License 2.0](./LICENSE). See [NOTICE](./NOTICE).