elasticsearch9 9.1.1__py3-none-any.whl → 9.1.3__py3-none-any.whl

elasticsearch9/dsl/types.py

@@ -938,6 +938,7 @@ class GeoDistanceSort(AttrDict[Any]):
             Dict[str, Any],
             "DefaultType",
         ] = DEFAULT,
+        /,
         *,
         mode: Union[
             Literal["min", "max", "sum", "avg", "median"], DefaultType
@@ -2774,6 +2775,31 @@ class NumericFielddata(AttrDict[Any]):
         super().__init__(kwargs)
 
 
+class PValueHeuristic(AttrDict[Any]):
+    """
+    :arg background_is_superset:
+    :arg normalize_above: Should the results be normalized when above the
+        given value. Allows for consistent significance results at various
+        scales. Note: `0` is a special value which means no normalization
+    """
+
+    background_is_superset: Union[bool, DefaultType]
+    normalize_above: Union[int, DefaultType]
+
+    def __init__(
+        self,
+        *,
+        background_is_superset: Union[bool, DefaultType] = DEFAULT,
+        normalize_above: Union[int, DefaultType] = DEFAULT,
+        **kwargs: Any,
+    ):
+        if background_is_superset is not DEFAULT:
+            kwargs["background_is_superset"] = background_is_superset
+        if normalize_above is not DEFAULT:
+            kwargs["normalize_above"] = normalize_above
+        super().__init__(kwargs)
+
+
 class PercentageScoreHeuristic(AttrDict[Any]):
     pass
 
@@ -3219,6 +3245,7 @@ class SortOptions(AttrDict[Any]):
         self,
         _field: Union[str, "InstrumentedField", "DefaultType"] = DEFAULT,
         _value: Union["FieldSort", Dict[str, Any], "DefaultType"] = DEFAULT,
+        /,
         *,
         _score: Union["ScoreSort", Dict[str, Any], DefaultType] = DEFAULT,
         _doc: Union["ScoreSort", Dict[str, Any], DefaultType] = DEFAULT,
@@ -4009,24 +4036,25 @@ class TestPopulation(AttrDict[Any]):
 
 class TextEmbedding(AttrDict[Any]):
     """
-    :arg model_id: (required)
     :arg model_text: (required)
+    :arg model_id: Model ID is required for all dense_vector fields but
+        may be inferred for semantic_text fields
     """
 
-    model_id: Union[str, DefaultType]
     model_text: Union[str, DefaultType]
+    model_id: Union[str, DefaultType]
 
     def __init__(
         self,
         *,
-        model_id: Union[str, DefaultType] = DEFAULT,
         model_text: Union[str, DefaultType] = DEFAULT,
+        model_id: Union[str, DefaultType] = DEFAULT,
         **kwargs: Any,
     ):
-        if model_id is not DEFAULT:
-            kwargs["model_id"] = model_id
         if model_text is not DEFAULT:
             kwargs["model_text"] = model_text
+        if model_id is not DEFAULT:
+            kwargs["model_id"] = model_id
         super().__init__(kwargs)
 
 
@@ -4659,6 +4687,82 @@ class CardinalityAggregate(AttrDict[Any]):
     meta: Mapping[str, Any]
 
 
+class CartesianBoundsAggregate(AttrDict[Any]):
+    """
+    :arg bounds:
+    :arg meta:
+    """
+
+    bounds: "TopLeftBottomRightGeoBounds"
+    meta: Mapping[str, Any]
+
+
+class CartesianCentroidAggregate(AttrDict[Any]):
+    """
+    :arg count: (required)
+    :arg location:
+    :arg meta:
+    """
+
+    count: int
+    location: "CartesianPoint"
+    meta: Mapping[str, Any]
+
+
+class CartesianPoint(AttrDict[Any]):
+    """
+    :arg x: (required)
+    :arg y: (required)
+    """
+
+    x: float
+    y: float
+
+
+class ChangePointAggregate(AttrDict[Any]):
+    """
+    :arg type: (required)
+    :arg bucket:
+    :arg meta:
+    """
+
+    type: "ChangeType"
+    bucket: "ChangePointBucket"
+    meta: Mapping[str, Any]
+
+
+class ChangePointBucket(AttrDict[Any]):
+    """
+    :arg key: (required)
+    :arg doc_count: (required)
+    """
+
+    key: Union[int, float, str, bool, None]
+    doc_count: int
+
+
+class ChangeType(AttrDict[Any]):
+    """
+    :arg dip:
+    :arg distribution_change:
+    :arg indeterminable:
+    :arg non_stationary:
+    :arg spike:
+    :arg stationary:
+    :arg step_change:
+    :arg trend_change:
+    """
+
+    dip: "Dip"
+    distribution_change: "DistributionChange"
+    indeterminable: "Indeterminable"
+    non_stationary: "NonStationary"
+    spike: "Spike"
+    stationary: "Stationary"
+    step_change: "StepChange"
+    trend_change: "TrendChange"
+
+
 class ChildrenAggregate(AttrDict[Any]):
     """
     :arg doc_count: (required)
@@ -4936,6 +5040,26 @@ class DfsStatisticsProfile(AttrDict[Any]):
     children: Sequence["DfsStatisticsProfile"]
 
 
+class Dip(AttrDict[Any]):
+    """
+    :arg p_value: (required)
+    :arg change_point: (required)
+    """
+
+    p_value: float
+    change_point: int
+
+
+class DistributionChange(AttrDict[Any]):
+    """
+    :arg p_value: (required)
+    :arg change_point: (required)
+    """
+
+    p_value: float
+    change_point: int
+
+
 class DoubleTermsAggregate(AttrDict[Any]):
     """
     Result of a `terms` aggregation when the field is some kind of decimal
@@ -5497,6 +5621,14 @@ class HitsMetadata(AttrDict[Any]):
     max_score: Union[float, None]
 
 
+class Indeterminable(AttrDict[Any]):
+    """
+    :arg reason: (required)
+    """
+
+    reason: str
+
+
 class InferenceAggregate(AttrDict[Any]):
     """
     :arg value:
@@ -5899,6 +6031,18 @@ class NestedIdentity(AttrDict[Any]):
     _nested: "NestedIdentity"
 
 
+class NonStationary(AttrDict[Any]):
+    """
+    :arg p_value: (required)
+    :arg r_value: (required)
+    :arg trend: (required)
+    """
+
+    p_value: float
+    r_value: float
+    trend: str
+
+
 class ParentAggregate(AttrDict[Any]):
     """
     :arg doc_count: (required)
@@ -6256,6 +6400,16 @@ class SimpleValueAggregate(AttrDict[Any]):
     meta: Mapping[str, Any]
 
 
+class Spike(AttrDict[Any]):
+    """
+    :arg p_value: (required)
+    :arg change_point: (required)
+    """
+
+    p_value: float
+    change_point: int
+
+
 class StandardDeviationBounds(AttrDict[Any]):
     """
     :arg upper: (required)
@@ -6292,6 +6446,10 @@ class StandardDeviationBoundsAsString(AttrDict[Any]):
     lower_sampling: str
 
 
+class Stationary(AttrDict[Any]):
+    pass
+
+
 class StatsAggregate(AttrDict[Any]):
     """
     Statistics aggregation result. `min`, `max` and `avg` are missing if
@@ -6347,6 +6505,16 @@ class StatsBucketAggregate(AttrDict[Any]):
     meta: Mapping[str, Any]
 
 
+class StepChange(AttrDict[Any]):
+    """
+    :arg p_value: (required)
+    :arg change_point: (required)
+    """
+
+    p_value: float
+    change_point: int
+
+
 class StringRareTermsAggregate(AttrDict[Any]):
     """
     Result of the `rare_terms` aggregation when the field is a string.
@@ -6578,6 +6746,18 @@ class TotalHits(AttrDict[Any]):
     value: int
 
 
+class TrendChange(AttrDict[Any]):
+    """
+    :arg p_value: (required)
+    :arg r_value: (required)
+    :arg change_point: (required)
+    """
+
+    p_value: float
+    r_value: float
+    change_point: int
+
+
 class UnmappedRareTermsAggregate(AttrDict[Any]):
     """
     Result of a `rare_terms` aggregation when the field is unmapped.

elasticsearch9/dsl/utils.py

@@ -44,8 +44,7 @@ from .exceptions import UnknownDslObject, ValidationException
 if TYPE_CHECKING:
     from elastic_transport import ObjectApiResponse
 
-    from elasticsearch import AsyncElasticsearch, Elasticsearch
-
+    from .. import AsyncElasticsearch, Elasticsearch
     from .document_base import DocumentOptions
     from .field import Field
     from .index_base import IndexBase

elasticsearch9/esql/esql.py

@@ -124,7 +124,7 @@ class ESQLBase(ABC):
         if re.fullmatch(r"[a-zA-Z_@][a-zA-Z0-9_\.]*", s):
             return s
         # this identifier needs to be escaped
-        s.replace("`", "``")
+        s = s.replace("`", "``")
         return f"`{s}`"
 
     @staticmethod

elasticsearch9/esql/functions.py

@@ -18,8 +18,8 @@
 import json
 from typing import Any
 
-from elasticsearch.dsl.document_base import InstrumentedExpression
-from elasticsearch.esql.esql import ESQLBase, ExpressionType
+from ..dsl.document_base import InstrumentedExpression
+from ..esql.esql import ESQLBase, ExpressionType
 
 
 def _render(v: Any) -> str:

elasticsearch9/helpers/__init__.py

@@ -19,12 +19,21 @@ from .._async.helpers import async_bulk, async_reindex, async_scan, async_stream
 from .._utils import fixup_module_metadata
 from .actions import _chunk_actions  # noqa: F401
 from .actions import _process_bulk_chunk  # noqa: F401
-from .actions import bulk, expand_action, parallel_bulk, reindex, scan, streaming_bulk
+from .actions import (
+    BULK_FLUSH,
+    bulk,
+    expand_action,
+    parallel_bulk,
+    reindex,
+    scan,
+    streaming_bulk,
+)
 from .errors import BulkIndexError, ScanError
 
 __all__ = [
     "BulkIndexError",
     "ScanError",
+    "BULK_FLUSH",
     "expand_action",
     "streaming_bulk",
     "bulk",

elasticsearch9/helpers/actions.py

@@ -16,9 +16,10 @@
 #  under the License.
 
 import logging
+import queue
 import time
+from enum import Enum
 from operator import methodcaller
-from queue import Queue
 from typing import (
     Any,
     Callable,
@@ -37,13 +38,21 @@ from typing import (
 from elastic_transport import OpenTelemetrySpan
 
 from .. import Elasticsearch
-from ..compat import to_bytes
+from ..compat import safe_thread, to_bytes
 from ..exceptions import ApiError, NotFoundError, TransportError
 from ..serializer import Serializer
 from .errors import BulkIndexError, ScanError
 
 logger = logging.getLogger("elasticsearch.helpers")
 
+
+class BulkMeta(Enum):
+    flush = 1
+    done = 2
+
+
+BULK_FLUSH = BulkMeta.flush
+
 _TYPE_BULK_ACTION = Union[bytes, str, Dict[str, Any]]
 _TYPE_BULK_ACTION_HEADER = Dict[str, Any]
 _TYPE_BULK_ACTION_BODY = Union[None, bytes, Dict[str, Any]]
@@ -51,6 +60,13 @@ _TYPE_BULK_ACTION_HEADER_AND_BODY = Tuple[
     _TYPE_BULK_ACTION_HEADER, _TYPE_BULK_ACTION_BODY
 ]
 
+_TYPE_BULK_ACTION_WITH_META = Union[bytes, str, Dict[str, Any], BulkMeta]
+_TYPE_BULK_ACTION_HEADER_WITH_META = Union[Dict[str, Any], BulkMeta]
+_TYPE_BULK_ACTION_HEADER_WITH_META_AND_BODY = Union[
+    Tuple[_TYPE_BULK_ACTION_HEADER, _TYPE_BULK_ACTION_BODY],
+    Tuple[BulkMeta, Any],
+]
+
 
 def expand_action(data: _TYPE_BULK_ACTION) -> _TYPE_BULK_ACTION_HEADER_AND_BODY:
     """
@@ -139,7 +155,9 @@ class _ActionChunker:
         ] = []
 
     def feed(
-        self, action: _TYPE_BULK_ACTION_HEADER, data: _TYPE_BULK_ACTION_BODY
+        self,
+        action: _TYPE_BULK_ACTION_HEADER_WITH_META,
+        data: _TYPE_BULK_ACTION_BODY,
     ) -> Optional[
         Tuple[
             List[
@@ -152,23 +170,25 @@ class _ActionChunker:
         ]
     ]:
         ret = None
-        raw_action = action
-        raw_data = data
-        action_bytes = to_bytes(self.serializer.dumps(action), "utf-8")
-        # +1 to account for the trailing new line character
-        cur_size = len(action_bytes) + 1
-
-        data_bytes: Optional[bytes]
-        if data is not None:
-            data_bytes = to_bytes(self.serializer.dumps(data), "utf-8")
-            cur_size += len(data_bytes) + 1
-        else:
-            data_bytes = None
+        action_bytes = b""
+        data_bytes: Optional[bytes] = None
+        cur_size = 0
+        if not isinstance(action, BulkMeta):
+            action_bytes = to_bytes(self.serializer.dumps(action), "utf-8")
+            # +1 to account for the trailing new line character
+            cur_size = len(action_bytes) + 1
+
+            if data is not None:
+                data_bytes = to_bytes(self.serializer.dumps(data), "utf-8")
+                cur_size += len(data_bytes) + 1
+            else:
+                data_bytes = None
 
         # full chunk, send it and start a new one
         if self.bulk_actions and (
             self.size + cur_size > self.max_chunk_bytes
             or self.action_count == self.chunk_size
+            or (action == BulkMeta.flush and self.bulk_actions)
         ):
             ret = (self.bulk_data, self.bulk_actions)
             self.bulk_actions = []
@@ -176,15 +196,16 @@ class _ActionChunker:
             self.size = 0
             self.action_count = 0
 
-        self.bulk_actions.append(action_bytes)
-        if data_bytes is not None:
-            self.bulk_actions.append(data_bytes)
-            self.bulk_data.append((raw_action, raw_data))
-        else:
-            self.bulk_data.append((raw_action,))
+        if not isinstance(action, BulkMeta):
+            self.bulk_actions.append(action_bytes)
+            if data_bytes is not None:
+                self.bulk_actions.append(data_bytes)
+                self.bulk_data.append((action, data))
+            else:
+                self.bulk_data.append((action,))
 
-        self.size += cur_size
-        self.action_count += 1
+            self.size += cur_size
+            self.action_count += 1
         return ret
 
     def flush(
@@ -209,9 +230,10 @@ class _ActionChunker:
 
 
 def _chunk_actions(
-    actions: Iterable[_TYPE_BULK_ACTION_HEADER_AND_BODY],
+    actions: Iterable[_TYPE_BULK_ACTION_HEADER_WITH_META_AND_BODY],
     chunk_size: int,
     max_chunk_bytes: int,
+    flush_after_seconds: Optional[float],
     serializer: Serializer,
 ) -> Iterable[
     Tuple[
@@ -231,10 +253,41 @@ def _chunk_actions(
     chunker = _ActionChunker(
         chunk_size=chunk_size, max_chunk_bytes=max_chunk_bytes, serializer=serializer
     )
-    for action, data in actions:
-        ret = chunker.feed(action, data)
-        if ret:
-            yield ret
+
+    if not flush_after_seconds:
+        for action, data in actions:
+            ret = chunker.feed(action, data)
+            if ret:
+                yield ret
+    else:
+        item_queue: queue.Queue[_TYPE_BULK_ACTION_HEADER_WITH_META_AND_BODY] = (
+            queue.Queue()
+        )
+
+        def get_items() -> None:
+            try:
+                for item in actions:
+                    item_queue.put(item)
+            finally:
+                # make sure we signal the end even if there is an exception
+                item_queue.put((BulkMeta.done, None))
+
+        with safe_thread(get_items):
+            timeout: Optional[float] = flush_after_seconds
+            while True:
+                try:
+                    action, data = item_queue.get(timeout=timeout)
+                    timeout = flush_after_seconds
+                except queue.Empty:
+                    action, data = BulkMeta.flush, None
+                    timeout = None
+
+                if action is BulkMeta.done:
+                    break
+                ret = chunker.feed(action, data)
+                if ret:
+                    yield ret
+
     ret = chunker.flush()
     if ret:
         yield ret
@@ -361,9 +414,10 @@ def _process_bulk_chunk(
 
 def streaming_bulk(
     client: Elasticsearch,
-    actions: Iterable[_TYPE_BULK_ACTION],
+    actions: Iterable[_TYPE_BULK_ACTION_WITH_META],
     chunk_size: int = 500,
     max_chunk_bytes: int = 100 * 1024 * 1024,
+    flush_after_seconds: Optional[float] = None,
     raise_on_error: bool = True,
     expand_action_callback: Callable[
         [_TYPE_BULK_ACTION], _TYPE_BULK_ACTION_HEADER_AND_BODY
@@ -397,6 +451,9 @@ def streaming_bulk(
     :arg actions: iterable containing the actions to be executed
     :arg chunk_size: number of docs in one chunk sent to es (default: 500)
     :arg max_chunk_bytes: the maximum size of the request in bytes (default: 100MB)
+    :arg flush_after_seconds: time in seconds after which a chunk is written even
+        if hasn't reached `chunk_size` or `max_chunk_bytes`. Set to 0 to not use a
+        timeout-based flush. (default: 0)
     :arg raise_on_error: raise ``BulkIndexError`` containing errors (as `.errors`)
         from the execution of the last chunk when some occur. By default we raise.
     :arg raise_on_exception: if ``False`` then don't propagate exceptions from
@@ -425,6 +482,13 @@ def streaming_bulk(
 
         serializer = client.transport.serializers.get_serializer("application/json")
 
+        def expand_action_with_meta(
+            data: _TYPE_BULK_ACTION_WITH_META,
+        ) -> _TYPE_BULK_ACTION_HEADER_WITH_META_AND_BODY:
+            if isinstance(data, BulkMeta):
+                return data, None
+            return expand_action_callback(data)
+
         bulk_data: List[
             Union[
                 Tuple[_TYPE_BULK_ACTION_HEADER],
@@ -433,9 +497,10 @@ def streaming_bulk(
         ]
         bulk_actions: List[bytes]
         for bulk_data, bulk_actions in _chunk_actions(
-            map(expand_action_callback, actions),
+            map(expand_action_with_meta, actions),
             chunk_size,
             max_chunk_bytes,
+            flush_after_seconds,
             serializer,
         ):
             for attempt in range(max_retries + 1):
@@ -557,6 +622,7 @@ def parallel_bulk(
     thread_count: int = 4,
     chunk_size: int = 500,
     max_chunk_bytes: int = 100 * 1024 * 1024,
+    flush_after_seconds: Optional[float] = None,
     queue_size: int = 4,
     expand_action_callback: Callable[
         [_TYPE_BULK_ACTION], _TYPE_BULK_ACTION_HEADER_AND_BODY
@@ -573,6 +639,9 @@ def parallel_bulk(
     :arg thread_count: size of the threadpool to use for the bulk requests
     :arg chunk_size: number of docs in one chunk sent to es (default: 500)
     :arg max_chunk_bytes: the maximum size of the request in bytes (default: 100MB)
+    :arg flush_after_seconds: time in seconds after which a chunk is written even
+        if hasn't reached `chunk_size` or `max_chunk_bytes`. Set to 0 to not use a
+        timeout-based flush. (default: 0)
     :arg raise_on_error: raise ``BulkIndexError`` containing errors (as `.errors`)
         from the execution of the last chunk when some occur. By default we raise.
     :arg raise_on_exception: if ``False`` then don't propagate exceptions from
@@ -596,7 +665,7 @@ def parallel_bulk(
             super()._setup_queues()  # type: ignore[misc]
             # The queue must be at least the size of the number of threads to
             # prevent hanging when inserting sentinel values during teardown.
-            self._inqueue: Queue[
+            self._inqueue: queue.Queue[
                 Tuple[
                     List[
                         Union[
@@ -605,7 +674,7 @@ def parallel_bulk(
                     ],
                     List[bytes],
                 ]
-            ] = Queue(max(queue_size, thread_count))
+            ] = queue.Queue(max(queue_size, thread_count))
             self._quick_put = self._inqueue.put
 
     with client._otel.helpers_span("helpers.parallel_bulk") as otel_span:
@@ -625,7 +694,11 @@ def parallel_bulk(
                     )
                 ),
                 _chunk_actions(
-                    expanded_actions, chunk_size, max_chunk_bytes, serializer
+                    expanded_actions,
+                    chunk_size,
+                    max_chunk_bytes,
+                    flush_after_seconds,
+                    serializer,
                 ),
             ):
                 yield from result

elasticsearch9/helpers/vectorstore/__init__.py

@@ -15,31 +15,31 @@
 #  specific language governing permissions and limitations
 #  under the License.
 
-from elasticsearch.helpers.vectorstore._async.embedding_service import (
+from ...helpers.vectorstore._async.embedding_service import (
     AsyncElasticsearchEmbeddings,
     AsyncEmbeddingService,
 )
-from elasticsearch.helpers.vectorstore._async.strategies import (
+from ...helpers.vectorstore._async.strategies import (
     AsyncBM25Strategy,
     AsyncDenseVectorScriptScoreStrategy,
     AsyncDenseVectorStrategy,
     AsyncRetrievalStrategy,
     AsyncSparseVectorStrategy,
 )
-from elasticsearch.helpers.vectorstore._async.vectorstore import AsyncVectorStore
-from elasticsearch.helpers.vectorstore._sync.embedding_service import (
+from ...helpers.vectorstore._async.vectorstore import AsyncVectorStore
+from ...helpers.vectorstore._sync.embedding_service import (
     ElasticsearchEmbeddings,
     EmbeddingService,
 )
-from elasticsearch.helpers.vectorstore._sync.strategies import (
+from ...helpers.vectorstore._sync.strategies import (
     BM25Strategy,
     DenseVectorScriptScoreStrategy,
     DenseVectorStrategy,
     RetrievalStrategy,
     SparseVectorStrategy,
 )
-from elasticsearch.helpers.vectorstore._sync.vectorstore import VectorStore
-from elasticsearch.helpers.vectorstore._utils import DistanceMetric
+from ...helpers.vectorstore._sync.vectorstore import VectorStore
+from ...helpers.vectorstore._utils import DistanceMetric
 
 __all__ = [
     "AsyncBM25Strategy",

elasticsearch9/helpers/vectorstore/_async/_utils.py

@@ -15,7 +15,7 @@
 #  specific language governing permissions and limitations
 #  under the License.
 
-from elasticsearch import AsyncElasticsearch, BadRequestError, NotFoundError
+from .... import AsyncElasticsearch, BadRequestError, NotFoundError
 
 
 async def model_must_be_deployed(client: AsyncElasticsearch, model_id: str) -> None:

elasticsearch9/helpers/vectorstore/_async/embedding_service.py

@@ -18,8 +18,8 @@
 from abc import ABC, abstractmethod
 from typing import List
 
-from elasticsearch import AsyncElasticsearch
-from elasticsearch._version import __versionstr__ as lib_version
+from .... import AsyncElasticsearch
+from ...._version import __versionstr__ as lib_version
 
 
 class AsyncEmbeddingService(ABC):

elasticsearch9/helpers/vectorstore/_async/strategies.py

@@ -18,9 +18,9 @@
 from abc import ABC, abstractmethod
 from typing import Any, Dict, List, Optional, Tuple, Union, cast
 
-from elasticsearch import AsyncElasticsearch
-from elasticsearch.helpers.vectorstore._async._utils import model_must_be_deployed
-from elasticsearch.helpers.vectorstore._utils import DistanceMetric
+from .... import AsyncElasticsearch
+from ....helpers.vectorstore._async._utils import model_must_be_deployed
+from ....helpers.vectorstore._utils import DistanceMetric
 
 
 class AsyncRetrievalStrategy(ABC):