google-cloud-bigtable 2.45.0__tar.gz → 2.46.0__tar.gz

{google_cloud_bigtable-2.45.0/google_cloud_bigtable.egg-info → google_cloud_bigtable-2.46.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: google-cloud-bigtable
-Version: 2.45.0
+Version: 2.46.0
 Summary: Google Cloud Bigtable API client library
 Home-page: https://github.com/googleapis/google-cloud-python/tree/main/packages/google-cloud-bigtable
 Author: Google LLC

{google_cloud_bigtable-2.45.0 → google_cloud_bigtable-2.46.0}/google/cloud/bigtable/data/_async/_read_rows.py

@@ -18,12 +18,12 @@ from __future__ import annotations
 import time
 from typing import TYPE_CHECKING, Sequence
 
-from google.api_core import retry as retries
 from grpc import StatusCode
 
 from google.cloud.bigtable.data._cross_sync import CrossSync
 from google.cloud.bigtable.data._helpers import (
     _attempt_timeout_generator,
+    _rst_stream_aware_predicate,
 )
 from google.cloud.bigtable.data._metrics.tracked_retry import tracked_retry
 from google.cloud.bigtable.data.exceptions import (
@@ -108,7 +108,7 @@ class _ReadRowsOperationAsync:
         else:
             self.request = query._to_pb(target)
         self.target = target
-        self._predicate = retries.if_exception_type(*retryable_exceptions)
+        self._predicate = _rst_stream_aware_predicate(*retryable_exceptions)
         self._last_yielded_row_key: bytes | None = None
         self._remaining_count: int | None = self.request.rows_limit or None
         self._operation_metric = metric

{google_cloud_bigtable-2.45.0 → google_cloud_bigtable-2.46.0}/google/cloud/bigtable/data/_helpers.py

@@ -20,9 +20,17 @@ from __future__ import annotations
 import enum
 import time
 from collections import namedtuple
-from typing import TYPE_CHECKING, List, Sequence, Tuple, Union
+from typing import (
+    TYPE_CHECKING,
+    Callable,
+    List,
+    Sequence,
+    Tuple,
+    Union,
+)
 
 from google.api_core import exceptions as core_exceptions
+from google.api_core import retry as retries
 from google.api_core.retry import RetryFailureReason, exponential_sleep_generator
 
 from google.cloud.bigtable.data.exceptions import RetryExceptionGroup
@@ -50,6 +58,15 @@ _CONCURRENCY_LIMIT = 10
 # used by every data client as a default project name for testing on Bigtable emulator.
 _DEFAULT_BIGTABLE_EMULATOR_CLIENT = "google-cloud-bigtable-emulator"
 
+# Internal error messages that can be retried during ReadRows. Internal error messages with this error
+# text should be treated as Unavailable error messages with the same error text, and will therefore be
+# treated as Unavailable errors rather than Internal errors.
+_RETRYABLE_INTERNAL_ERROR_MESSAGES = (
+    "rst_stream",
+    "rst stream",
+    "received unexpected eos on data frame from server",
+)
+
 # used to identify an active bigtable resource that needs to be warmed through PingAndWarm
 # each instance/app_profile_id pair needs to be individually tracked
 _WarmedInstanceKey = namedtuple(
@@ -126,6 +143,36 @@ def _retry_exception_factory(
     return source_exc, cause_exc
 
 
+def _rst_stream_aware_predicate(
+    *exception_types: type[Exception],
+) -> Callable[[Exception], bool]:
+    """A custom retry predicate.
+
+    This predicate treats Internal error messages with RST_STREAM errors as
+    ServiceUnavailable errors and will retry them if the Unavailable exception is retryable.
+
+    Args:
+        exception_types: Exception types to be retried during operation
+
+    Returns:
+        Callable[[Exception], bool]: A retry predicate that takes in an exception and
+            returns whether or not that exception is retryable
+    """
+    # predicate to check for retryable error types
+    if_exception_type = retries.if_exception_type(*exception_types)
+
+    # special case: treat InternalServerError with rst_stream error message as ServiceUnavailable
+    def rst_check(e):
+        return (
+            core_exceptions.ServiceUnavailable in exception_types
+            and isinstance(e, core_exceptions.InternalServerError)
+            and isinstance(e.message, str)
+            and any(m in e.message.lower() for m in _RETRYABLE_INTERNAL_ERROR_MESSAGES)
+        )
+
+    return lambda e: if_exception_type(e) or rst_check(e)
+
+
 def _get_timeouts(
     operation: float | TABLE_DEFAULT,
     attempt: float | None | TABLE_DEFAULT,

{google_cloud_bigtable-2.45.0 → google_cloud_bigtable-2.46.0}/google/cloud/bigtable/data/_sync_autogen/_read_rows.py

@@ -21,11 +21,13 @@ from __future__ import annotations
 import time
 from typing import TYPE_CHECKING, Sequence
 
-from google.api_core import retry as retries
 from grpc import StatusCode
 
 from google.cloud.bigtable.data._cross_sync import CrossSync
-from google.cloud.bigtable.data._helpers import _attempt_timeout_generator
+from google.cloud.bigtable.data._helpers import (
+    _attempt_timeout_generator,
+    _rst_stream_aware_predicate,
+)
 from google.cloud.bigtable.data._metrics.tracked_retry import tracked_retry
 from google.cloud.bigtable.data.exceptions import (
     InvalidChunk,
@@ -98,7 +100,7 @@ class _ReadRowsOperation:
         else:
             self.request = query._to_pb(target)
         self.target = target
-        self._predicate = retries.if_exception_type(*retryable_exceptions)
+        self._predicate = _rst_stream_aware_predicate(*retryable_exceptions)
         self._last_yielded_row_key: bytes | None = None
         self._remaining_count: int | None = self.request.rows_limit or None
         self._operation_metric = metric

{google_cloud_bigtable-2.45.0 → google_cloud_bigtable-2.46.0}/google/cloud/bigtable/data/exceptions.py

@@ -141,6 +141,8 @@ class _BigtableExceptionGroup(ExceptionGroup if is_311_plus else Exception):  #
         return f"{self.__class__.__name__}({message!r}, {self.exceptions!r})"
 
 
+# TODO: When working on mutations batcher, rework exception handling to guarantee that
+# MutationsExceptionGroup only stores FailedMutationEntryErrors.
 class MutationsExceptionGroup(_BigtableExceptionGroup):
     """
     Represents one or more exceptions that occur during a bulk mutation operation

{google_cloud_bigtable-2.45.0 → google_cloud_bigtable-2.46.0}/google/cloud/bigtable/gapic_version.py

@@ -13,4 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-__version__ = "2.45.0"  # {x-release-please-version}
+__version__ = "2.46.0"  # {x-release-please-version}

google_cloud_bigtable-2.46.0/google/cloud/bigtable/helpers.py

@@ -0,0 +1,62 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from itertools import islice
+from typing import Generator, Iterable, Tuple, TypeVar
+
+T = TypeVar("T")
+
+
+# batched landed in standard library in Python 3.11.
+def batched(iterable: Iterable[T], n) -> Generator[Tuple[T, ...], None, None]:
+    # batched('ABCDEFG', 3) → ABC DEF G
+    if n < 1:
+        raise ValueError("n must be at least one")
+    it = iter(iterable)
+    batch = tuple(islice(it, n))
+    while batch:
+        yield batch
+        batch = tuple(islice(it, n))
+
+
+class _MappableAttributesMixin:
+    """
+    Mixin for classes that need some of their attribute names remapped.
+
+    This is for taking some of the classes from the data client row filters
+    and row range classes that are 1:1 with their legacy client counterparts but with
+    some of their attributes renamed. To use in a class, override the base class with this mixin
+    class and define a map _attribute_map from legacy client attributes to data client
+    attributes.
+
+    Attributes are remapped and redefined in __init__ as well as getattr/setattr.
+    """
+
+    def __init__(self, *args, **kwargs):
+        new_kwargs = {self._attribute_map.get(k, k): v for (k, v) in kwargs.items()}
+        super(_MappableAttributesMixin, self).__init__(*args, **new_kwargs)
+
+    def __getattr__(self, name):
+        if name == "_attribute_map":
+            raise AttributeError
+        if name not in self._attribute_map:
+            raise AttributeError
+        return getattr(self, self._attribute_map[name])
+
+    def __setattr__(self, name, value):
+        if name == "_attribute_map":
+            super(_MappableAttributesMixin, self).__setattr__(name, value)
+            return
+        attribute = self._attribute_map.get(name, name)
+        super(_MappableAttributesMixin, self).__setattr__(attribute, value)

{google_cloud_bigtable-2.45.0 → google_cloud_bigtable-2.46.0}/google/cloud/bigtable/row.py

@@ -1009,6 +1009,16 @@ class PartialRowData(object):
         self._row_key = row_key
         self._cells = {}
 
+    @classmethod
+    def _from_data_client_row(cls, row):
+        partial_row_data = cls(row.row_key)
+        for column_family in row._index:
+            columns = {}
+            for column, items in row._index[column_family].items():
+                columns[column] = [Cell._from_data_client_cell(item) for item in items]
+            partial_row_data._cells[column_family] = columns
+        return partial_row_data
+
     def __eq__(self, other):
         if not isinstance(other, self.__class__):
             return NotImplemented
@@ -1212,6 +1222,14 @@ class Cell(object):
         else:
             return cls(cell_pb.value, cell_pb.timestamp_micros)
 
+    @classmethod
+    def _from_data_client_cell(cls, cell):
+        return cls(
+            cell.value,
+            cell.timestamp_micros,
+            cell.labels,
+        )
+
     @property
     def timestamp(self):
         return _datetime_from_microseconds(self.timestamp_micros)

google_cloud_bigtable-2.46.0/google/cloud/bigtable/row_data.py

@@ -0,0 +1,220 @@
+# Copyright 2016 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Container for Google Cloud Bigtable Cells and Streaming Row Contents."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Generator
+
+from google.api_core import exceptions, retry
+
+from google.cloud.bigtable.row import Cell, InvalidChunk, PartialRowData
+
+if TYPE_CHECKING:
+    from google.cloud.bigtable.data.row import Row
+
+# Some classes need to be re-exported here to keep backwards
+# compatibility. Those classes were moved to row_merger, but we dont want to
+# break enduser's imports. This hack, ensures they don't get marked as unused.
+_ = (Cell, InvalidChunk, PartialRowData)
+
+
+class PartialCellData(object):  # pragma: NO COVER
+    """This class is no longer used and will be removed in the future"""
+
+    def __init__(
+        self, row_key, family_name, qualifier, timestamp_micros, labels=(), value=b""
+    ):
+        self.row_key = row_key
+        self.family_name = family_name
+        self.qualifier = qualifier
+        self.timestamp_micros = timestamp_micros
+        self.labels = labels
+        self.value = value
+
+    def append_value(self, value):
+        self.value += value
+
+
+class InvalidReadRowsResponse(RuntimeError):
+    """Exception raised to invalid response data from back-end."""
+
+
+class InvalidRetryRequest(RuntimeError):
+    """Exception raised when retry request is invalid."""
+
+
+DEFAULT_RETRY_READ_ROWS = retry.Retry(
+    initial=1.0,
+    maximum=15.0,
+    multiplier=2.0,
+    deadline=60.0,  # 60 seconds
+)
+"""The default retry strategy to be used on retry-able errors.
+
+Used by
+:meth:`~google.cloud.bigtable.row_data.PartialRowsData._read_next_response`.
+"""
+
+
+class PartialRowsData(object):
+    """Convenience wrapper for consuming a ``ReadRows`` streaming response.
+
+    This class will be returned by the ``read_rows`` method, and should not
+    be constructed manually.
+
+    :type read_method: :class:`client._table_data_client.read_rows`
+    :param read_method: ``ReadRows`` method.
+
+    :type generator: :class:`Iterable[Row]`
+    :param generator: The `Row` iterator from :meth:`Table.read_rows`.
+    """
+
+    def __init__(
+        self,
+        *args: Any,
+        generator: Generator[Row, Any, Any] | None = None,
+        **kwargs: Any,
+    ) -> None:
+        self._generator: Generator[Row, Any, Any]
+        if generator is not None and not args and not kwargs:
+            self._generator = generator
+        else:
+            import warnings
+
+            warnings.warn(
+                "Direct instantiation of `PartialRowsData` is deprecated "
+                "and will be removed in a future release. `PartialRowsData` should only be obtained "
+                "by calling `Table.read_rows()`. An empty stream has been initialized.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+            self._generator = (x for x in ())
+
+        self.rows: dict[bytes, PartialRowData] = {}
+
+    @classmethod
+    def _from_generator(cls, generator: Generator[Row, Any, Any]) -> PartialRowsData:
+        """Internal constructor for Table.read_rows."""
+        return cls(generator=generator)
+
+    @property
+    def state(self):
+        import warnings
+
+        warnings.warn(
+            "The `state` attribute on `PartialRowsData` is deprecated "
+            "and will return None.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return None
+
+    @property
+    def last_scanned_row_key(self):
+        import warnings
+
+        warnings.warn(
+            "The `last_scanned_row_key` attribute on `PartialRowsData` is deprecated "
+            "and will return None.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return None
+
+    @property
+    def read_method(self):
+        import warnings
+
+        warnings.warn(
+            "The `read_method` attribute on `PartialRowsData` is deprecated "
+            "and will return None.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return None
+
+    @property
+    def request(self):
+        import warnings
+
+        warnings.warn(
+            "The `request` attribute on `PartialRowsData` is deprecated "
+            "and will return None.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return None
+
+    @property
+    def retry(self):
+        import warnings
+
+        warnings.warn(
+            "The `retry` attribute on `PartialRowsData` is deprecated "
+            "and will return None.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return None
+
+    @property
+    def response_iterator(self):
+        import warnings
+
+        warnings.warn(
+            "The `response_iterator` attribute on `PartialRowsData` is deprecated "
+            "and will return None.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return None
+
+    def cancel(self):
+        """Cancels the iterator, closing the stream."""
+        self._generator.close()
+
+    def consume_all(self, max_loops=None):
+        """Consume the streamed responses until there are no more.
+
+        .. warning::
+           This method will be removed in future releases.  Please use this
+           class as a generator instead.
+
+        :type max_loops: int
+        :param max_loops: (Deprecated). Maximum number of times to try to consume
+                          an additional ``ReadRowsResponse``. This parameter is
+                          deprecated and is only kept for backwards compatibility.
+        """
+        for row in self:
+            self.rows[row.row_key] = row
+
+    def __iter__(self):
+        """Consume the ``Row`` s from the stream.
+        Convert them to ``PartialRowData`` and yield each to the reader.
+        """
+        try:
+            for row in self._generator:
+                yield PartialRowData._from_data_client_row(row)
+
+        # Any exception from the generator should cancel the iterator. A
+        # timeout, defined by catching a DeadlineExceeded, should be reraised
+        # as a RetryError instead.
+        except exceptions.DeadlineExceeded as e:
+            self.cancel()
+            raise exceptions.RetryError(e.message, e.__cause__)
+        except Exception as e:
+            self.cancel()
+            raise e

{google_cloud_bigtable-2.45.0 → google_cloud_bigtable-2.46.0}/google/cloud/bigtable/row_filters.py

@@ -50,6 +50,7 @@ from google.cloud.bigtable.data.row_filters import (
 from google.cloud.bigtable.data.row_filters import (
     TimestampRangeFilter as BaseTimestampRangeFilter,
 )
+from google.cloud.bigtable.helpers import _MappableAttributesMixin
 
 _PACK_I64 = struct.Struct(">q").pack
 
@@ -70,35 +71,6 @@ def _deprecated_to_pb(self):
 # Provide to_pb alias with DeprecationWarning for backwards compatibility with legacy client code
 RowFilter.to_pb = _deprecated_to_pb  # type: ignore[attr-defined]
 TimestampRange.to_pb = _deprecated_to_pb  # type: ignore[attr-defined]
-
-
-class _MappableAttributesMixin:
-    """
-    Mixin for classes that need some of their attribute names remapped.
-
-    This is for taking some of the classes from the data client row filters
-    that are 1:1 with their legacy client counterparts but with some of their
-    attributes renamed. To use in a class, override the base class with this mixin
-    class and define a map _attribute_map from legacy client attributes to data client
-    attributes.
-
-    Attributes are remapped and redefined in __init__ as well as getattr/setattr.
-    """
-
-    def __init__(self, *args, **kwargs):
-        new_kwargs = {self._attribute_map.get(k, k): v for (k, v) in kwargs.items()}
-        super(_MappableAttributesMixin, self).__init__(*args, **new_kwargs)
-
-    def __getattr__(self, name):
-        if name not in self._attribute_map:
-            raise AttributeError
-        return getattr(self, self._attribute_map[name])
-
-    def __setattr__(self, name, value):
-        attribute = self._attribute_map.get(name, name)
-        super(_MappableAttributesMixin, self).__setattr__(attribute, value)
-
-
 # The classes defined below are to provide constructors and members
 # that have an interface that does not match the one used by the data
 # client, for backwards compatibility purposes.

{google_cloud_bigtable-2.45.0 → google_cloud_bigtable-2.46.0}/google/cloud/bigtable/row_set.py

@@ -14,7 +14,13 @@
 
 """User-friendly container for Google Cloud Bigtable RowSet"""
 
-from google.cloud._helpers import _to_bytes  # type: ignore
+from google.cloud.bigtable.data.read_rows_query import (
+    ReadRowsQuery,
+)
+from google.cloud.bigtable.data.read_rows_query import (
+    RowRange as BaseRowRange,
+)
+from google.cloud.bigtable.helpers import _MappableAttributesMixin
 
 
 class RowSet(object):
@@ -25,30 +31,25 @@ class RowSet(object):
     """
 
     def __init__(self):
-        self.row_keys = []
-        self.row_ranges = []
+        self._read_rows_query = ReadRowsQuery()
 
     def __eq__(self, other):
         if not isinstance(other, self.__class__):
             return NotImplemented
 
-        if len(other.row_keys) != len(self.row_keys):
-            return False
-
-        if len(other.row_ranges) != len(self.row_ranges):
-            return False
-
-        if not set(other.row_keys) == set(self.row_keys):
-            return False
-
-        if not set(other.row_ranges) == set(self.row_ranges):
-            return False
-
-        return True
+        return self._read_rows_query == other._read_rows_query
 
     def __ne__(self, other):
         return not self == other
 
+    @property
+    def row_keys(self):
+        return self._read_rows_query.row_keys
+
+    @property
+    def row_ranges(self):
+        return self._read_rows_query.row_ranges
+
     def add_row_key(self, row_key):
         """Add row key to row_keys list.
 
@@ -62,7 +63,7 @@ class RowSet(object):
         :type row_key: bytes
         :param row_key: The key of a row to read
         """
-        self.row_keys.append(row_key)
+        self._read_rows_query.add_key(row_key)
 
     def add_row_range(self, row_range):
         """Add row_range to row_ranges list.
@@ -77,7 +78,7 @@ class RowSet(object):
         :type row_range: class:`RowRange`
         :param row_range: The row range object having start and end key
         """
-        self.row_ranges.append(row_range)
+        self._read_rows_query.add_range(row_range)
 
     def add_row_range_from_keys(
         self, start_key=None, end_key=None, start_inclusive=True, end_inclusive=False
@@ -109,7 +110,7 @@ class RowSet(object):
                   considered inclusive. The default is False (exclusive).
         """
         row_range = RowRange(start_key, end_key, start_inclusive, end_inclusive)
-        self.row_ranges.append(row_range)
+        self._read_rows_query.add_range(row_range)
 
     def add_row_range_with_prefix(self, row_key_prefix):
         """Add row range to row_ranges list that start with the row_key_prefix from the row keys
@@ -129,28 +130,15 @@ class RowSet(object):
             row_key_prefix.encode("utf-8"), end_key.encode("utf-8")
         )
 
-    def _update_message_request(self, message):
-        """Add row keys and row range to given request message
 
-        :type message: class:`data_messages_v2_pb2.ReadRowsRequest`
-        :param message: The ``ReadRowsRequest`` protobuf
-        """
-        for each in self.row_keys:
-            message.rows.row_keys._pb.append(_to_bytes(each))
-
-        for each in self.row_ranges:
-            r_kwrags = each.get_range_kwargs()
-            message.rows.row_ranges.append(r_kwrags)
-
-
-class RowRange(object):
+class RowRange(_MappableAttributesMixin, BaseRowRange):
     """Convenience wrapper of google.bigtable.v2.RowRange
 
-    :type start_key: bytes
+    :type start_key: str | bytes
     :param start_key: (Optional) Start key of the row range. If left empty,
                       will be interpreted as the empty string.
 
-    :type end_key: bytes
+    :type end_key: str | bytes
     :param end_key: (Optional) End key of the row range. If left empty,
                     will be interpreted as the empty string and range will
                     be unbounded on the high end.
@@ -164,49 +152,31 @@ class RowRange(object):
                   considered inclusive. The default is False (exclusive).
     """
 
-    def __init__(
-        self, start_key=None, end_key=None, start_inclusive=True, end_inclusive=False
-    ):
-        self.start_key = start_key
-        self.start_inclusive = start_inclusive
-        self.end_key = end_key
-        self.end_inclusive = end_inclusive
+    _attribute_map = {
+        "start_inclusive": "start_is_inclusive",
+        "end_inclusive": "end_is_inclusive",
+    }
 
     def _key(self):
-        """A tuple key that uniquely describes this field.
-
-        Used to compute this instance's hashcode and evaluate equality.
-
-        Returns:
-            Tuple[str]: The contents of this :class:`.RowRange`.
-        """
-        return (self.start_key, self.start_inclusive, self.end_key, self.end_inclusive)
+        return (
+            self.start_key,
+            self.end_key,
+            self.start_is_inclusive,
+            self.end_is_inclusive,
+        )
 
     def __hash__(self):
         return hash(self._key())
 
-    def __eq__(self, other):
-        if not isinstance(other, self.__class__):
-            return NotImplemented
-        return self._key() == other._key()
-
-    def __ne__(self, other):
-        return not self == other
-
     def get_range_kwargs(self):
         """Convert row range object to dict which can be passed to
         google.bigtable.v2.RowRange add method.
         """
         range_kwargs = {}
         if self.start_key is not None:
-            start_key_key = "start_key_open"
-            if self.start_inclusive:
-                start_key_key = "start_key_closed"
-            range_kwargs[start_key_key] = _to_bytes(self.start_key)
-
+            key = "start_key_closed" if self.start_is_inclusive else "start_key_open"
+            range_kwargs[key] = self.start_key
         if self.end_key is not None:
-            end_key_key = "end_key_open"
-            if self.end_inclusive:
-                end_key_key = "end_key_closed"
-            range_kwargs[end_key_key] = _to_bytes(self.end_key)
+            key = "end_key_closed" if self.end_is_inclusive else "end_key_open"
+            range_kwargs[key] = self.end_key
         return range_kwargs