bigframes 1.42.0__tar.gz → 2.0.0.dev0__tar.gz

{bigframes-1.42.0/bigframes.egg-info → bigframes-2.0.0.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bigframes
-Version: 1.42.0
+Version: 2.0.0.dev0
 Summary: BigQuery DataFrames -- scalable analytics and machine learning with BigQuery
 Home-page: https://github.com/googleapis/python-bigquery-dataframes
 Author: Google LLC
@@ -20,6 +20,7 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Operating System :: OS Independent
 Classifier: Topic :: Internet
 Requires-Python: >=3.9
+Description-Content-Type: text/x-rst
 License-File: LICENSE
 Requires-Dist: cloudpickle>=2.0.0
 Requires-Dist: fsspec>=2023.3.0
@@ -28,7 +29,7 @@ Requires-Dist: geopandas>=0.12.2
 Requires-Dist: google-auth<3.0dev,>=2.15.0
 Requires-Dist: google-cloud-bigtable>=2.24.0
 Requires-Dist: google-cloud-pubsub>=2.21.4
-Requires-Dist: google-cloud-bigquery[bqstorage,pandas]>=3.18.0
+Requires-Dist: google-cloud-bigquery[bqstorage,pandas]>=3.31.0
 Requires-Dist: google-cloud-functions>=1.12.0
 Requires-Dist: google-cloud-bigquery-connection>=1.12.0
 Requires-Dist: google-cloud-iam>=2.12.1
@@ -37,7 +38,7 @@ Requires-Dist: google-cloud-storage>=2.0.0
 Requires-Dist: jellyfish<1.1.2,>=0.8.9
 Requires-Dist: numpy>=1.24.0
 Requires-Dist: pandas>=1.5.3
-Requires-Dist: pandas-gbq>=0.26.0
+Requires-Dist: pandas-gbq>=0.26.1
 Requires-Dist: pyarrow>=15.0.2
 Requires-Dist: pydata-google-auth>=1.8.2
 Requires-Dist: requests>=2.27.1
@@ -88,6 +89,30 @@ powered by the BigQuery engine.
 BigQuery DataFrames is an open-source package. You can run
 ``pip install --upgrade bigframes`` to install the latest version.
 
+⚠️ Warning: Breaking Changes in BigQuery DataFrames v2.0
+--------------------------------------------------------
+
+Version 2.0 introduces breaking changes for improved security and performance. Key default behaviors have changed, including
+
+* **Large Results (>10GB):** The default value for ``allow_large_results`` has changed to ``False``.
+  Methods like ``to_pandas()`` will now fail if the query result's compressed data size exceeds 10GB,
+  unless large results are explicitly permitted.
+* **Remote Function Security:** The library no longer automatically lets the Compute Engine default service
+  account become the identity of the Cloud Run functions. If that is desired, it has to be indicated by passing
+  ``cloud_function_service_account="default"``. And network ingress now defaults to ``"internal-only"``.
+* **@remote_function Argument Passing:** Arguments other than ``input_types``, ``output_type``, and ``dataset``
+  to ``remote_function`` must now be passed using keyword syntax, as positional arguments are no longer supported.
+* **Endpoint Connections:** Automatic fallback to locational endpoints in certain regions is removed.
+* **LLM Updates (Gemini Integration):** Integrations now default to the ``gemini-2.0-flash-001`` model.
+  PaLM2 support has been removed; please migrate any existing PaLM2 usage to Gemini. **Note:** The current default
+  model will be removed in Version 3.0.
+
+**Important:** If you are not ready to adapt to these changes, please pin your dependency to a version less than 2.0
+(e.g., ``bigframes==1.42.0``) to avoid disruption.
+
+To learn about these changes and how to migrate to version 2.0, see the
+`updated introduction guide <https://cloud.google.com/bigquery/docs/bigquery-dataframes-introduction>`_.
+
 .. |GA| image:: https://img.shields.io/badge/support-GA-gold.svg
    :target: https://github.com/googleapis/google-cloud-python/blob/main/README.rst#general-availability
 .. |pypi| image:: https://img.shields.io/pypi/v/bigframes.svg

{bigframes-1.42.0 → bigframes-2.0.0.dev0}/README.rst

@@ -12,6 +12,30 @@ powered by the BigQuery engine.
 BigQuery DataFrames is an open-source package. You can run
 ``pip install --upgrade bigframes`` to install the latest version.
 
+⚠️ Warning: Breaking Changes in BigQuery DataFrames v2.0
+--------------------------------------------------------
+
+Version 2.0 introduces breaking changes for improved security and performance. Key default behaviors have changed, including
+
+* **Large Results (>10GB):** The default value for ``allow_large_results`` has changed to ``False``.
+  Methods like ``to_pandas()`` will now fail if the query result's compressed data size exceeds 10GB,
+  unless large results are explicitly permitted.
+* **Remote Function Security:** The library no longer automatically lets the Compute Engine default service
+  account become the identity of the Cloud Run functions. If that is desired, it has to be indicated by passing
+  ``cloud_function_service_account="default"``. And network ingress now defaults to ``"internal-only"``.
+* **@remote_function Argument Passing:** Arguments other than ``input_types``, ``output_type``, and ``dataset``
+  to ``remote_function`` must now be passed using keyword syntax, as positional arguments are no longer supported.
+* **Endpoint Connections:** Automatic fallback to locational endpoints in certain regions is removed.
+* **LLM Updates (Gemini Integration):** Integrations now default to the ``gemini-2.0-flash-001`` model.
+  PaLM2 support has been removed; please migrate any existing PaLM2 usage to Gemini. **Note:** The current default
+  model will be removed in Version 3.0.
+
+**Important:** If you are not ready to adapt to these changes, please pin your dependency to a version less than 2.0
+(e.g., ``bigframes==1.42.0``) to avoid disruption.
+
+To learn about these changes and how to migrate to version 2.0, see the
+`updated introduction guide <https://cloud.google.com/bigquery/docs/bigquery-dataframes-introduction>`_.
+
 .. |GA| image:: https://img.shields.io/badge/support-GA-gold.svg
    :target: https://github.com/googleapis/google-cloud-python/blob/main/README.rst#general-availability
 .. |pypi| image:: https://img.shields.io/pypi/v/bigframes.svg

{bigframes-1.42.0 → bigframes-2.0.0.dev0}/bigframes/_config/bigquery_options.py

@@ -89,7 +89,7 @@ class BigQueryOptions:
         kms_key_name: Optional[str] = None,
         skip_bq_connection_check: bool = False,
         *,
-        allow_large_results: bool = True,
+        allow_large_results: bool = False,
         ordering_mode: Literal["strict", "partial"] = "strict",
         client_endpoints_override: Optional[dict] = None,
     ):
@@ -258,7 +258,8 @@ class BigQueryOptions:
 
     @property
     def use_regional_endpoints(self) -> bool:
-        """Flag to connect to regional API endpoints.
+        """Flag to connect to regional API endpoints for BigQuery API and
+        BigQuery Storage API.
 
         .. note::
             Use of regional endpoints is a feature in Preview and available only
@@ -267,18 +268,16 @@ class BigQueryOptions:
             "us-east5", "us-east7", "us-south1", "us-west1", "us-west2", "us-west3"
             and "us-west4".
 
-        .. deprecated:: 0.13.0
-            Use of locational endpoints is available only in selected projects.
-
-        Requires that ``location`` is set. For supported regions, for example
-        ``europe-west3``, you need to specify ``location='europe-west3'`` and
-        ``use_regional_endpoints=True``, and then BigQuery DataFrames would
-        connect to the BigQuery endpoint ``bigquery.europe-west3.rep.googleapis.com``.
-        For not supported regions, for example ``asia-northeast1``, when you
-        specify ``location='asia-northeast1'`` and ``use_regional_endpoints=True``,
-        a different endpoint (called locational endpoint, now deprecated, used
-        to provide weaker promise on the request remaining within the location
-        during transit) ``europe-west3-bigquery.googleapis.com`` would be used.
+        Requires that ``location`` is set. For [supported regions](https://cloud.google.com/bigquery/docs/regional-endpoints),
+        for example ``europe-west3``, you need to specify
+        ``location='europe-west3'`` and ``use_regional_endpoints=True``, and
+        then BigQuery DataFrames would connect to the BigQuery endpoint
+        ``bigquery.europe-west3.rep.googleapis.com``. For not supported regions,
+        for example ``asia-northeast1``, when you specify
+        ``location='asia-northeast1'`` and ``use_regional_endpoints=True``,
+        the global endpoint ``bigquery.googleapis.com`` would be used, which
+        does not promise any guarantee on the request remaining within the
+        location during transit.
 
         Returns:
             bool:

{bigframes-1.42.0 → bigframes-2.0.0.dev0}/bigframes/clients.py

@@ -17,6 +17,7 @@
 from __future__ import annotations
 
 import logging
+import textwrap
 import time
 from typing import cast, Optional
 
@@ -28,21 +29,46 @@ from google.iam.v1 import iam_policy_pb2, policy_pb2
 logger = logging.getLogger(__name__)
 
 
-def resolve_full_bq_connection_name(
-    connection_name: str, default_project: str, default_location: str
+def get_canonical_bq_connection_id(
+    connection_id: str, default_project: str, default_location: str
 ) -> str:
-    """Retrieve the full connection name of the form <PROJECT_NUMBER/PROJECT_ID>.<LOCATION>.<CONNECTION_ID>.
-    Use default project, location or connection_id when any of them are missing."""
-    if connection_name.count(".") == 2:
-        return connection_name
-
-    if connection_name.count(".") == 1:
-        return f"{default_project}.{connection_name}"
-
-    if connection_name.count(".") == 0:
-        return f"{default_project}.{default_location}.{connection_name}"
-
-    raise ValueError(f"Invalid connection name format: {connection_name}.")
+    """
+    Retrieve the full connection id of the form
+    <PROJECT_NUMBER/PROJECT_ID>.<LOCATION>.<CONNECTION_ID>.
+    Use default project, location or connection_id when any of them are missing.
+    """
+
+    if "/" in connection_id:
+        fields = connection_id.split("/")
+        if (
+            len(fields) == 6
+            and fields[0] == "projects"
+            and fields[2] == "locations"
+            and fields[4] == "connections"
+        ):
+            return ".".join((fields[1], fields[3], fields[5]))
+    else:
+        if connection_id.count(".") == 2:
+            return connection_id
+
+        if connection_id.count(".") == 1:
+            return f"{default_project}.{connection_id}"
+
+        if connection_id.count(".") == 0:
+            return f"{default_project}.{default_location}.{connection_id}"
+
+    raise ValueError(
+        textwrap.dedent(
+            f"""
+        Invalid connection id format: {connection_id}.
+        Only the following formats are supported:
+            <project-id>.<location>.<connection-id>,
+            <location>.<connection-id>,
+            <connection-id>,
+            projects/<project-id>/locations/<location>/connections/<connection-id>
+        """
+        ).strip()
+    )
 
 
 class BqConnectionManager:
@@ -80,7 +106,7 @@ class BqConnectionManager:
         )
         if service_account_id:
             logger.info(
-                f"Connector {project_id}.{location}.{connection_id} already exists"
+                f"BQ connection {project_id}.{location}.{connection_id} already exists"
             )
         else:
             connection_name, service_account_id = self._create_bq_connection(
@@ -90,9 +116,14 @@ class BqConnectionManager:
                 f"Created BQ connection {connection_name} with service account id: {service_account_id}"
             )
         service_account_id = cast(str, service_account_id)
+
         # Ensure IAM role on the BQ connection
         # https://cloud.google.com/bigquery/docs/reference/standard-sql/remote-functions#grant_permission_on_function
-        self._ensure_iam_binding(project_id, service_account_id, iam_role)
+        try:
+            self._ensure_iam_binding(project_id, service_account_id, iam_role)
+        except google.api_core.exceptions.PermissionDenied as ex:
+            ex.message = f"Failed ensuring IAM binding (role={iam_role}, service-account={service_account_id}). {ex.message}"
+            raise
 
     # Introduce retries to accommodate transient errors like:
     # (1) Etag mismatch,

{bigframes-1.42.0 → bigframes-2.0.0.dev0}/bigframes/constants.py

@@ -96,22 +96,27 @@ REP_ENABLED_BIGQUERY_LOCATIONS = frozenset(
     }
 )
 
-# https://cloud.google.com/storage/docs/locational-endpoints
-LEP_ENABLED_BIGQUERY_LOCATIONS = frozenset(
+REP_NOT_ENABLED_BIGQUERY_LOCATIONS = frozenset(
     ALL_BIGQUERY_LOCATIONS - REP_ENABLED_BIGQUERY_LOCATIONS
 )
 
-LEP_DEPRECATION_WARNING_MESSAGE = textwrap.dedent(
+LOCATION_NEEDED_FOR_REP_MESSAGE = textwrap.dedent(
     """
-    Support for regional endpoints is not yet available in the location
-    {location} for BigQuery and BigQuery Storage APIs. For the supported
-    locations and APIs see https://cloud.google.com/bigquery/docs/regional-endpoints.
-    For other locations and APIs, currently an older, now deprecated locational
-    endpoints are being used, which requires your project to be allowlisted. In
-    future version 2.0 onwards the locational endpoints will no longer be
-    supported automatically when you enable regional endpoints. However, if you
-    still need them, you will be able to override the endpoints directly by
-    doing the following:
+    Must set location to use regional endpoints.
+    You can do it via bigframaes.pandas.options.bigquery.location.
+    The supported locations can be found at
+    https://cloud.google.com/bigquery/docs/regional-endpoints#supported-locations.
+    """
+).strip()
+
+REP_NOT_SUPPORTED_MESSAGE = textwrap.dedent(
+    """
+    Support for regional endpoints for BigQuery and BigQuery Storage APIs may
+    not be available in the location {location}. For the supported APIs and
+    locations see https://cloud.google.com/bigquery/docs/regional-endpoints.
+    If you have the (deprecated) locational endpoints enabled in your project
+    (which requires your project to be allowlisted), you can override the
+    endpoints directly by doing the following:
         bigframes.pandas.options.bigquery.client_endpoints_override = {{
             "bqclient": "https://{location}-bigquery.googleapis.com",
             "bqconnectionclient": "{location}-bigqueryconnection.googleapis.com",

{bigframes-1.42.0 → bigframes-2.0.0.dev0}/bigframes/core/compile/aggregate_compiler.py

@@ -360,69 +360,73 @@ def _(
     if isinstance(op.bins, int):
         col_min = _apply_window_if_present(x.min(), window)
         col_max = _apply_window_if_present(x.max(), window)
+        adj = (col_max - col_min) * 0.001
         bin_width = (col_max - col_min) / op.bins
 
-        if op.labels is False:
-            for this_bin in range(op.bins - 1):
-                if op.right:
-                    case_expr = x <= (col_min + (this_bin + 1) * bin_width)
-                else:
-                    case_expr = x < (col_min + (this_bin + 1) * bin_width)
-                out = out.when(
-                    case_expr,
-                    compile_ibis_types.literal_to_ibis_scalar(
-                        this_bin, force_dtype=pd.Int64Dtype()
-                    ),
+        for this_bin in range(op.bins):
+            if op.labels is False:
+                value = compile_ibis_types.literal_to_ibis_scalar(
+                    this_bin,
+                    force_dtype=pd.Int64Dtype(),
                 )
-            out = out.when(x.notnull(), op.bins - 1)
-        else:
-            interval_struct = None
-            adj = (col_max - col_min) * 0.001
-            for this_bin in range(op.bins):
-                left_edge_adj = adj if this_bin == 0 and op.right else 0
-                right_edge_adj = adj if this_bin == op.bins - 1 and not op.right else 0
+            elif isinstance(op.labels, typing.Iterable):
+                value = compile_ibis_types.literal_to_ibis_scalar(
+                    list(op.labels)[this_bin],
+                    force_dtype=pd.StringDtype(storage="pyarrow"),
+                )
+            else:
+                left_adj = adj if this_bin == 0 and op.right else 0
+                right_adj = adj if this_bin == op.bins - 1 and not op.right else 0
 
-                left_edge = col_min + this_bin * bin_width - left_edge_adj
-                right_edge = col_min + (this_bin + 1) * bin_width + right_edge_adj
+                left = col_min + this_bin * bin_width - left_adj
+                right = col_min + (this_bin + 1) * bin_width + right_adj
 
                 if op.right:
-                    interval_struct = ibis_types.struct(
-                        {
-                            "left_exclusive": left_edge,
-                            "right_inclusive": right_edge,
-                        }
+                    value = ibis_types.struct(
+                        {"left_exclusive": left, "right_inclusive": right}
                     )
                 else:
-                    interval_struct = ibis_types.struct(
-                        {
-                            "left_inclusive": left_edge,
-                            "right_exclusive": right_edge,
-                        }
+                    value = ibis_types.struct(
+                        {"left_inclusive": left, "right_exclusive": right}
                     )
-
-                if this_bin < op.bins - 1:
-                    if op.right:
-                        case_expr = x <= (col_min + (this_bin + 1) * bin_width)
-                    else:
-                        case_expr = x < (col_min + (this_bin + 1) * bin_width)
-                    out = out.when(case_expr, interval_struct)
+            if this_bin == op.bins - 1:
+                case_expr = x.notnull()
+            else:
+                if op.right:
+                    case_expr = x <= (col_min + (this_bin + 1) * bin_width)
                 else:
-                    out = out.when(x.notnull(), interval_struct)
+                    case_expr = x < (col_min + (this_bin + 1) * bin_width)
+            out = out.when(case_expr, value)
     else:  # Interpret as intervals
-        for interval in op.bins:
+        for this_bin, interval in enumerate(op.bins):
             left = compile_ibis_types.literal_to_ibis_scalar(interval[0])
             right = compile_ibis_types.literal_to_ibis_scalar(interval[1])
             if op.right:
                 condition = (x > left) & (x <= right)
-                interval_struct = ibis_types.struct(
-                    {"left_exclusive": left, "right_inclusive": right}
-                )
             else:
                 condition = (x >= left) & (x < right)
-                interval_struct = ibis_types.struct(
-                    {"left_inclusive": left, "right_exclusive": right}
+
+            if op.labels is False:
+                value = compile_ibis_types.literal_to_ibis_scalar(
+                    this_bin,
+                    force_dtype=pd.Int64Dtype(),
                 )
-            out = out.when(condition, interval_struct)
+            elif isinstance(op.labels, typing.Iterable):
+                value = compile_ibis_types.literal_to_ibis_scalar(
+                    list(op.labels)[this_bin],
+                    force_dtype=pd.StringDtype(storage="pyarrow"),
+                )
+            else:
+                if op.right:
+                    value = ibis_types.struct(
+                        {"left_exclusive": left, "right_inclusive": right}
+                    )
+                else:
+                    value = ibis_types.struct(
+                        {"left_inclusive": left, "right_exclusive": right}
+                    )
+
+            out = out.when(condition, value)
     return out.end()
 
 

{bigframes-1.42.0 → bigframes-2.0.0.dev0}/bigframes/core/compile/ibis_types.py

@@ -388,7 +388,8 @@ def literal_to_ibis_scalar(
         # Ibis has bug for casting nulltype to geospatial, so we perform intermediate cast first
         geotype = ibis_dtypes.GeoSpatial(geotype="geography", srid=4326, nullable=True)
         return bigframes_vendored.ibis.literal(None, geotype)
-    ibis_dtype = BIGFRAMES_TO_IBIS[force_dtype] if force_dtype else None
+
+    ibis_dtype = bigframes_dtype_to_ibis_dtype(force_dtype) if force_dtype else None
 
     if pd.api.types.is_list_like(literal):
         if validate:

{bigframes-1.42.0 → bigframes-2.0.0.dev0}/bigframes/core/reshape/tile.py

@@ -20,6 +20,7 @@ import bigframes_vendored.constants as constants
 import bigframes_vendored.pandas.core.reshape.tile as vendored_pandas_tile
 import pandas as pd
 
+import bigframes.constants
 import bigframes.core.expression as ex
 import bigframes.core.ordering as order
 import bigframes.core.utils as utils
@@ -41,15 +42,37 @@ def cut(
     right: typing.Optional[bool] = True,
     labels: typing.Union[typing.Iterable[str], bool, None] = None,
 ) -> bigframes.series.Series:
-    if labels is not None and labels is not False:
+    if (
+        labels is not None
+        and labels is not False
+        and not isinstance(labels, typing.Iterable)
+    ):
+        raise ValueError(
+            "Bin labels must either be False, None or passed in as a list-like argument"
+        )
+    if (
+        isinstance(labels, typing.Iterable)
+        and len(list(labels)) > 0
+        and not isinstance(list(labels)[0], str)
+    ):
         raise NotImplementedError(
-            "The 'labels' parameter must be either False or None. "
-            "Please provide a valid value for 'labels'."
+            "When using an iterable for labels, only iterables of strings are supported "
+            f"but found {type(list(labels)[0])}. {constants.FEEDBACK_LINK}"
         )
 
+    if x.size == 0:
+        raise ValueError("Cannot cut empty array.")
+
     if isinstance(bins, int):
         if bins <= 0:
             raise ValueError("`bins` should be a positive integer.")
+        if isinstance(labels, typing.Iterable):
+            labels = tuple(labels)
+            if len(labels) != bins:
+                raise ValueError(
+                    f"Bin labels({len(labels)}) must be same as the value of bins({bins})"
+                )
+
         op = agg_ops.CutOp(bins, right=right, labels=labels)
         return x._apply_window_op(op, window_spec=window_specs.unbound())
     elif isinstance(bins, typing.Iterable):
@@ -58,6 +81,7 @@ def cut(
             bins = tuple((bin.left.item(), bin.right.item()) for bin in bins)
             # To maintain consistency with pandas' behavior
             right = True
+            labels = None
         elif len(list(bins)) == 0:
             as_index = pd.IntervalIndex.from_tuples(list(bins))
             bins = tuple()
@@ -66,6 +90,7 @@ def cut(
             bins = tuple(bins)
             # To maintain consistency with pandas' behavior
             right = True
+            labels = None
         elif pd.api.types.is_number(list(bins)[0]):
             bins_list = list(bins)
             as_index = pd.IntervalIndex.from_breaks(bins_list)
@@ -81,11 +106,24 @@ def cut(
             raise ValueError("`bins` iterable should contain tuples or numerics.")
 
         if as_index.is_overlapping:
-            raise ValueError("Overlapping IntervalIndex is not accepted.")
-        elif len(as_index) == 0:
-            op = agg_ops.CutOp(bins, right=right, labels=labels)
+            raise ValueError("Overlapping IntervalIndex is not accepted.")  # TODO: test
+
+        if isinstance(labels, typing.Iterable):
+            labels = tuple(labels)
+            if len(labels) != len(as_index):
+                raise ValueError(
+                    f"Bin labels({len(labels)}) must be same as the number of bin edges"
+                    f"({len(as_index)})"
+                )
+
+        if len(as_index) == 0:
+            dtype = agg_ops.CutOp(bins, right=right, labels=labels).output_type()
             return bigframes.series.Series(
-                [pd.NA] * len(x), dtype=op.output_type(), name=x.name
+                [pd.NA] * len(x),
+                dtype=dtype,
+                name=x.name,
+                index=x.index,
+                session=x._session,
             )
         else:
             op = agg_ops.CutOp(bins, right=right, labels=labels)

{bigframes-1.42.0 → bigframes-2.0.0.dev0}/bigframes/functions/_function_client.py

@@ -196,6 +196,7 @@ class FunctionClient:
         name,
         packages,
         is_row_processor,
+        bq_connection_id,
         *,
         capture_references=False,
     ):
@@ -273,12 +274,21 @@ class FunctionClient:
             udf_code = textwrap.dedent(inspect.getsource(func))
             udf_code = udf_code[udf_code.index("def") :]
 
+        with_connection_clause = (
+            (
+                f"WITH CONNECTION `{self._gcp_project_id}.{self._bq_location}.{self._bq_connection_id}`"
+            )
+            if bq_connection_id
+            else ""
+        )
+
         create_function_ddl = (
             textwrap.dedent(
                 f"""
                 CREATE OR REPLACE FUNCTION {persistent_func_id}({','.join(bq_function_args)})
                 RETURNS {bq_function_return_type}
                 LANGUAGE python
+                {with_connection_clause}
                 OPTIONS ({managed_function_options_str})
                 AS r'''
                 __UDF_PLACE_HOLDER__
@@ -365,7 +375,7 @@ class FunctionClient:
         is_row_processor=False,
         vpc_connector=None,
         memory_mib=1024,
-        ingress_settings="all",
+        ingress_settings="internal-only",
     ):
         """Create a cloud function from the given user defined function."""