bigframes 2.2.0__tar.gz → 2.4.0__tar.gz

{bigframes-2.2.0/bigframes.egg-info → bigframes-2.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bigframes
-Version: 2.2.0
+Version: 2.4.0
 Summary: BigQuery DataFrames -- scalable analytics and machine learning with BigQuery
 Home-page: https://github.com/googleapis/python-bigquery-dataframes
 Author: Google LLC
@@ -27,8 +27,6 @@ Requires-Dist: fsspec>=2023.3.0
 Requires-Dist: gcsfs>=2023.3.0
 Requires-Dist: geopandas>=0.12.2
 Requires-Dist: google-auth<3.0,>=2.15.0
-Requires-Dist: google-cloud-bigtable>=2.24.0
-Requires-Dist: google-cloud-pubsub>=2.21.4
 Requires-Dist: google-cloud-bigquery[bqstorage,pandas]>=3.31.0
 Requires-Dist: google-cloud-bigquery-storage<3.0.0,>=2.30.0
 Requires-Dist: google-cloud-functions>=1.12.0
@@ -58,6 +56,8 @@ Requires-Dist: rich<14,>=12.4.4
 Provides-Extra: tests
 Requires-Dist: freezegun; extra == "tests"
 Requires-Dist: pytest-snapshot; extra == "tests"
+Requires-Dist: google-cloud-bigtable>=2.24.0; extra == "tests"
+Requires-Dist: google-cloud-pubsub>=2.21.4; extra == "tests"
 Provides-Extra: polars
 Requires-Dist: polars>=1.7.0; extra == "polars"
 Provides-Extra: scikit-learn
@@ -69,6 +69,8 @@ Requires-Dist: nox; extra == "dev"
 Requires-Dist: google-cloud-testutils; extra == "dev"
 Provides-Extra: all
 Requires-Dist: freezegun; extra == "all"
+Requires-Dist: google-cloud-bigtable>=2.24.0; extra == "all"
+Requires-Dist: google-cloud-pubsub>=2.21.4; extra == "all"
 Requires-Dist: google-cloud-testutils; extra == "all"
 Requires-Dist: nox; extra == "all"
 Requires-Dist: polars>=1.7.0; extra == "all"

{bigframes-2.2.0 → bigframes-2.4.0}/bigframes/_config/__init__.py

@@ -150,6 +150,24 @@ class Options:
         """
         return self._local.bigquery_options is not None
 
+    @property
+    def _allow_large_results(self) -> bool:
+        """The effective 'allow_large_results' setting.
+
+        This value is `self.compute.allow_large_results` if set (not `None`),
+        otherwise it defaults to `self.bigquery.allow_large_results`.
+
+        Returns:
+            bool:
+                Whether large query results are permitted.
+                - `True`: The BigQuery result size limit (e.g., 10 GB) is removed.
+                - `False`: Results are restricted to this limit (potentially faster).
+                BigQuery will raise an error if this limit is exceeded.
+        """
+        if self.compute.allow_large_results is None:
+            return self.bigquery.allow_large_results
+        return self.compute.allow_large_results
+
 
 options = Options()
 """Global options for default session."""

{bigframes-2.2.0 → bigframes-2.4.0}/bigframes/_config/bigquery_options.py

@@ -19,10 +19,8 @@ from __future__ import annotations
 from typing import Literal, Optional
 import warnings
 
-import google.api_core.exceptions
 import google.auth.credentials
 
-import bigframes.constants
 import bigframes.enums
 import bigframes.exceptions as bfe
 
@@ -239,21 +237,34 @@ class BigQueryOptions:
     @property
     def allow_large_results(self) -> bool:
         """
-        Sets the flag to allow or disallow query results larger than 10 GB.
+        DEPRECATED: Checks the legacy global setting for allowing large results.
+        Use ``bpd.options.compute.allow_large_results`` instead.
 
-        The default setting for this flag is True, which allows queries to return results
-        exceeding 10 GB by creating an explicit destination table. If set to False, it
-        restricts the result size to 10 GB, and BigQuery will raise an error if this limit
-        is exceeded.
+        Warning: Accessing ``bpd.options.bigquery.allow_large_results`` is deprecated
+        and this property will be removed in a future version. The configuration for
+        handling large results has moved.
 
         Returns:
-            bool: True if large results are allowed with an explicit destination table,
-            False if results are limited to 10 GB and errors are raised when exceeded.
+            bool: The value of the deprecated setting.
         """
         return self._allow_large_results
 
     @allow_large_results.setter
     def allow_large_results(self, value: bool):
+        warnings.warn(
+            "Setting `bpd.options.bigquery.allow_large_results` is deprecated, "
+            "and will be removed in the future. "
+            "Please use `bpd.options.compute.allow_large_results = <value>` instead. "
+            "The `bpd.options.bigquery.allow_large_results` option is ignored if "
+            "`bpd.options.compute.allow_large_results` is set.",
+            FutureWarning,
+            stacklevel=2,
+        )
+        if self._session_started and self._allow_large_results != value:
+            raise ValueError(
+                SESSION_STARTED_MESSAGE.format(attribute="allow_large_results")
+            )
+
         self._allow_large_results = value
 
     @property

{bigframes-2.2.0 → bigframes-2.4.0}/bigframes/_config/compute_options.py

@@ -86,6 +86,12 @@ class ComputeOptions:
         ai_ops_threshold_autofail (bool):
             Guards against unexpected processing of large amount of rows by semantic operators.
             When set to True, the operation automatically fails without asking for user inputs.
+
+        allow_large_results (bool):
+            Specifies whether query results can exceed 10 GB. Defaults to False. Setting this
+            to False (the default) restricts results to 10 GB for potentially faster execution;
+            BigQuery will raise an error if this limit is exceeded. Setting to True removes
+            this result size limit.
     """
 
     maximum_bytes_billed: Optional[int] = None
@@ -97,7 +103,9 @@ class ComputeOptions:
     semantic_ops_threshold_autofail = False
 
     ai_ops_confirmation_threshold: Optional[int] = 0
-    ai_ops_threshold_autofail = False
+    ai_ops_threshold_autofail: bool = False
+
+    allow_large_results: Optional[bool] = None
 
     def assign_extra_query_labels(self, **kwargs: Any) -> None:
         """

{bigframes-2.2.0 → bigframes-2.4.0}/bigframes/_config/display_options.py

@@ -35,6 +35,10 @@ class DisplayOptions:
     max_info_rows: Optional[int] = 200000
     memory_usage: bool = True
 
+    blob_display: bool = True
+    blob_display_width: Optional[int] = None
+    blob_display_height: Optional[int] = None
+
 
 @contextlib.contextmanager
 def pandas_repr(display_options: DisplayOptions):

{bigframes-2.2.0 → bigframes-2.4.0}/bigframes/_config/experiment_options.py

@@ -15,6 +15,7 @@
 from typing import Optional
 import warnings
 
+import bigframes
 import bigframes.exceptions as bfe
 
 
@@ -26,10 +27,6 @@ class ExperimentOptions:
     def __init__(self):
         self._semantic_operators: bool = False
         self._ai_operators: bool = False
-        self._blob: bool = False
-        self._blob_display: bool = True
-        self._blob_display_width: Optional[int] = None
-        self._blob_display_height: Optional[int] = None
 
     @property
     def semantic_operators(self) -> bool:
@@ -60,41 +57,72 @@ class ExperimentOptions:
 
     @property
     def blob(self) -> bool:
-        return self._blob
+        msg = bfe.format_message(
+            "BigFrames Blob is in preview now. This flag is no longer needed."
+        )
+        warnings.warn(msg, category=bfe.ApiDeprecationWarning)
+        return True
 
     @blob.setter
     def blob(self, value: bool):
-        if value is True:
-            msg = bfe.format_message(
-                "BigFrames Blob is still under experiments. It may not work and "
-                "subject to change in the future."
-            )
-            warnings.warn(msg, category=bfe.PreviewWarning)
-        self._blob = value
+        msg = bfe.format_message(
+            "BigFrames Blob is in preview now. This flag is no longer needed."
+        )
+        warnings.warn(msg, category=bfe.ApiDeprecationWarning)
 
     @property
     def blob_display(self) -> bool:
         """Whether to display the blob content in notebook DataFrame preview. Default True."""
-        return self._blob_display
+        msg = bfe.format_message(
+            "BigFrames Blob is in preview now. The option has been moved to bigframes.options.display.blob_display."
+        )
+        warnings.warn(msg, category=bfe.ApiDeprecationWarning)
+
+        return bigframes.options.display.blob_display
 
     @blob_display.setter
     def blob_display(self, value: bool):
-        self._blob_display = value
+        msg = bfe.format_message(
+            "BigFrames Blob is in preview now. The option has been moved to bigframes.options.display.blob_display."
+        )
+        warnings.warn(msg, category=bfe.ApiDeprecationWarning)
+
+        bigframes.options.display.blob_display = value
 
     @property
     def blob_display_width(self) -> Optional[int]:
         """Width in pixels that the blob constrained to."""
-        return self._blob_display_width
+        msg = bfe.format_message(
+            "BigFrames Blob is in preview now. The option has been moved to bigframes.options.display.blob_display_width."
+        )
+        warnings.warn(msg, category=bfe.ApiDeprecationWarning)
+
+        return bigframes.options.display.blob_display_width
 
     @blob_display_width.setter
     def blob_display_width(self, value: Optional[int]):
-        self._blob_display_width = value
+        msg = bfe.format_message(
+            "BigFrames Blob is in preview now. The option has been moved to bigframes.options.display.blob_display_width."
+        )
+        warnings.warn(msg, category=bfe.ApiDeprecationWarning)
+
+        bigframes.options.display.blob_display_width = value
 
     @property
     def blob_display_height(self) -> Optional[int]:
         """Height in pixels that the blob constrained to."""
-        return self._blob_display_height
+        msg = bfe.format_message(
+            "BigFrames Blob is in preview now. The option has been moved to bigframes.options.display.blob_display_height."
+        )
+        warnings.warn(msg, category=bfe.ApiDeprecationWarning)
+
+        return bigframes.options.display.blob_display_height
 
     @blob_display_height.setter
     def blob_display_height(self, value: Optional[int]):
-        self._blob_display_height = value
+        msg = bfe.format_message(
+            "BigFrames Blob is in preview now. The option has been moved to bigframes.options.display.blob_display_height."
+        )
+        warnings.warn(msg, category=bfe.ApiDeprecationWarning)
+
+        bigframes.options.display.blob_display_height = value

{bigframes-2.2.0 → bigframes-2.4.0}/bigframes/bigquery/__init__.py

@@ -38,6 +38,7 @@ from bigframes.bigquery._operations.json import (
     json_extract_array,
     json_extract_string_array,
     json_set,
+    json_value,
     parse_json,
 )
 from bigframes.bigquery._operations.search import create_vector_index, vector_search
@@ -61,6 +62,7 @@ __all__ = [
     "json_extract",
     "json_extract_array",
     "json_extract_string_array",
+    "json_value",
     "parse_json",
     # search ops
     "create_vector_index",

{bigframes-2.2.0 → bigframes-2.4.0}/bigframes/bigquery/_operations/json.py

@@ -231,6 +231,40 @@ def json_extract_string_array(
     return array_series
 
 
+def json_value(
+    input: series.Series,
+    json_path: str,
+) -> series.Series:
+    """Extracts a JSON scalar value and converts it to a SQL ``STRING`` value. In
+    addtion, this function:
+    - Removes the outermost quotes and unescapes the values.
+    - Returns a SQL ``NULL`` if a non-scalar value is selected.
+    - Uses double quotes to escape invalid ``JSON_PATH`` characters in JSON keys.
+
+    **Examples:**
+
+        >>> import bigframes.pandas as bpd
+        >>> import bigframes.bigquery as bbq
+        >>> bpd.options.display.progress_bar = None
+
+        >>> s = bpd.Series(['{"name": "Jakob", "age": "6"}', '{"name": "Jakob", "age": []}'])
+        >>> bbq.json_value(s, json_path="$.age")
+        0    6
+        1  <NA>
+        dtype: string
+
+    Args:
+        input (bigframes.series.Series):
+            The Series containing JSON data (as native JSON objects or JSON-formatted strings).
+        json_path (str):
+            The JSON path identifying the data that you want to obtain from the input.
+
+    Returns:
+        bigframes.series.Series: A new Series with the JSON-formatted STRING.
+    """
+    return input._apply_unary_op(ops.JSONValue(json_path=json_path))
+
+
 @utils.preview(name="The JSON-related API `parse_json`")
 def parse_json(
     input: series.Series,

{bigframes-2.2.0 → bigframes-2.4.0}/bigframes/blob/_functions.py

@@ -108,6 +108,38 @@ AS r\"\"\"
         return self._session.read_gbq_function(udf_name)
 
 
+def exif_func(src_obj_ref_rt: str) -> str:
+    import io
+    import json
+
+    from PIL import ExifTags, Image
+    import requests
+    from requests import adapters
+
+    session = requests.Session()
+    session.mount("https://", adapters.HTTPAdapter(max_retries=3))
+
+    src_obj_ref_rt_json = json.loads(src_obj_ref_rt)
+
+    src_url = src_obj_ref_rt_json["access_urls"]["read_url"]
+
+    response = session.get(src_url, timeout=30)
+    bts = response.content
+
+    image = Image.open(io.BytesIO(bts))
+    exif_data = image.getexif()
+    exif_dict = {}
+    if exif_data:
+        for tag, value in exif_data.items():
+            tag_name = ExifTags.TAGS.get(tag, tag)
+            exif_dict[tag_name] = value
+
+    return json.dumps(exif_dict)
+
+
+exif_func_def = FunctionDef(exif_func, ["pillow", "requests"])
+
+
 # Blur images. Takes ObjectRefRuntime as JSON string. Outputs ObjectRefRuntime JSON string.
 def image_blur_func(
     src_obj_ref_rt: str, dst_obj_ref_rt: str, ksize_x: int, ksize_y: int, ext: str

{bigframes-2.2.0 → bigframes-2.4.0}/bigframes/core/array_value.py

@@ -172,33 +172,6 @@ class ArrayValue:
     def supports_fast_peek(self) -> bool:
         return bigframes.core.tree_properties.can_fast_peek(self.node)
 
-    def as_cached(
-        self: ArrayValue,
-        cache_table: google.cloud.bigquery.Table,
-        ordering: Optional[orderings.RowOrdering],
-    ) -> ArrayValue:
-        """
-        Replace the node with an equivalent one that references a table where the value has been materialized to.
-        """
-        table = nodes.GbqTable.from_table(cache_table)
-        source = nodes.BigqueryDataSource(
-            table, ordering=ordering, n_rows=cache_table.num_rows
-        )
-        # Assumption: GBQ cached table uses field name as bq column name
-        scan_list = nodes.ScanList(
-            tuple(
-                nodes.ScanItem(field.id, field.dtype, field.id.name)
-                for field in self.node.fields
-            )
-        )
-        node = nodes.CachedTableNode(
-            original_node=self.node,
-            source=source,
-            table_session=self.session,
-            scan_list=scan_list,
-        )
-        return ArrayValue(node)
-
     def get_column_type(self, key: str) -> bigframes.dtypes.Dtype:
         return self.schema.get_type(key)
 

{bigframes-2.2.0 → bigframes-2.4.0}/bigframes/core/blocks.py

@@ -22,7 +22,6 @@ circular dependencies.
 from __future__ import annotations
 
 import ast
-import copy
 import dataclasses
 import datetime
 import functools
@@ -30,17 +29,7 @@ import itertools
 import random
 import textwrap
 import typing
-from typing import (
-    Any,
-    Iterable,
-    List,
-    Literal,
-    Mapping,
-    Optional,
-    Sequence,
-    Tuple,
-    Union,
-)
+from typing import Iterable, List, Literal, Mapping, Optional, Sequence, Tuple, Union
 import warnings
 
 import bigframes_vendored.constants as constants
@@ -69,6 +58,8 @@ import bigframes.dtypes
 import bigframes.exceptions as bfe
 import bigframes.operations as ops
 import bigframes.operations.aggregations as agg_ops
+from bigframes.session import dry_runs
+from bigframes.session import executor as executors
 
 # Type constraint for wherever column labels are used
 Label = typing.Hashable
@@ -595,10 +586,10 @@ class Block:
             self.expr,
             ordered=True,
             use_explicit_destination=allow_large_results,
-            page_size=page_size,
-            max_results=max_results,
         )
-        for df in execute_result.to_pandas_batches():
+        for df in execute_result.to_pandas_batches(
+            page_size=page_size, max_results=max_results
+        ):
             self._copy_index_to_pandas(df)
             if squeeze:
                 yield df.squeeze(axis=1)
@@ -821,59 +812,18 @@ class Block:
         if sampling.enable_downsampling:
             raise NotImplementedError("Dry run with sampling is not supported")
 
-        index: List[Any] = []
-        values: List[Any] = []
-
-        index.append("columnCount")
-        values.append(len(self.value_columns))
-        index.append("columnDtypes")
-        values.append(
-            {
-                col: self.expr.get_column_type(self.resolve_label_exact_or_error(col))
-                for col in self.column_labels
-            }
-        )
-
-        index.append("indexLevel")
-        values.append(self.index.nlevels)
-        index.append("indexDtypes")
-        values.append(self.index.dtypes)
-
         expr = self._apply_value_keys_to_expr(value_keys=value_keys)
         query_job = self.session._executor.dry_run(expr, ordered)
-        job_api_repr = copy.deepcopy(query_job._properties)
-
-        job_ref = job_api_repr["jobReference"]
-        for key, val in job_ref.items():
-            index.append(key)
-            values.append(val)
-
-        index.append("jobType")
-        values.append(job_api_repr["configuration"]["jobType"])
-
-        query_config = job_api_repr["configuration"]["query"]
-        for key in ("destinationTable", "useLegacySql"):
-            index.append(key)
-            values.append(query_config.get(key))
-
-        query_stats = job_api_repr["statistics"]["query"]
-        for key in (
-            "referencedTables",
-            "totalBytesProcessed",
-            "cacheHit",
-            "statementType",
-        ):
-            index.append(key)
-            values.append(query_stats.get(key))
 
-        index.append("creationTime")
-        values.append(
-            pd.Timestamp(
-                job_api_repr["statistics"]["creationTime"], unit="ms", tz="UTC"
-            )
-        )
+        column_dtypes = {
+            col: self.expr.get_column_type(self.resolve_label_exact_or_error(col))
+            for col in self.column_labels
+        }
 
-        return pd.Series(values, index=index), query_job
+        dry_run_stats = dry_runs.get_query_stats_with_dtypes(
+            query_job, column_dtypes, self.index.dtypes
+        )
+        return dry_run_stats, query_job
 
     def _apply_value_keys_to_expr(self, value_keys: Optional[Iterable[str]] = None):
         expr = self._expr
@@ -1560,12 +1510,19 @@ class Block:
         """
 
         # head caches full underlying expression, so row_count will be free after
-        head_result = self.session._executor.head(self.expr, max_results)
+        executor = self.session._executor
+        executor.cached(
+            array_value=self.expr,
+            config=executors.CacheConfig(optimize_for="head", if_cached="reuse-strict"),
+        )
+        head_result = self.session._executor.execute(
+            self.expr.slice(start=None, stop=max_results, step=None)
+        )
         row_count = self.session._executor.execute(self.expr.row_count()).to_py_scalar()
 
-        df = head_result.to_pandas()
-        self._copy_index_to_pandas(df)
-        return df, row_count, head_result.query_job
+        head_df = head_result.to_pandas()
+        self._copy_index_to_pandas(head_df)
+        return head_df, row_count, head_result.query_job
 
     def promote_offsets(self, label: Label = None) -> typing.Tuple[Block, str]:
         expr, result_id = self._expr.promote_offsets()
@@ -2535,9 +2492,12 @@ class Block:
         # use a heuristic for whether something needs to be cached
         self.session._executor.cached(
             self.expr,
-            force=force,
-            use_session=session_aware,
-            cluster_cols=self.index_columns,
+            config=executors.CacheConfig(
+                optimize_for="auto"
+                if session_aware
+                else executors.HierarchicalKey(tuple(self.index_columns)),
+                if_cached="replace" if force else "reuse-any",
+            ),
         )
 
     def _is_monotonic(

{bigframes-2.2.0 → bigframes-2.4.0}/bigframes/core/compile/__init__.py

@@ -13,9 +13,13 @@
 # limitations under the License.
 from __future__ import annotations
 
-from bigframes.core.compile.api import SQLCompiler, test_only_ibis_inferred_schema
+from bigframes.core.compile.api import test_only_ibis_inferred_schema
+from bigframes.core.compile.compiler import compile_sql
+from bigframes.core.compile.configs import CompileRequest, CompileResult
 
 __all__ = [
-    "SQLCompiler",
     "test_only_ibis_inferred_schema",
+    "compile_sql",
+    "CompileRequest",
+    "CompileResult",
 ]

{bigframes-2.2.0 → bigframes-2.4.0}/bigframes/core/compile/api.py

@@ -13,43 +13,13 @@
 # limitations under the License.
 from __future__ import annotations
 
-from typing import Optional, Sequence, Tuple, TYPE_CHECKING
-
-import google.cloud.bigquery as bigquery
+from typing import TYPE_CHECKING
 
 from bigframes.core import rewrite
-from bigframes.core.compile import compiler, configs
+from bigframes.core.compile import compiler
 
 if TYPE_CHECKING:
     import bigframes.core.nodes
-    import bigframes.core.ordering
-
-
-class SQLCompiler:
-    def compile(
-        self,
-        node: bigframes.core.nodes.BigFrameNode,
-        *,
-        ordered: bool = True,
-        limit: Optional[int] = None,
-    ) -> str:
-        """Compile node into sql where rows are sorted with ORDER BY."""
-        request = configs.CompileRequest(node, sort_rows=ordered, peek_count=limit)
-        return compiler.compile_sql(request).sql
-
-    def compile_raw(
-        self,
-        node: bigframes.core.nodes.BigFrameNode,
-    ) -> Tuple[
-        str, Sequence[bigquery.SchemaField], bigframes.core.ordering.RowOrdering
-    ]:
-        """Compile node into sql that exposes all columns, including hidden ordering-only columns."""
-        request = configs.CompileRequest(
-            node, sort_rows=False, materialize_all_order_keys=True
-        )
-        result = compiler.compile_sql(request)
-        assert result.row_order is not None
-        return result.sql, result.sql_schema, result.row_order
 
 
 def test_only_ibis_inferred_schema(node: bigframes.core.nodes.BigFrameNode):