google-cloud-bigquery 3.22.0__tar.gz → 3.23.1__tar.gz

{google-cloud-bigquery-3.22.0/google_cloud_bigquery.egg-info → google-cloud-bigquery-3.23.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: google-cloud-bigquery
-Version: 3.22.0
+Version: 3.23.1
 Summary: Google BigQuery API client library
 Home-page: https://github.com/googleapis/python-bigquery
 Author: Google LLC

{google-cloud-bigquery-3.22.0 → google-cloud-bigquery-3.23.1}/google/cloud/bigquery/_helpers.py

@@ -50,6 +50,7 @@ _INTERVAL_PATTERN = re.compile(
     r"(?P<days>-?\d+) "
     r"(?P<time_sign>-?)(?P<hours>\d+):(?P<minutes>\d+):(?P<seconds>\d+)\.?(?P<fraction>\d*)?$"
 )
+_RANGE_PATTERN = re.compile(r"\[.*, .*\)")
 
 BIGQUERY_EMULATOR_HOST = "BIGQUERY_EMULATOR_HOST"
 """Environment variable defining host for emulator."""
@@ -334,9 +335,8 @@ def _range_from_json(value, field):
             The parsed range object from ``value`` if the ``field`` is not
             null (otherwise it is :data:`None`).
     """
-    range_literal = re.compile(r"\[.*, .*\)")
     if _not_null(value, field):
-        if range_literal.match(value):
+        if _RANGE_PATTERN.match(value):
             start, end = value[1:-1].split(", ")
             start = _range_element_from_json(start, field.range_element_type)
             end = _range_element_from_json(end, field.range_element_type)
@@ -531,6 +531,52 @@ def _time_to_json(value):
     return value
 
 
+def _range_element_to_json(value, element_type=None):
+    """Coerce 'value' to an JSON-compatible representation."""
+    if value is None:
+        return None
+    elif isinstance(value, str):
+        if value.upper() in ("UNBOUNDED", "NULL"):
+            return None
+        else:
+            # We do not enforce range element value to be valid to reduce
+            # redundancy with backend.
+            return value
+    elif (
+        element_type and element_type.element_type.upper() in _SUPPORTED_RANGE_ELEMENTS
+    ):
+        converter = _SCALAR_VALUE_TO_JSON_ROW.get(element_type.element_type.upper())
+        return converter(value)
+    else:
+        raise ValueError(
+            f"Unsupported RANGE element type {element_type}, or "
+            "element type is empty. Must be DATE, DATETIME, or "
+            "TIMESTAMP"
+        )
+
+
+def _range_field_to_json(range_element_type, value):
+    """Coerce 'value' to an JSON-compatible representation."""
+    if isinstance(value, str):
+        # string literal
+        if _RANGE_PATTERN.match(value):
+            start, end = value[1:-1].split(", ")
+        else:
+            raise ValueError(f"RANGE literal {value} has incorrect format")
+    elif isinstance(value, dict):
+        # dictionary
+        start = value.get("start")
+        end = value.get("end")
+    else:
+        raise ValueError(
+            f"Unsupported type of RANGE value {value}, must be " "string or dict"
+        )
+
+    start = _range_element_to_json(start, range_element_type)
+    end = _range_element_to_json(end, range_element_type)
+    return {"start": start, "end": end}
+
+
 # Converters used for scalar values marshalled to the BigQuery API, such as in
 # query parameters or the tabledata.insert API.
 _SCALAR_VALUE_TO_JSON_ROW = {
@@ -676,6 +722,8 @@ def _single_field_to_json(field, row_value):
 
     if field.field_type == "RECORD":
         return _record_field_to_json(field.fields, row_value)
+    if field.field_type == "RANGE":
+        return _range_field_to_json(field.range_element_type, row_value)
 
     return _scalar_field_to_json(field, row_value)
 

{google-cloud-bigquery-3.22.0 → google-cloud-bigquery-3.23.1}/google/cloud/bigquery/_versions_helpers.py

@@ -26,6 +26,9 @@ _MIN_BQ_STORAGE_VERSION = packaging.version.Version("2.0.0")
 _BQ_STORAGE_OPTIONAL_READ_SESSION_VERSION = packaging.version.Version("2.6.0")
 _MIN_PANDAS_VERSION = packaging.version.Version("1.1.0")
 
+_MIN_PANDAS_VERSION_RANGE = packaging.version.Version("1.5.0")
+_MIN_PYARROW_VERSION_RANGE = packaging.version.Version("10.0.1")
+
 
 class PyarrowVersions:
     """Version comparisons for pyarrow package."""
@@ -234,3 +237,14 @@ class PandasVersions:
 
 
 PANDAS_VERSIONS = PandasVersions()
+
+# Since RANGE support in pandas requires specific versions
+# of both pyarrow and pandas, we make this a separate
+# constant instead of as a property of PANDAS_VERSIONS
+# or PYARROW_VERSIONS.
+SUPPORTS_RANGE_PYARROW = (
+    PANDAS_VERSIONS.try_import() is not None
+    and PANDAS_VERSIONS.installed_version >= _MIN_PANDAS_VERSION_RANGE
+    and PYARROW_VERSIONS.try_import() is not None
+    and PYARROW_VERSIONS.installed_version >= _MIN_PYARROW_VERSION_RANGE
+)

{google-cloud-bigquery-3.22.0 → google-cloud-bigquery-3.23.1}/google/cloud/bigquery/table.py

@@ -100,9 +100,25 @@ _NO_SUPPORTED_DTYPE = (
     "because the necessary `__from_arrow__` attribute is missing."
 )
 
+_RANGE_PYARROW_WARNING = (
+    "Unable to represent RANGE schema as struct using pandas ArrowDtype. Using "
+    "`object` instead. To use ArrowDtype, use pandas >= 1.5 and "
+    "pyarrow >= 10.0.1."
+)
+
 # How many of the total rows need to be downloaded already for us to skip
 # calling the BQ Storage API?
-ALMOST_COMPLETELY_CACHED_RATIO = 0.333
+#
+# In microbenchmarks on 2024-05-21, I (tswast@) measure that at about 2 MB of
+# remaining results, it's faster to use the BQ Storage Read API to download
+# the results than use jobs.getQueryResults. Since we don't have a good way to
+# know the remaining bytes, we estimate by remaining number of rows.
+#
+# Except when rows themselves are larger, I observe that the a single page of
+# results will be around 10 MB. Therefore, the proportion of rows already
+# downloaded should be 10 (first page) / 12 (all results) or less for it to be
+# worth it to make a call to jobs.getQueryResults.
+ALMOST_COMPLETELY_CACHED_RATIO = 0.833333
 
 
 def _reference_getter(table):
@@ -2279,26 +2295,18 @@ class RowIterator(HTTPIterator):
             time_dtype = db_dtypes.TimeDtype()
 
         if range_date_dtype is DefaultPandasDTypes.RANGE_DATE_DTYPE:
-            try:
+            if _versions_helpers.SUPPORTS_RANGE_PYARROW:
                 range_date_dtype = pandas.ArrowDtype(
                     pyarrow.struct(
                         [("start", pyarrow.date32()), ("end", pyarrow.date32())]
                     )
                 )
-            except AttributeError:
-                # pandas.ArrowDtype was introduced in pandas 1.5, but python 3.7
-                # only supports upto pandas 1.3. If pandas.ArrowDtype is not
-                # present, we raise a warning and set range_date_dtype to None.
-                msg = (
-                    "Unable to find class ArrowDtype in pandas, setting "
-                    "range_date_dtype to be None. To use ArrowDtype, please "
-                    "use pandas >= 1.5 and python >= 3.8."
-                )
-                warnings.warn(msg)
+            else:
+                warnings.warn(_RANGE_PYARROW_WARNING)
                 range_date_dtype = None
 
         if range_datetime_dtype is DefaultPandasDTypes.RANGE_DATETIME_DTYPE:
-            try:
+            if _versions_helpers.SUPPORTS_RANGE_PYARROW:
                 range_datetime_dtype = pandas.ArrowDtype(
                     pyarrow.struct(
                         [
@@ -2307,20 +2315,12 @@ class RowIterator(HTTPIterator):
                         ]
                     )
                 )
-            except AttributeError:
-                # pandas.ArrowDtype was introduced in pandas 1.5, but python 3.7
-                # only supports upto pandas 1.3. If pandas.ArrowDtype is not
-                # present, we raise a warning and set range_datetime_dtype to None.
-                msg = (
-                    "Unable to find class ArrowDtype in pandas, setting "
-                    "range_datetime_dtype to be None. To use ArrowDtype, "
-                    "please use pandas >= 1.5 and python >= 3.8."
-                )
-                warnings.warn(msg)
+            else:
+                warnings.warn(_RANGE_PYARROW_WARNING)
                 range_datetime_dtype = None
 
         if range_timestamp_dtype is DefaultPandasDTypes.RANGE_TIMESTAMP_DTYPE:
-            try:
+            if _versions_helpers.SUPPORTS_RANGE_PYARROW:
                 range_timestamp_dtype = pandas.ArrowDtype(
                     pyarrow.struct(
                         [
@@ -2329,16 +2329,8 @@ class RowIterator(HTTPIterator):
                         ]
                     )
                 )
-            except AttributeError:
-                # pandas.ArrowDtype was introduced in pandas 1.5, but python 3.7
-                # only supports upto pandas 1.3. If pandas.ArrowDtype is not
-                # present, we raise a warning and set range_timestamp_dtype to None.
-                msg = (
-                    "Unable to find class ArrowDtype in pandas, setting "
-                    "range_timestamp_dtype to be None. To use ArrowDtype, "
-                    "please use pandas >= 1.5 and python >= 3.8."
-                )
-                warnings.warn(msg)
+            else:
+                warnings.warn(_RANGE_PYARROW_WARNING)
                 range_timestamp_dtype = None
 
         if bool_dtype is not None and not hasattr(bool_dtype, "__from_arrow__"):

{google-cloud-bigquery-3.22.0 → google-cloud-bigquery-3.23.1}/google/cloud/bigquery/version.py

@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "3.22.0"
+__version__ = "3.23.1"

{google-cloud-bigquery-3.22.0 → google-cloud-bigquery-3.23.1/google_cloud_bigquery.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: google-cloud-bigquery
-Version: 3.22.0
+Version: 3.23.1
 Summary: Google BigQuery API client library
 Home-page: https://github.com/googleapis/python-bigquery
 Author: Google LLC

{google-cloud-bigquery-3.22.0 → google-cloud-bigquery-3.23.1}/tests/unit/test__helpers.py

@@ -1049,10 +1049,22 @@ class Test_time_to_json(unittest.TestCase):
         self.assertEqual(self._call_fut(when), "12:13:41")
 
 
-def _make_field(field_type, mode="NULLABLE", name="testing", fields=()):
+def _make_field(
+    field_type,
+    mode="NULLABLE",
+    name="testing",
+    fields=(),
+    range_element_type=None,
+):
     from google.cloud.bigquery.schema import SchemaField
 
-    return SchemaField(name=name, field_type=field_type, mode=mode, fields=fields)
+    return SchemaField(
+        name=name,
+        field_type=field_type,
+        mode=mode,
+        fields=fields,
+        range_element_type=range_element_type,
+    )
 
 
 class Test_scalar_field_to_json(unittest.TestCase):
@@ -1251,6 +1263,98 @@ class Test_record_field_to_json(unittest.TestCase):
         )
 
 
+class Test_range_field_to_json(unittest.TestCase):
+    def _call_fut(self, field, value):
+        from google.cloud.bigquery._helpers import _range_field_to_json
+
+        return _range_field_to_json(field, value)
+
+    def test_w_date(self):
+        field = _make_field("RANGE", range_element_type="DATE")
+        start = datetime.date(2016, 12, 3)
+        original = {"start": start}
+        converted = self._call_fut(field.range_element_type, original)
+        expected = {"start": "2016-12-03", "end": None}
+        self.assertEqual(converted, expected)
+
+    def test_w_date_string(self):
+        field = _make_field("RANGE", range_element_type="DATE")
+        original = {"start": "2016-12-03"}
+        converted = self._call_fut(field.range_element_type, original)
+        expected = {"start": "2016-12-03", "end": None}
+        self.assertEqual(converted, expected)
+
+    def test_w_datetime(self):
+        field = _make_field("RANGE", range_element_type="DATETIME")
+        start = datetime.datetime(2016, 12, 3, 14, 11, 27, 123456)
+        original = {"start": start}
+        converted = self._call_fut(field.range_element_type, original)
+        expected = {"start": "2016-12-03T14:11:27.123456", "end": None}
+        self.assertEqual(converted, expected)
+
+    def test_w_datetime_string(self):
+        field = _make_field("RANGE", range_element_type="DATETIME")
+        original = {"start": "2016-12-03T14:11:27.123456"}
+        converted = self._call_fut(field.range_element_type, original)
+        expected = {"start": "2016-12-03T14:11:27.123456", "end": None}
+        self.assertEqual(converted, expected)
+
+    def test_w_timestamp(self):
+        from google.cloud._helpers import UTC
+
+        field = _make_field("RANGE", range_element_type="TIMESTAMP")
+        start = datetime.datetime(2016, 12, 3, 14, 11, 27, 123456, tzinfo=UTC)
+        original = {"start": start}
+        converted = self._call_fut(field.range_element_type, original)
+        expected = {"start": "2016-12-03T14:11:27.123456Z", "end": None}
+        self.assertEqual(converted, expected)
+
+    def test_w_timestamp_string(self):
+        field = _make_field("RANGE", range_element_type="TIMESTAMP")
+        original = {"start": "2016-12-03T14:11:27.123456Z"}
+        converted = self._call_fut(field.range_element_type, original)
+        expected = {"start": "2016-12-03T14:11:27.123456Z", "end": None}
+        self.assertEqual(converted, expected)
+
+    def test_w_timestamp_float(self):
+        field = _make_field("RANGE", range_element_type="TIMESTAMP")
+        original = {"start": 12.34567}
+        converted = self._call_fut(field.range_element_type, original)
+        expected = {"start": 12.34567, "end": None}
+        self.assertEqual(converted, expected)
+
+    def test_w_string_literal(self):
+        field = _make_field("RANGE", range_element_type="DATE")
+        original = "[2016-12-03, UNBOUNDED)"
+        converted = self._call_fut(field.range_element_type, original)
+        expected = {"start": "2016-12-03", "end": None}
+        self.assertEqual(converted, expected)
+
+    def test_w_unsupported_range_element_type(self):
+        field = _make_field("RANGE", range_element_type="TIME")
+        with self.assertRaises(ValueError):
+            self._call_fut(
+                field.range_element_type,
+                {"start": datetime.time(12, 13, 41)},
+            )
+
+    def test_w_no_range_element_type(self):
+        field = _make_field("RANGE")
+        with self.assertRaises(ValueError):
+            self._call_fut(field.range_element_type, "2016-12-03")
+
+    def test_w_incorrect_literal_format(self):
+        field = _make_field("RANGE", range_element_type="DATE")
+        original = "[2016-12-03, UNBOUNDED]"
+        with self.assertRaises(ValueError):
+            self._call_fut(field.range_element_type, original)
+
+    def test_w_unsupported_representation(self):
+        field = _make_field("RANGE", range_element_type="DATE")
+        with self.assertRaises(ValueError):
+            self._call_fut(field.range_element_type, object())
+
+
 class Test_field_to_json(unittest.TestCase):
     def _call_fut(self, field, value):
         from google.cloud.bigquery._helpers import _field_to_json
@@ -1285,6 +1389,12 @@ class Test_field_to_json(unittest.TestCase):
         converted = self._call_fut(field, original)
         self.assertEqual(converted, str(original))
 
+    def test_w_range(self):
+        field = _make_field("RANGE", range_element_type="DATE")
+        original = {"start": "2016-12-03", "end": "2024-12-03"}
+        converted = self._call_fut(field, original)
+        self.assertEqual(converted, original)
+
 
 class Test_snake_to_camel_case(unittest.TestCase):
     def _call_fut(self, value):

{google-cloud-bigquery-3.22.0 → google-cloud-bigquery-3.23.1}/tests/unit/test_table.py

@@ -2307,9 +2307,17 @@ class TestRowIterator(unittest.TestCase):
         rows = [
             {"f": [{"v": "Phred Phlyntstone"}, {"v": "32"}]},
             {"f": [{"v": "Bharney Rhubble"}, {"v": "33"}]},
+            {"f": [{"v": "Whillma Phlyntstone"}, {"v": "27"}]},
+            {"f": [{"v": "Bhetty Rhubble"}, {"v": "28"}]},
+            {"f": [{"v": "Pebbles Phlyntstone"}, {"v": "4"}]},
+            {"f": [{"v": "Bamm-Bamm Rhubble"}, {"v": "5"}]},
+            {"f": [{"v": "Joseph Rockhead"}, {"v": "32"}]},
+            {"f": [{"v": "Perry Masonry"}, {"v": "33"}]},
         ]
         first_page = {"pageToken": "next-page", "rows": rows}
-        iterator = self._make_one(first_page_response=first_page, total_rows=6)
+        iterator = self._make_one(
+            first_page_response=first_page, total_rows=len(rows) + 1
+        )
         self.assertTrue(iterator._is_almost_completely_cached())
 
     def test__is_almost_completely_cached_returns_true_with_no_rows_remaining(self):