snowpark-connect 0.24.0__py3-none-any.whl → 0.25.0__py3-none-any.whl

snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_arrow.py

@@ -1,1346 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import datetime
-import os
-import threading
-import calendar
-import time
-import unittest
-import warnings
-from distutils.version import LooseVersion
-from typing import cast
-from collections import namedtuple
-import sys
-
-from pyspark import SparkContext, SparkConf
-from pyspark.sql import Row, SparkSession
-from pyspark.sql.functions import rand, udf, assert_true, lit
-from pyspark.sql.types import (
-    StructType,
-    StringType,
-    IntegerType,
-    LongType,
-    FloatType,
-    DoubleType,
-    DecimalType,
-    DateType,
-    TimestampType,
-    TimestampNTZType,
-    BinaryType,
-    StructField,
-    ArrayType,
-    MapType,
-    NullType,
-    DayTimeIntervalType,
-)
-from pyspark.testing.objects import ExamplePoint, ExamplePointUDT
-from pyspark.testing.sqlutils import (
-    ReusedSQLTestCase,
-    have_pandas,
-    have_pyarrow,
-    pandas_requirement_message,
-    pyarrow_requirement_message,
-)
-from pyspark.testing.utils import QuietTest
-from pyspark.errors import ArithmeticException, PySparkTypeError, UnsupportedOperationException
-
-if have_pandas:
-    import pandas as pd
-    from pandas.testing import assert_frame_equal
-
-if have_pyarrow:
-    import pyarrow as pa  # noqa: F401
-
-
-class ArrowTestsMixin:
-    @classmethod
-    def setUpClass(cls):
-        from datetime import date, datetime
-        from decimal import Decimal
-
-        super().setUpClass()
-        cls.warnings_lock = threading.Lock()
-
-        # Synchronize default timezone between Python and Java
-        cls.tz_prev = os.environ.get("TZ", None)  # save current tz if set
-        tz = "America/Los_Angeles"
-        os.environ["TZ"] = tz
-        time.tzset()
-
-        cls.spark.conf.set("spark.sql.session.timeZone", tz)
-
-        # Test fallback
-        cls.spark.conf.set("spark.sql.execution.arrow.enabled", "false")
-        assert cls.spark.conf.get("spark.sql.execution.arrow.pyspark.enabled") == "false"
-        cls.spark.conf.set("spark.sql.execution.arrow.enabled", "true")
-        assert cls.spark.conf.get("spark.sql.execution.arrow.pyspark.enabled") == "true"
-
-        cls.spark.conf.set("spark.sql.execution.arrow.fallback.enabled", "true")
-        assert cls.spark.conf.get("spark.sql.execution.arrow.pyspark.fallback.enabled") == "true"
-        cls.spark.conf.set("spark.sql.execution.arrow.fallback.enabled", "false")
-        assert cls.spark.conf.get("spark.sql.execution.arrow.pyspark.fallback.enabled") == "false"
-
-        # Enable Arrow optimization in this tests.
-        cls.spark.conf.set("spark.sql.execution.arrow.pyspark.enabled", "true")
-        # Disable fallback by default to easily detect the failures.
-        cls.spark.conf.set("spark.sql.execution.arrow.pyspark.fallback.enabled", "false")
-
-        cls.schema_wo_null = StructType(
-            [
-                StructField("1_str_t", StringType(), True),
-                StructField("2_int_t", IntegerType(), True),
-                StructField("3_long_t", LongType(), True),
-                StructField("4_float_t", FloatType(), True),
-                StructField("5_double_t", DoubleType(), True),
-                StructField("6_decimal_t", DecimalType(38, 18), True),
-                StructField("7_date_t", DateType(), True),
-                StructField("8_timestamp_t", TimestampType(), True),
-                StructField("9_binary_t", BinaryType(), True),
-            ]
-        )
-        cls.schema = cls.schema_wo_null.add("10_null_t", NullType(), True)
-        cls.data_wo_null = [
-            (
-                "a",
-                1,
-                10,
-                0.2,
-                2.0,
-                Decimal("2.0"),
-                date(1969, 1, 1),
-                datetime(1969, 1, 1, 1, 1, 1),
-                bytearray(b"a"),
-            ),
-            (
-                "b",
-                2,
-                20,
-                0.4,
-                4.0,
-                Decimal("4.0"),
-                date(2012, 2, 2),
-                datetime(2012, 2, 2, 2, 2, 2),
-                bytearray(b"bb"),
-            ),
-            (
-                "c",
-                3,
-                30,
-                0.8,
-                6.0,
-                Decimal("6.0"),
-                date(2100, 3, 3),
-                datetime(2100, 3, 3, 3, 3, 3),
-                bytearray(b"ccc"),
-            ),
-            (
-                "d",
-                4,
-                40,
-                1.0,
-                8.0,
-                Decimal("8.0"),
-                date(2262, 4, 12),
-                datetime(2262, 3, 3, 3, 3, 3),
-                bytearray(b"dddd"),
-            ),
-        ]
-        cls.data = [tuple(list(d) + [None]) for d in cls.data_wo_null]
-
-    @classmethod
-    def tearDownClass(cls):
-        del os.environ["TZ"]
-        if cls.tz_prev is not None:
-            os.environ["TZ"] = cls.tz_prev
-        time.tzset()
-        super().tearDownClass()
-
-    def create_pandas_data_frame(self):
-        import numpy as np
-
-        data_dict = {}
-        for j, name in enumerate(self.schema.names):
-            data_dict[name] = [self.data[i][j] for i in range(len(self.data))]
-        # need to convert these to numpy types first
-        data_dict["2_int_t"] = np.int32(data_dict["2_int_t"])
-        data_dict["4_float_t"] = np.float32(data_dict["4_float_t"])
-        return pd.DataFrame(data=data_dict)
-
-    @property
-    def create_np_arrs(self):
-        import numpy as np
-
-        int_dtypes = ["int8", "int16", "int32", "int64"]
-        float_dtypes = ["float32", "float64"]
-        return (
-            [np.array([1, 2]).astype(t) for t in int_dtypes]
-            + [np.array([0.1, 0.2]).astype(t) for t in float_dtypes]
-            + [np.array([[1], [2]]).astype(t) for t in int_dtypes]
-            + [np.array([[0.1], [0.2]]).astype(t) for t in float_dtypes]
-            + [np.array([[1, 1, 1], [2, 2, 2]]).astype(t) for t in int_dtypes]
-            + [np.array([[0.1, 0.1, 0.1], [0.2, 0.2, 0.2]]).astype(t) for t in float_dtypes]
-        )
-
-    @unittest.skipIf(
-        not have_pyarrow or LooseVersion(pa.__version__) >= "2.0",
-        "will not fallback with pyarrow>=2.0",
-    )
-    def test_toPandas_fallback_enabled(self):
-        with self.sql_conf({"spark.sql.execution.arrow.pyspark.fallback.enabled": True}):
-            schema = StructType([StructField("a", ArrayType(StructType()), True)])
-            df = self.spark.createDataFrame([([Row()],)], schema=schema)
-            with QuietTest(self.sc):
-                with self.warnings_lock:
-                    with warnings.catch_warnings(record=True) as warns:
-                        # we want the warnings to appear even if this test is run from a subclass
-                        warnings.simplefilter("always")
-                        pdf = df.toPandas()
-                        # Catch and check the last UserWarning.
-                        user_warns = [
-                            warn.message for warn in warns if isinstance(warn.message, UserWarning)
-                        ]
-                        self.assertTrue(len(user_warns) > 0)
-                        self.assertTrue("Attempting non-optimization" in str(user_warns[-1]))
-                        assert_frame_equal(pdf, pd.DataFrame({"a": [[Row()]]}))
-
-    @unittest.skipIf(
-        not have_pyarrow or LooseVersion(pa.__version__) >= "2.0",
-        "will not fallback with pyarrow>=2.0",
-    )
-    def test_toPandas_fallback_disabled(self):
-        schema = StructType([StructField("a", ArrayType(StructType()), True)])
-        df = self.spark.createDataFrame([(None,)], schema=schema)
-        with QuietTest(self.sc):
-            with self.warnings_lock:
-                with self.assertRaises(PySparkTypeError) as pe:
-                    df.toPandas()
-
-                self.check_error(
-                    exception=pe.exception,
-                    error_class="UNSUPPORTED_DATA_TYPE_FOR_ARROW_VERSION",
-                    message_parameters={"data_type": "Array of StructType"},
-                )
-
-    def test_toPandas_empty_df_arrow_enabled(self):
-        for arrow_enabled in [True, False]:
-            with self.subTest(arrow_enabled=arrow_enabled):
-                self.check_toPandas_empty_df_arrow_enabled(arrow_enabled)
-
-    def check_toPandas_empty_df_arrow_enabled(self, arrow_enabled):
-        # SPARK-30537 test that toPandas() on an empty dataframe has the correct dtypes
-        # when arrow is enabled
-        from datetime import date
-        from decimal import Decimal
-
-        schema = StructType(
-            [
-                StructField("a", StringType(), True),
-                StructField("a", IntegerType(), True),
-                StructField("c", TimestampType(), True),
-                StructField("d", NullType(), True),
-                StructField("e", LongType(), True),
-                StructField("f", FloatType(), True),
-                StructField("g", DateType(), True),
-                StructField("h", BinaryType(), True),
-                StructField("i", DecimalType(38, 18), True),
-                StructField("k", TimestampNTZType(), True),
-                StructField("L", DayTimeIntervalType(0, 3), True),
-            ]
-        )
-        df = self.spark.createDataFrame([], schema=schema)
-        non_empty_df = self.spark.createDataFrame(
-            [
-                (
-                    "a",
-                    1,
-                    datetime.datetime(1969, 1, 1, 1, 1, 1),
-                    None,
-                    10,
-                    0.2,
-                    date(1969, 1, 1),
-                    bytearray(b"a"),
-                    Decimal("2.0"),
-                    datetime.datetime(1969, 1, 1, 1, 1, 1),
-                    datetime.timedelta(microseconds=123),
-                )
-            ],
-            schema=schema,
-        )
-
-        with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": arrow_enabled}):
-            pdf = df.toPandas()
-            pdf_non_empty = non_empty_df.toPandas()
-        self.assertTrue(pdf.dtypes.equals(pdf_non_empty.dtypes))
-
-    def test_null_conversion(self):
-        df_null = self.spark.createDataFrame(
-            [tuple([None for _ in range(len(self.data_wo_null[0]))])] + self.data_wo_null
-        )
-        pdf = df_null.toPandas()
-        null_counts = pdf.isnull().sum().tolist()
-        self.assertTrue(all([c == 1 for c in null_counts]))
-
-    def _toPandas_arrow_toggle(self, df):
-        with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": False}):
-            pdf = df.toPandas()
-
-        pdf_arrow = df.toPandas()
-
-        return pdf, pdf_arrow
-
-    def test_toPandas_arrow_toggle(self):
-        df = self.spark.createDataFrame(self.data, schema=self.schema)
-        pdf, pdf_arrow = self._toPandas_arrow_toggle(df)
-        expected = self.create_pandas_data_frame()
-        assert_frame_equal(expected, pdf)
-        assert_frame_equal(expected, pdf_arrow)
-
-    def test_create_data_frame_to_pandas_timestamp_ntz(self):
-        for arrow_enabled in [True, False]:
-            with self.subTest(arrow_enabled=arrow_enabled):
-                self.check_create_data_frame_to_pandas_timestamp_ntz(arrow_enabled)
-
-    def check_create_data_frame_to_pandas_timestamp_ntz(self, arrow_enabled):
-        # SPARK-36626: Test TimestampNTZ in createDataFrame and toPandas
-        with self.sql_conf({"spark.sql.session.timeZone": "America/Los_Angeles"}):
-            origin = pd.DataFrame({"a": [datetime.datetime(2012, 2, 2, 2, 2, 2)]})
-            df = self.spark.createDataFrame(
-                origin, schema=StructType([StructField("a", TimestampNTZType(), True)])
-            )
-            df.selectExpr("assert_true('2012-02-02 02:02:02' == CAST(a AS STRING))").collect()
-
-            with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": arrow_enabled}):
-                pdf = df.toPandas()
-            assert_frame_equal(origin, pdf)
-
-    def test_create_data_frame_to_pandas_day_time_internal(self):
-        for arrow_enabled in [True, False]:
-            with self.subTest(arrow_enabled=arrow_enabled):
-                self.check_create_data_frame_to_pandas_day_time_internal(arrow_enabled)
-
-    def check_create_data_frame_to_pandas_day_time_internal(self, arrow_enabled):
-        # SPARK-37279: Test DayTimeInterval in createDataFrame and toPandas
-        origin = pd.DataFrame({"a": [datetime.timedelta(microseconds=123)]})
-        df = self.spark.createDataFrame(origin)
-        df.select(
-            assert_true(lit("INTERVAL '0 00:00:00.000123' DAY TO SECOND") == df.a.cast("string"))
-        ).collect()
-
-        with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": arrow_enabled}):
-            pdf = df.toPandas()
-        assert_frame_equal(origin, pdf)
-
-    def test_toPandas_respect_session_timezone(self):
-        for arrow_enabled in [True, False]:
-            with self.subTest(arrow_enabled=arrow_enabled):
-                self.check_toPandas_respect_session_timezone(arrow_enabled)
-
-    def check_toPandas_respect_session_timezone(self, arrow_enabled):
-        df = self.spark.createDataFrame(self.data, schema=self.schema)
-
-        timezone = "America/Los_Angeles"
-        with self.sql_conf({"spark.sql.session.timeZone": timezone}):
-            with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": arrow_enabled}):
-                pdf_la = df.toPandas()
-
-        timezone = "America/New_York"
-        with self.sql_conf({"spark.sql.session.timeZone": timezone}):
-            with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": arrow_enabled}):
-                pdf_ny = df.toPandas()
-
-            self.assertFalse(pdf_ny.equals(pdf_la))
-
-            from pyspark.sql.pandas.types import _check_series_convert_timestamps_local_tz
-
-            pdf_la_corrected = pdf_la.copy()
-            for field in self.schema:
-                if isinstance(field.dataType, TimestampType):
-                    pdf_la_corrected[field.name] = _check_series_convert_timestamps_local_tz(
-                        pdf_la_corrected[field.name], timezone
-                    )
-            assert_frame_equal(pdf_ny, pdf_la_corrected)
-
-    def test_pandas_round_trip(self):
-        pdf = self.create_pandas_data_frame()
-        df = self.spark.createDataFrame(self.data, schema=self.schema)
-        pdf_arrow = df.toPandas()
-        assert_frame_equal(pdf_arrow, pdf)
-
-    def test_pandas_self_destruct(self):
-        import pyarrow as pa
-
-        rows = 2**10
-        cols = 4
-        expected_bytes = rows * cols * 8
-        df = self.spark.range(0, rows).select(*[rand() for _ in range(cols)])
-        # Test the self_destruct behavior by testing _collect_as_arrow directly
-        allocation_before = pa.total_allocated_bytes()
-        batches = df._collect_as_arrow(split_batches=True)
-        table = pa.Table.from_batches(batches)
-        del batches
-        pdf_split = table.to_pandas(self_destruct=True, split_blocks=True, use_threads=False)
-        allocation_after = pa.total_allocated_bytes()
-        difference = allocation_after - allocation_before
-        # Should be around 1x the data size (table should not hold on to any memory)
-        self.assertGreaterEqual(difference, 0.9 * expected_bytes)
-        self.assertLessEqual(difference, 1.1 * expected_bytes)
-
-        with self.sql_conf({"spark.sql.execution.arrow.pyspark.selfDestruct.enabled": False}):
-            no_self_destruct_pdf = df.toPandas()
-            # Note while memory usage is 2x data size here (both table and pdf hold on to
-            # memory), in this case Arrow still only tracks 1x worth of memory (since the
-            # batches are not allocated by Arrow in this case), so we can't make any
-            # assertions here
-
-        with self.sql_conf({"spark.sql.execution.arrow.pyspark.selfDestruct.enabled": True}):
-            self_destruct_pdf = df.toPandas()
-
-        assert_frame_equal(pdf_split, no_self_destruct_pdf)
-        assert_frame_equal(pdf_split, self_destruct_pdf)
-
-    def test_filtered_frame(self):
-        df = self.spark.range(3).toDF("i")
-        pdf = df.filter("i < 0").toPandas()
-        self.assertEqual(len(pdf.columns), 1)
-        self.assertEqual(pdf.columns[0], "i")
-        self.assertTrue(pdf.empty)
-
-    def test_no_partition_frame(self):
-        schema = StructType([StructField("field1", StringType(), True)])
-        df = self.spark.createDataFrame(self.sc.emptyRDD(), schema)
-        pdf = df.toPandas()
-        self.assertEqual(len(pdf.columns), 1)
-        self.assertEqual(pdf.columns[0], "field1")
-        self.assertTrue(pdf.empty)
-
-    def test_propagates_spark_exception(self):
-        with QuietTest(self.sc):
-            self.check_propagates_spark_exception()
-
-    def check_propagates_spark_exception(self):
-        df = self.spark.range(3).toDF("i")
-
-        def raise_exception():
-            raise RuntimeError("My error")
-
-        exception_udf = udf(raise_exception, IntegerType())
-        df = df.withColumn("error", exception_udf())
-
-        with self.assertRaisesRegex(Exception, "My error"):
-            df.toPandas()
-
-    def _createDataFrame_toggle(self, data, schema=None):
-        with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": False}):
-            df_no_arrow = self.spark.createDataFrame(data, schema=schema)
-
-        df_arrow = self.spark.createDataFrame(data, schema=schema)
-
-        return df_no_arrow, df_arrow
-
-    def test_createDataFrame_toggle(self):
-        pdf = self.create_pandas_data_frame()
-        df_no_arrow, df_arrow = self._createDataFrame_toggle(pdf, schema=self.schema)
-        self.assertEqual(df_no_arrow.collect(), df_arrow.collect())
-
-    def test_createDataFrame_respect_session_timezone(self):
-        for arrow_enabled in [True, False]:
-            with self.subTest(arrow_enabled=arrow_enabled):
-                self.check_createDataFrame_respect_session_timezone(arrow_enabled)
-
-    def check_createDataFrame_respect_session_timezone(self, arrow_enabled):
-        from datetime import timedelta
-
-        pdf = self.create_pandas_data_frame()
-        timezone = "America/Los_Angeles"
-        with self.sql_conf({"spark.sql.session.timeZone": timezone}):
-            with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": arrow_enabled}):
-                df_la = self.spark.createDataFrame(pdf, schema=self.schema)
-            result_la = df_la.collect()
-
-        timezone = "America/New_York"
-        with self.sql_conf({"spark.sql.session.timeZone": timezone}):
-            with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": arrow_enabled}):
-                df_ny = self.spark.createDataFrame(pdf, schema=self.schema)
-            result_ny = df_ny.collect()
-
-            self.assertNotEqual(result_ny, result_la)
-
-            # Correct result_la by adjusting 3 hours difference between Los Angeles and New York
-            result_la_corrected = [
-                Row(
-                    **{
-                        k: v - timedelta(hours=3) if k == "8_timestamp_t" else v
-                        for k, v in row.asDict().items()
-                    }
-                )
-                for row in result_la
-            ]
-            self.assertEqual(result_ny, result_la_corrected)
-
-    def test_createDataFrame_with_schema(self):
-        pdf = self.create_pandas_data_frame()
-        df = self.spark.createDataFrame(pdf, schema=self.schema)
-        self.assertEqual(self.schema, df.schema)
-        pdf_arrow = df.toPandas()
-        assert_frame_equal(pdf_arrow, pdf)
-
-    def test_createDataFrame_with_incorrect_schema(self):
-        with QuietTest(self.sc):
-            self.check_createDataFrame_with_incorrect_schema()
-
-    def check_createDataFrame_with_incorrect_schema(self):
-        pdf = self.create_pandas_data_frame()
-        fields = list(self.schema)
-        fields[5], fields[6] = fields[6], fields[5]  # swap decimal with date
-        wrong_schema = StructType(fields)
-        with self.sql_conf({"spark.sql.execution.pandas.convertToArrowArraySafely": False}):
-            with self.assertRaises(Exception) as context:
-                self.spark.createDataFrame(pdf, schema=wrong_schema)
-
-            # the exception provides us with the column that is incorrect
-            exception = context.exception
-            self.assertTrue(hasattr(exception, "args"))
-            self.assertEqual(len(exception.args), 1)
-            self.assertRegex(
-                exception.args[0],
-                "with name '7_date_t' " "to Arrow Array \\(decimal128\\(38, 18\\)\\)",
-            )
-
-            # the inner exception provides us with the incorrect types
-            exception = exception.__context__
-            self.assertTrue(hasattr(exception, "args"))
-            self.assertEqual(len(exception.args), 1)
-            self.assertRegex(exception.args[0], "[D|d]ecimal.*got.*date")
-
-    def test_createDataFrame_with_names(self):
-        pdf = self.create_pandas_data_frame()
-        new_names = list(map(str, range(len(self.schema.fieldNames()))))
-        # Test that schema as a list of column names gets applied
-        df = self.spark.createDataFrame(pdf, schema=list(new_names))
-        self.assertEqual(df.schema.fieldNames(), new_names)
-        # Test that schema as tuple of column names gets applied
-        df = self.spark.createDataFrame(pdf, schema=tuple(new_names))
-        self.assertEqual(df.schema.fieldNames(), new_names)
-
-    def test_createDataFrame_column_name_encoding(self):
-        pdf = pd.DataFrame({"a": [1]})
-        columns = self.spark.createDataFrame(pdf).columns
-        self.assertTrue(isinstance(columns[0], str))
-        self.assertEqual(columns[0], "a")
-        columns = self.spark.createDataFrame(pdf, ["b"]).columns
-        self.assertTrue(isinstance(columns[0], str))
-        self.assertEqual(columns[0], "b")
-
-    def test_createDataFrame_with_single_data_type(self):
-        with QuietTest(self.sc):
-            self.check_createDataFrame_with_single_data_type()
-
-    def check_createDataFrame_with_single_data_type(self):
-        for schema in ["int", IntegerType()]:
-            with self.subTest(schema=schema):
-                with self.assertRaises(PySparkTypeError) as pe:
-                    self.spark.createDataFrame(pd.DataFrame({"a": [1]}), schema=schema).collect()
-
-                self.check_error(
-                    exception=pe.exception,
-                    error_class="UNSUPPORTED_DATA_TYPE_FOR_ARROW",
-                    message_parameters={"data_type": "IntegerType()"},
-                )
-
-    def test_createDataFrame_does_not_modify_input(self):
-        # Some series get converted for Spark to consume, this makes sure input is unchanged
-        pdf = self.create_pandas_data_frame()
-        # Use a nanosecond value to make sure it is not truncated
-        pdf.iloc[0, 7] = pd.Timestamp(1)
-        # Integers with nulls will get NaNs filled with 0 and will be casted
-        pdf.iloc[1, 1] = None
-        pdf_copy = pdf.copy(deep=True)
-        self.spark.createDataFrame(pdf, schema=self.schema)
-        self.assertTrue(pdf.equals(pdf_copy))
-
-    def test_schema_conversion_roundtrip(self):
-        from pyspark.sql.pandas.types import from_arrow_schema, to_arrow_schema
-
-        arrow_schema = to_arrow_schema(self.schema)
-        schema_rt = from_arrow_schema(arrow_schema, prefer_timestamp_ntz=True)
-        self.assertEqual(self.schema, schema_rt)
-
-    def test_createDataFrame_with_ndarray(self):
-        for arrow_enabled in [True, False]:
-            with self.subTest(arrow_enabled=arrow_enabled):
-                self.check_createDataFrame_with_ndarray(arrow_enabled)
-
-    def check_createDataFrame_with_ndarray(self, arrow_enabled):
-        import numpy as np
-
-        dtypes = ["tinyint", "smallint", "int", "bigint", "float", "double"]
-        expected_dtypes = (
-            [[("value", t)] for t in dtypes]
-            + [[("value", t)] for t in dtypes]
-            + [[("_1", t), ("_2", t), ("_3", t)] for t in dtypes]
-        )
-        arrs = self.create_np_arrs
-
-        with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": arrow_enabled}):
-            for arr, dtypes in zip(arrs, expected_dtypes):
-                df = self.spark.createDataFrame(arr)
-                self.assertEqual(df.dtypes, dtypes)
-                np.array_equal(np.array(df.collect()), arr)
-
-            with self.assertRaisesRegex(
-                ValueError, "NumPy array input should be of 1 or 2 dimensions"
-            ):
-                self.spark.createDataFrame(np.array(0))
-
-    def test_createDataFrame_with_array_type(self):
-        for arrow_enabled in [True, False]:
-            with self.subTest(arrow_enabled=arrow_enabled):
-                self.check_createDataFrame_with_array_type(arrow_enabled)
-
-    def check_createDataFrame_with_array_type(self, arrow_enabled):
-        pdf = pd.DataFrame({"a": [[1, 2], [3, 4]], "b": [["x", "y"], ["y", "z"]]})
-        with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": arrow_enabled}):
-            df = self.spark.createDataFrame(pdf)
-        result = df.collect()
-        expected = [tuple(list(e) for e in rec) for rec in pdf.to_records(index=False)]
-        for r in range(len(expected)):
-            for e in range(len(expected[r])):
-                self.assertTrue(expected[r][e] == result[r][e])
-
-    def test_toPandas_with_array_type(self):
-        for arrow_enabled in [True, False]:
-            with self.subTest(arrow_enabled=arrow_enabled):
-                self.check_toPandas_with_array_type(arrow_enabled)
-
-    def check_toPandas_with_array_type(self, arrow_enabled):
-        expected = [([1, 2], ["x", "y"]), ([3, 4], ["y", "z"])]
-        array_schema = StructType(
-            [StructField("a", ArrayType(IntegerType())), StructField("b", ArrayType(StringType()))]
-        )
-        df = self.spark.createDataFrame(expected, schema=array_schema)
-        pdf = df.toPandas()
-        result = [tuple(list(e) for e in rec) for rec in pdf.to_records(index=False)]
-        for r in range(len(expected)):
-            for e in range(len(expected[r])):
-                self.assertTrue(expected[r][e] == result[r][e])
-
-    def test_createDataFrame_with_map_type(self):
-        with QuietTest(self.sc):
-            for arrow_enabled in [True, False]:
-                with self.subTest(arrow_enabled=arrow_enabled):
-                    self.check_createDataFrame_with_map_type(arrow_enabled)
-
-    def check_createDataFrame_with_map_type(self, arrow_enabled):
-        map_data = [{"a": 1}, {"b": 2, "c": 3}, {}, None, {"d": None}]
-
-        pdf = pd.DataFrame({"id": [0, 1, 2, 3, 4], "m": map_data})
-        for schema in (
-            "id long, m map<string, long>",
-            StructType().add("id", LongType()).add("m", MapType(StringType(), LongType())),
-        ):
-            with self.subTest(schema=schema):
-                with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": arrow_enabled}):
-                    if arrow_enabled and LooseVersion(pa.__version__) < LooseVersion("2.0.0"):
-                        with self.assertRaisesRegex(Exception, "MapType.*only.*pyarrow 2.0.0"):
-                            self.spark.createDataFrame(pdf, schema=schema).collect()
-                    else:
-                        df = self.spark.createDataFrame(pdf, schema=schema)
-
-                        result = df.collect()
-
-                        for row in result:
-                            i, m = row
-                            self.assertEqual(m, map_data[i])
-
-    def test_createDataFrame_with_struct_type(self):
-        for arrow_enabled in [True, False]:
-            with self.subTest(arrow_enabled=arrow_enabled):
-                self.check_createDataFrame_with_struct_type(arrow_enabled)
-
-    def check_createDataFrame_with_struct_type(self, arrow_enabled):
-        pdf = pd.DataFrame(
-            {"a": [Row(1, "a"), Row(2, "b")], "b": [{"s": 3, "t": "x"}, {"s": 4, "t": "y"}]}
-        )
-        for schema in (
-            "a struct<x int, y string>, b struct<s int, t string>",
-            StructType()
-            .add("a", StructType().add("x", LongType()).add("y", StringType()))
-            .add("b", StructType().add("s", LongType()).add("t", StringType())),
-        ):
-            with self.subTest(schema=schema):
-                with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": arrow_enabled}):
-                    df = self.spark.createDataFrame(pdf, schema)
-                result = df.collect()
-                expected = [(rec[0], Row(**rec[1])) for rec in pdf.to_records(index=False)]
-                for r in range(len(expected)):
-                    for e in range(len(expected[r])):
-                        self.assertTrue(
-                            expected[r][e] == result[r][e], f"{expected[r][e]} == {result[r][e]}"
-                        )
-
-    def test_createDataFrame_with_string_dtype(self):
-        # SPARK-34521: spark.createDataFrame does not support Pandas StringDtype extension type
-        with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": True}):
-            data = [["abc"], ["def"], [None], ["ghi"], [None]]
-            pandas_df = pd.DataFrame(data, columns=["col"], dtype="string")
-            schema = StructType([StructField("col", StringType(), True)])
-            df = self.spark.createDataFrame(pandas_df, schema=schema)
-
-            # dtypes won't match. Pandas has two different ways to store string columns:
-            # using ndarray (when dtype isn't specified) or using a StringArray when dtype="string".
-            # When calling dataframe#toPandas() it will use the ndarray version.
-            # Changing that to use a StringArray would be backwards incompatible.
-            assert_frame_equal(pandas_df, df.toPandas(), check_dtype=False)
-
-    def test_createDataFrame_with_int64(self):
-        # SPARK-34521: spark.createDataFrame does not support Pandas StringDtype extension type
-        with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": True}):
-            pandas_df = pd.DataFrame({"col": [1, 2, 3, None]}, dtype="Int64")
-            df = self.spark.createDataFrame(pandas_df)
-            assert_frame_equal(pandas_df, df.toPandas(), check_dtype=False)
-
-    def test_toPandas_with_map_type(self):
-        with QuietTest(self.sc):
-            for arrow_enabled in [True, False]:
-                with self.subTest(arrow_enabled=arrow_enabled):
-                    self.check_toPandas_with_map_type(arrow_enabled)
-
-    def check_toPandas_with_map_type(self, arrow_enabled):
-        origin = pd.DataFrame(
-            {"id": [0, 1, 2, 3], "m": [{}, {"a": 1}, {"a": 1, "b": 2}, {"a": 1, "b": 2, "c": 3}]}
-        )
-
-        for schema in [
-            "id long, m map<string, long>",
-            StructType().add("id", LongType()).add("m", MapType(StringType(), LongType())),
-        ]:
-            with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": False}):
-                df = self.spark.createDataFrame(origin, schema=schema)
-
-            with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": arrow_enabled}):
-                if arrow_enabled and LooseVersion(pa.__version__) < LooseVersion("2.0.0"):
-                    with self.assertRaisesRegex(Exception, "MapType.*only.*pyarrow 2.0.0"):
-                        df.toPandas()
-                else:
-                    pdf = df.toPandas()
-                    assert_frame_equal(origin, pdf)
-
-    def test_toPandas_with_map_type_nulls(self):
-        with QuietTest(self.sc):
-            for arrow_enabled in [True, False]:
-                with self.subTest(arrow_enabled=arrow_enabled):
-                    self.check_toPandas_with_map_type_nulls(arrow_enabled)
-
-    def check_toPandas_with_map_type_nulls(self, arrow_enabled):
-        origin = pd.DataFrame(
-            {"id": [0, 1, 2, 3, 4], "m": [{"a": 1}, {"b": 2, "c": 3}, {}, None, {"d": None}]}
-        )
-
-        for schema in [
-            "id long, m map<string, long>",
-            StructType().add("id", LongType()).add("m", MapType(StringType(), LongType())),
-        ]:
-            with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": False}):
-                df = self.spark.createDataFrame(origin, schema=schema)
-
-            with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": arrow_enabled}):
-                if arrow_enabled and LooseVersion(pa.__version__) < LooseVersion("2.0.0"):
-                    with self.assertRaisesRegex(Exception, "MapType.*only.*pyarrow 2.0.0"):
-                        df.toPandas()
-                else:
-                    pdf = df.toPandas()
-                    assert_frame_equal(origin, pdf)
-
-    def test_createDataFrame_with_int_col_names(self):
-        for arrow_enabled in [True, False]:
-            with self.subTest(arrow_enabled=arrow_enabled):
-                self.check_createDataFrame_with_int_col_names(arrow_enabled)
-
-    def check_createDataFrame_with_int_col_names(self, arrow_enabled):
-        import numpy as np
-
-        pdf = pd.DataFrame(np.random.rand(4, 2))
-        with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": arrow_enabled}):
-            df = self.spark.createDataFrame(pdf)
-        pdf_col_names = [str(c) for c in pdf.columns]
-        self.assertEqual(pdf_col_names, df.columns)
-
-    @unittest.skipIf(
-        not have_pyarrow or LooseVersion(pa.__version__) >= "2.0",
-        "will not fallback with pyarrow>=2.0",
-    )
-    def test_createDataFrame_fallback_enabled(self):
-        with QuietTest(self.sc):
-            with self.sql_conf({"spark.sql.execution.arrow.pyspark.fallback.enabled": True}):
-                with warnings.catch_warnings(record=True) as warns:
-                    # we want the warnings to appear even if this test is run from a subclass
-                    warnings.simplefilter("always")
-                    df = self.spark.createDataFrame(
-                        pd.DataFrame({"a": [[Row()]]}), "a: array<struct<>>"
-                    )
-                    # Catch and check the last UserWarning.
-                    user_warns = [
-                        warn.message for warn in warns if isinstance(warn.message, UserWarning)
-                    ]
-                    self.assertTrue(len(user_warns) > 0)
-                    self.assertTrue("Attempting non-optimization" in str(user_warns[-1]))
-                    self.assertEqual(df.collect(), [Row(a=[Row()])])
-
-    @unittest.skipIf(
-        not have_pyarrow or LooseVersion(pa.__version__) >= "2.0",
-        "will not fallback with pyarrow>=2.0",
-    )
-    def test_createDataFrame_fallback_disabled(self):
-        with QuietTest(self.sc):
-            with self.assertRaises(PySparkTypeError) as pe:
-                self.spark.createDataFrame(pd.DataFrame({"a": [[Row()]]}), "a: array<struct<>>")
-
-            self.check_error(
-                exception=pe.exception,
-                error_class="UNSUPPORTED_DATA_TYPE_FOR_ARROW_VERSION",
-                message_parameters={"data_type": "Array of StructType"},
-            )
-
-    # Regression test for SPARK-23314
-    def test_timestamp_dst(self):
-        # Daylight saving time for Los Angeles for 2015 is Sun, Nov 1 at 2:00 am
-        dt = [
-            datetime.datetime(2015, 11, 1, 0, 30),
-            datetime.datetime(2015, 11, 1, 1, 30),
-            datetime.datetime(2015, 11, 1, 2, 30),
-        ]
-        pdf = pd.DataFrame({"time": dt})
-
-        df_from_python = self.spark.createDataFrame(dt, "timestamp").toDF("time")
-        df_from_pandas = self.spark.createDataFrame(pdf)
-
-        assert_frame_equal(pdf, df_from_python.toPandas())
-        assert_frame_equal(pdf, df_from_pandas.toPandas())
-
-    # Regression test for SPARK-28003
-    def test_timestamp_nat(self):
-        for arrow_enabled in [True, False]:
-            with self.subTest(arrow_enabled=arrow_enabled):
-                self.check_timestamp_nat(arrow_enabled)
-
-    def check_timestamp_nat(self, arrow_enabled):
-        dt = [pd.NaT, pd.Timestamp("2019-06-11"), None] * 100
-        pdf = pd.DataFrame({"time": dt})
-        with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": arrow_enabled}):
-            df = self.spark.createDataFrame(pdf)
-
-        assert_frame_equal(pdf, df.toPandas())
-
-    def test_toPandas_batch_order(self):
-        def delay_first_part(partition_index, iterator):
-            if partition_index == 0:
-                time.sleep(0.1)
-            return iterator
-
-        # Collects Arrow RecordBatches out of order in driver JVM then re-orders in Python
-        def run_test(num_records, num_parts, max_records, use_delay=False):
-            df = self.spark.range(num_records, numPartitions=num_parts).toDF("a")
-            if use_delay:
-                df = df.rdd.mapPartitionsWithIndex(delay_first_part).toDF()
-            with self.sql_conf({"spark.sql.execution.arrow.maxRecordsPerBatch": max_records}):
-                pdf, pdf_arrow = self._toPandas_arrow_toggle(df)
-                assert_frame_equal(pdf, pdf_arrow)
-
-        cases = [
-            (1024, 512, 2),  # Use large num partitions for more likely collecting out of order
-            (64, 8, 2, True),  # Use delay in first partition to force collecting out of order
-            (64, 64, 1),  # Test single batch per partition
-            (64, 1, 64),  # Test single partition, single batch
-            (64, 1, 8),  # Test single partition, multiple batches
-            (30, 7, 2),  # Test different sized partitions
-        ]
-
-        for case in cases:
-            run_test(*case)
-
-    def test_createDataFrame_with_category_type(self):
-        pdf = pd.DataFrame({"A": ["a", "b", "c", "a"]})
-        pdf["B"] = pdf["A"].astype("category")
-        category_first_element = dict(enumerate(pdf["B"].cat.categories))[0]
-
-        with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": True}):
-            arrow_df = self.spark.createDataFrame(pdf)
-            arrow_type = arrow_df.dtypes[1][1]
-            result_arrow = arrow_df.toPandas()
-            arrow_first_category_element = result_arrow["B"][0]
-
-        with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": False}):
-            df = self.spark.createDataFrame(pdf)
-            spark_type = df.dtypes[1][1]
-            result_spark = df.toPandas()
-            spark_first_category_element = result_spark["B"][0]
-
-        assert_frame_equal(result_spark, result_arrow)
-
-        # ensure original category elements are string
-        self.assertIsInstance(category_first_element, str)
-        # spark data frame and arrow execution mode enabled data frame type must match pandas
-        self.assertEqual(spark_type, "string")
-        self.assertEqual(arrow_type, "string")
-        self.assertIsInstance(arrow_first_category_element, str)
-        self.assertIsInstance(spark_first_category_element, str)
-
-    def test_createDataFrame_with_float_index(self):
-        # SPARK-32098: float index should not produce duplicated or truncated Spark DataFrame
-        self.assertEqual(
-            self.spark.createDataFrame(pd.DataFrame({"a": [1, 2, 3]}, index=[2.0, 3.0, 4.0]))
-            .distinct()
-            .count(),
-            3,
-        )
-
-    def test_no_partition_toPandas(self):
-        # SPARK-32301: toPandas should work from a Spark DataFrame with no partitions
-        # Forward-ported from SPARK-32300.
-        pdf = self.spark.sparkContext.emptyRDD().toDF("col1 int").toPandas()
-        self.assertEqual(len(pdf), 0)
-        self.assertEqual(list(pdf.columns), ["col1"])
-
-    def test_createDataFrame_empty_partition(self):
-        pdf = pd.DataFrame({"c1": [1], "c2": ["string"]})
-        df = self.spark.createDataFrame(pdf)
-        self.assertEqual([Row(c1=1, c2="string")], df.collect())
-        self.assertGreater(self.spark.sparkContext.defaultParallelism, len(pdf))
-
-    def test_toPandas_error(self):
-        for arrow_enabled in [True, False]:
-            with self.subTest(arrow_enabled=arrow_enabled):
-                self.check_toPandas_error(arrow_enabled)
-
-    def check_toPandas_error(self, arrow_enabled):
-        with self.sql_conf(
-            {
-                "spark.sql.ansi.enabled": True,
-                "spark.sql.execution.arrow.pyspark.enabled": arrow_enabled,
-            }
-        ):
-            with self.assertRaises(ArithmeticException):
-                self.spark.sql("select 1/0").toPandas()
-
-    def test_toPandas_duplicate_field_names(self):
-        for arrow_enabled in [True, False]:
-            with self.subTest(arrow_enabled=arrow_enabled):
-                self.check_toPandas_duplicate_field_names(arrow_enabled)
-
-    def check_toPandas_duplicate_field_names(self, arrow_enabled):
-        data = [Row(Row("a", 1), Row(2, 3, "b", 4, "c")), Row(Row("x", 6), Row(7, 8, "y", 9, "z"))]
-        schema = (
-            StructType()
-            .add("struct", StructType().add("x", StringType()).add("x", IntegerType()))
-            .add(
-                "struct",
-                StructType()
-                .add("a", IntegerType())
-                .add("x", IntegerType())
-                .add("x", StringType())
-                .add("y", IntegerType())
-                .add("y", StringType()),
-            )
-        )
-        for struct_in_pandas in ["legacy", "row", "dict"]:
-            df = self.spark.createDataFrame(data, schema=schema)
-
-            with self.subTest(struct_in_pandas=struct_in_pandas):
-                with self.sql_conf(
-                    {
-                        "spark.sql.execution.arrow.pyspark.enabled": arrow_enabled,
-                        "spark.sql.execution.pandas.structHandlingMode": struct_in_pandas,
-                    }
-                ):
-                    if arrow_enabled and struct_in_pandas == "legacy":
-                        with self.assertRaisesRegexp(
-                            UnsupportedOperationException, "DUPLICATED_FIELD_NAME_IN_ARROW_STRUCT"
-                        ):
-                            df.toPandas()
-                    else:
-                        if struct_in_pandas == "dict":
-                            expected = pd.DataFrame(
-                                [
-                                    [
-                                        {"x_0": "a", "x_1": 1},
-                                        {"a": 2, "x_0": 3, "x_1": "b", "y_0": 4, "y_1": "c"},
-                                    ],
-                                    [
-                                        {"x_0": "x", "x_1": 6},
-                                        {"a": 7, "x_0": 8, "x_1": "y", "y_0": 9, "y_1": "z"},
-                                    ],
-                                ],
-                                columns=schema.names,
-                            )
-                        else:
-                            expected = pd.DataFrame.from_records(data, columns=schema.names)
-                        assert_frame_equal(df.toPandas(), expected)
-
-    def test_createDataFrame_duplicate_field_names(self):
-        for arrow_enabled in [True, False]:
-            with self.subTest(arrow_enabled=arrow_enabled):
-                self.check_createDataFrame_duplicate_field_names(arrow_enabled)
-
-    def check_createDataFrame_duplicate_field_names(self, arrow_enabled):
-        schema = (
-            StructType()
-            .add("struct", StructType().add("x", StringType()).add("x", IntegerType()))
-            .add(
-                "struct",
-                StructType()
-                .add("a", IntegerType())
-                .add("x", IntegerType())
-                .add("x", StringType())
-                .add("y", IntegerType())
-                .add("y", StringType()),
-            )
-        )
-
-        data = [Row(Row("a", 1), Row(2, 3, "b", 4, "c")), Row(Row("x", 6), Row(7, 8, "y", 9, "z"))]
-        pdf = pd.DataFrame.from_records(data, columns=schema.names)
-
-        with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": arrow_enabled}):
-            df = self.spark.createDataFrame(pdf, schema)
-
-        self.assertEqual(df.collect(), data)
-
-    @unittest.skipIf(
-        LooseVersion(pd.__version__) >= LooseVersion("2.0.0"),
-        "TODO(SPARK-43506): Enable ArrowTests.test_toPandas_empty_columns for pandas 2.0.0.",
-    )
-    def test_toPandas_empty_columns(self):
-        for arrow_enabled in [True, False]:
-            with self.subTest(arrow_enabled=arrow_enabled):
-                self.check_toPandas_empty_columns(arrow_enabled)
-
-    def check_toPandas_empty_columns(self, arrow_enabled):
-        df = self.spark.range(2).select([])
-
-        with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": arrow_enabled}):
-            assert_frame_equal(df.toPandas(), pd.DataFrame(columns=[], index=range(2)))
-
-    def test_createDataFrame_nested_timestamp(self):
-        for arrow_enabled in [True, False]:
-            with self.subTest(arrow_enabled=arrow_enabled):
-                self.check_createDataFrame_nested_timestamp(arrow_enabled)
-
-    def check_createDataFrame_nested_timestamp(self, arrow_enabled):
-        schema = (
-            StructType()
-            .add("ts", TimestampType())
-            .add("ts_ntz", TimestampNTZType())
-            .add(
-                "struct", StructType().add("ts", TimestampType()).add("ts_ntz", TimestampNTZType())
-            )
-            .add("array", ArrayType(TimestampType()))
-            .add("array_ntz", ArrayType(TimestampNTZType()))
-            .add("map", MapType(StringType(), TimestampType()))
-            .add("map_ntz", MapType(StringType(), TimestampNTZType()))
-        )
-        data = [
-            Row(
-                datetime.datetime(2023, 1, 1, 0, 0, 0),
-                datetime.datetime(2023, 1, 1, 0, 0, 0),
-                Row(
-                    datetime.datetime(2023, 1, 1, 0, 0, 0),
-                    datetime.datetime(2023, 1, 1, 0, 0, 0),
-                ),
-                [datetime.datetime(2023, 1, 1, 0, 0, 0)],
-                [datetime.datetime(2023, 1, 1, 0, 0, 0)],
-                dict(ts=datetime.datetime(2023, 1, 1, 0, 0, 0)),
-                dict(ts_ntz=datetime.datetime(2023, 1, 1, 0, 0, 0)),
-            )
-        ]
-        pdf = pd.DataFrame.from_records(data, columns=schema.names)
-
-        with self.sql_conf(
-            {
-                "spark.sql.session.timeZone": "America/New_York",
-                "spark.sql.execution.arrow.pyspark.enabled": arrow_enabled,
-            }
-        ):
-            df = self.spark.createDataFrame(pdf, schema)
-
-        expected = Row(
-            ts=datetime.datetime(2022, 12, 31, 21, 0, 0),
-            ts_ntz=datetime.datetime(2023, 1, 1, 0, 0, 0),
-            struct=Row(
-                ts=datetime.datetime(2022, 12, 31, 21, 0, 0),
-                ts_ntz=datetime.datetime(2023, 1, 1, 0, 0, 0),
-            ),
-            array=[datetime.datetime(2022, 12, 31, 21, 0, 0)],
-            array_ntz=[datetime.datetime(2023, 1, 1, 0, 0, 0)],
-            map=dict(ts=datetime.datetime(2022, 12, 31, 21, 0, 0)),
-            map_ntz=dict(ts_ntz=datetime.datetime(2023, 1, 1, 0, 0, 0)),
-        )
-
-        self.assertEqual(df.first(), expected)
-
-    @unittest.skipIf(sys.version_info < (3, 9), "zoneinfo is available from Python 3.9+")
-    def test_toPandas_timestmap_tzinfo(self):
-        for arrow_enabled in [True, False]:
-            with self.subTest(arrow_enabled=arrow_enabled):
-                self.check_toPandas_timestmap_tzinfo(arrow_enabled)
-
-    def check_toPandas_timestmap_tzinfo(self, arrow_enabled):
-        # SPARK-47202: Test timestamp with tzinfo in toPandas and createDataFrame
-        from zoneinfo import ZoneInfo
-
-        ts_tzinfo = datetime.datetime(2023, 1, 1, 0, 0, 0, tzinfo=ZoneInfo("America/Los_Angeles"))
-        data = pd.DataFrame({"a": [ts_tzinfo]})
-        df = self.spark.createDataFrame(data)
-
-        with self.sql_conf(
-            {
-                "spark.sql.execution.arrow.pyspark.enabled": arrow_enabled,
-            }
-        ):
-            pdf = df.toPandas()
-
-        expected = pd.DataFrame(
-            # Spark unsets tzinfo and converts them to localtimes.
-            {"a": [datetime.datetime.fromtimestamp(calendar.timegm(ts_tzinfo.utctimetuple()))]}
-        )
-
-        assert_frame_equal(pdf, expected)
-
-    def test_toPandas_nested_timestamp(self):
-        for arrow_enabled in [True, False]:
-            with self.subTest(arrow_enabled=arrow_enabled):
-                self.check_toPandas_nested_timestamp(arrow_enabled)
-
-    def check_toPandas_nested_timestamp(self, arrow_enabled):
-        schema = (
-            StructType()
-            .add("ts", TimestampType())
-            .add("ts_ntz", TimestampNTZType())
-            .add(
-                "struct", StructType().add("ts", TimestampType()).add("ts_ntz", TimestampNTZType())
-            )
-            .add("array", ArrayType(TimestampType()))
-            .add("array_ntz", ArrayType(TimestampNTZType()))
-            .add("map", MapType(StringType(), TimestampType()))
-            .add("map_ntz", MapType(StringType(), TimestampNTZType()))
-        )
-        data = [
-            Row(
-                datetime.datetime(2023, 1, 1, 0, 0, 0),
-                datetime.datetime(2023, 1, 1, 0, 0, 0),
-                Row(
-                    datetime.datetime(2023, 1, 1, 0, 0, 0),
-                    datetime.datetime(2023, 1, 1, 0, 0, 0),
-                ),
-                [datetime.datetime(2023, 1, 1, 0, 0, 0)],
-                [datetime.datetime(2023, 1, 1, 0, 0, 0)],
-                dict(ts=datetime.datetime(2023, 1, 1, 0, 0, 0)),
-                dict(ts_ntz=datetime.datetime(2023, 1, 1, 0, 0, 0)),
-            )
-        ]
-        df = self.spark.createDataFrame(data, schema)
-
-        with self.sql_conf(
-            {
-                "spark.sql.session.timeZone": "America/New_York",
-                "spark.sql.execution.arrow.pyspark.enabled": arrow_enabled,
-                "spark.sql.execution.pandas.structHandlingMode": "row",
-            }
-        ):
-            pdf = df.toPandas()
-
-        expected = pd.DataFrame(
-            {
-                "ts": [datetime.datetime(2023, 1, 1, 3, 0, 0)],
-                "ts_ntz": [datetime.datetime(2023, 1, 1, 0, 0, 0)],
-                "struct": [
-                    Row(
-                        datetime.datetime(2023, 1, 1, 3, 0, 0),
-                        datetime.datetime(2023, 1, 1, 0, 0, 0),
-                    )
-                ],
-                "array": [[datetime.datetime(2023, 1, 1, 3, 0, 0)]],
-                "array_ntz": [[datetime.datetime(2023, 1, 1, 0, 0, 0)]],
-                "map": [dict(ts=datetime.datetime(2023, 1, 1, 3, 0, 0))],
-                "map_ntz": [dict(ts_ntz=datetime.datetime(2023, 1, 1, 0, 0, 0))],
-            }
-        )
-
-        assert_frame_equal(pdf, expected)
-
-    def test_createDataFrame_udt(self):
-        for arrow_enabled in [True, False]:
-            with self.subTest(arrow_enabled=arrow_enabled):
-                self.check_createDataFrame_udt(arrow_enabled)
-
-    def check_createDataFrame_udt(self, arrow_enabled):
-        schema = (
-            StructType()
-            .add("point", ExamplePointUDT())
-            .add("struct", StructType().add("point", ExamplePointUDT()))
-            .add("array", ArrayType(ExamplePointUDT()))
-            .add("map", MapType(StringType(), ExamplePointUDT()))
-        )
-        data = [
-            Row(
-                ExamplePoint(1.0, 2.0),
-                Row(ExamplePoint(3.0, 4.0)),
-                [ExamplePoint(5.0, 6.0)],
-                dict(point=ExamplePoint(7.0, 8.0)),
-            )
-        ]
-        pdf = pd.DataFrame.from_records(data, columns=schema.names)
-
-        with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": arrow_enabled}):
-            df = self.spark.createDataFrame(pdf, schema)
-
-        self.assertEqual(df.collect(), data)
-
-    def test_toPandas_udt(self):
-        for arrow_enabled in [True, False]:
-            with self.subTest(arrow_enabled=arrow_enabled):
-                self.check_toPandas_udt(arrow_enabled)
-
-    def check_toPandas_udt(self, arrow_enabled):
-        schema = (
-            StructType()
-            .add("point", ExamplePointUDT())
-            .add("struct", StructType().add("point", ExamplePointUDT()))
-            .add("array", ArrayType(ExamplePointUDT()))
-            .add("map", MapType(StringType(), ExamplePointUDT()))
-        )
-        data = [
-            Row(
-                ExamplePoint(1.0, 2.0),
-                Row(ExamplePoint(3.0, 4.0)),
-                [ExamplePoint(5.0, 6.0)],
-                dict(point=ExamplePoint(7.0, 8.0)),
-            )
-        ]
-        df = self.spark.createDataFrame(data, schema)
-
-        with self.sql_conf(
-            {
-                "spark.sql.execution.arrow.pyspark.enabled": arrow_enabled,
-                "spark.sql.execution.pandas.structHandlingMode": "row",
-            }
-        ):
-            pdf = df.toPandas()
-
-        expected = pd.DataFrame.from_records(data, columns=schema.names)
-
-        assert_frame_equal(pdf, expected)
-
-    def test_create_dataframe_namedtuples(self):
-        # SPARK-44980: Inherited namedtuples in createDataFrame
-        for arrow_enabled in [True, False]:
-            with self.subTest(arrow_enabled=arrow_enabled):
-                self.check_create_dataframe_namedtuples(arrow_enabled)
-
-    def check_create_dataframe_namedtuples(self, arrow_enabled):
-        MyTuple = namedtuple("MyTuple", ["a", "b", "c"])
-
-        class MyInheritedTuple(MyTuple):
-            pass
-
-        with self.sql_conf(
-            {
-                "spark.sql.execution.arrow.pyspark.enabled": arrow_enabled,
-            }
-        ):
-            df = self.spark.createDataFrame([MyInheritedTuple(1, 2, 3)])
-            self.assertEqual(df.first(), Row(a=1, b=2, c=3))
-
-            df = self.spark.createDataFrame([MyInheritedTuple(1, 2, MyInheritedTuple(1, 2, 3))])
-            self.assertEqual(df.first(), Row(a=1, b=2, c=Row(a=1, b=2, c=3)))
-
-    def test_negative_and_zero_batch_size(self):
-        # SPARK-47068: Negative and zero value should work as unlimited batch size.
-        with self.sql_conf({"spark.sql.execution.arrow.maxRecordsPerBatch": 0}):
-            pdf = pd.DataFrame({"a": [123]})
-            assert_frame_equal(pdf, self.spark.createDataFrame(pdf).toPandas())
-
-        with self.sql_conf({"spark.sql.execution.arrow.maxRecordsPerBatch": -1}):
-            pdf = pd.DataFrame({"a": [123]})
-            assert_frame_equal(pdf, self.spark.createDataFrame(pdf).toPandas())
-
-
-@unittest.skipIf(
-    not have_pandas or not have_pyarrow,
-    cast(str, pandas_requirement_message or pyarrow_requirement_message),
-)
-class ArrowTests(ArrowTestsMixin, ReusedSQLTestCase):
-    pass
-
-
-@unittest.skipIf(
-    not have_pandas or not have_pyarrow,
-    cast(str, pandas_requirement_message or pyarrow_requirement_message),
-)
-class MaxResultArrowTests(unittest.TestCase):
-    # These tests are separate as 'spark.driver.maxResultSize' configuration
-    # is a static configuration to Spark context.
-
-    @classmethod
-    def setUpClass(cls):
-        cls.spark = SparkSession(
-            SparkContext(
-                "local[4]", cls.__name__, conf=SparkConf().set("spark.driver.maxResultSize", "10k")
-            )
-        )
-
-        # Explicitly enable Arrow and disable fallback.
-        cls.spark.conf.set("spark.sql.execution.arrow.pyspark.enabled", "true")
-        cls.spark.conf.set("spark.sql.execution.arrow.pyspark.fallback.enabled", "false")
-
-    @classmethod
-    def tearDownClass(cls):
-        if hasattr(cls, "spark"):
-            cls.spark.stop()
-
-    def test_exception_by_max_results(self):
-        with self.assertRaisesRegex(Exception, "is bigger than"):
-            self.spark.range(0, 10000, 1, 100).toPandas()
-
-
-class EncryptionArrowTests(ArrowTests):
-    @classmethod
-    def conf(cls):
-        return super(EncryptionArrowTests, cls).conf().set("spark.io.encryption.enabled", "true")
-
-
-class RDDBasedArrowTests(ArrowTests):
-    @classmethod
-    def conf(cls):
-        return (
-            super(RDDBasedArrowTests, cls)
-            .conf()
-            .set("spark.sql.execution.arrow.localRelationThreshold", "0")
-            # to test multiple partitions
-            .set("spark.sql.execution.arrow.maxRecordsPerBatch", "2")
-        )
-
-
-if __name__ == "__main__":
-    from pyspark.sql.tests.test_arrow import *  # noqa: F401
-
-    try:
-        import xmlrunner  # type: ignore
-
-        testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
-    except ImportError:
-        testRunner = None
-    unittest.main(testRunner=testRunner, verbosity=2)