snowpark-connect 0.24.0__py3-none-any.whl → 0.26.0__py3-none-any.whl

snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_udf.py

@@ -1,100 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import unittest
-
-from pyspark.testing.connectutils import should_test_connect
-
-if should_test_connect:
-    from pyspark import sql
-    from pyspark.sql.connect.udf import UserDefinedFunction
-
-    sql.udf.UserDefinedFunction = UserDefinedFunction
-
-from pyspark.sql.tests.test_udf import BaseUDFTestsMixin
-from pyspark.testing.connectutils import ReusedConnectTestCase
-
-
-class UDFParityTests(BaseUDFTestsMixin, ReusedConnectTestCase):
-    @unittest.skip("Spark Connect does not support mapPartitions() but the test depends on it.")
-    def test_worker_original_stdin_closed(self):
-        super().test_worker_original_stdin_closed()
-
-    @unittest.skip(
-        "Spark Connect does not support reading from Hadoop RDD but the test depends on it."
-    )
-    def test_udf_with_input_file_name_for_hadooprdd(self):
-        super().test_udf_with_input_file_name_for_hadooprdd()
-
-    @unittest.skip("Spark Connect does not support accumulator but the test depends on it.")
-    def test_same_accumulator_in_udfs(self):
-        super().test_same_accumulator_in_udfs()
-
-    @unittest.skip("Spark Connect does not support spark.conf but the test depends on it.")
-    def test_udf_timestamp_ntz(self):
-        super().test_udf_timestamp_ntz()
-
-    @unittest.skip("Spark Connect does not support broadcast but the test depends on it.")
-    def test_broadcast_in_udf(self):
-        super().test_broadcast_in_udf()
-
-    @unittest.skip("Spark Connect does not support cache() but the test depends on it.")
-    def test_udf_cache(self):
-        super().test_udf_cache()
-
-    @unittest.skip("Requires JVM access.")
-    def test_udf_defers_judf_initialization(self):
-        super().test_udf_defers_judf_initialization()
-
-    @unittest.skip("Requires JVM access.")
-    def test_nondeterministic_udf3(self):
-        super().test_nondeterministic_udf3()
-
-    def test_nondeterministic_udf_in_aggregate(self):
-        self.check_nondeterministic_udf_in_aggregate()
-
-    def test_udf_registration_return_type_not_none(self):
-        self.check_udf_registration_return_type_not_none()
-
-    @unittest.skip("Spark Connect doesn't support RDD but the test depends on it.")
-    def test_worker_original_stdin_closed(self):
-        super().test_worker_original_stdin_closed()
-
-    @unittest.skip("Spark Connect does not support SQLContext but the test depends on it.")
-    def test_udf_on_sql_context(self):
-        super().test_udf_on_sql_context()
-
-    @unittest.skip("Spark Connect does not support SQLContext but the test depends on it.")
-    def test_non_existed_udf_with_sql_context(self):
-        super().test_non_existed_udf_with_sql_context()
-
-    @unittest.skip("Spark Connect does not support SQLContext but the test depends on it.")
-    def test_udf_registration_returns_udf_on_sql_context(self):
-        super().test_udf_registration_returns_udf_on_sql_context()
-
-
-if __name__ == "__main__":
-    import unittest
-    from pyspark.sql.tests.connect.test_parity_udf import *  # noqa: F401
-
-    try:
-        import xmlrunner  # type: ignore[import]
-
-        testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
-    except ImportError:
-        testRunner = None
-    unittest.main(testRunner=testRunner, verbosity=2)

snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_parity_udtf.py

@@ -1,163 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-import os
-import unittest
-from pyspark.testing.connectutils import should_test_connect
-
-if should_test_connect:
-    from pyspark import sql
-    from pyspark.sql.connect.udtf import UserDefinedTableFunction
-
-    sql.udtf.UserDefinedTableFunction = UserDefinedTableFunction
-
-from pyspark.sql.connect.functions import lit, udtf
-from pyspark.sql.tests.test_udtf import BaseUDTFTestsMixin, UDTFArrowTestsMixin
-from pyspark.testing.connectutils import ReusedConnectTestCase
-from pyspark.errors.exceptions.connect import SparkConnectGrpcException
-
-
-class UDTFParityTests(BaseUDTFTestsMixin, ReusedConnectTestCase):
-    @classmethod
-    def setUpClass(cls):
-        super(UDTFParityTests, cls).setUpClass()
-        cls.spark.conf.set("spark.sql.execution.pythonUDTF.arrow.enabled", "false")
-
-    @classmethod
-    def tearDownClass(cls):
-        try:
-            cls.spark.conf.unset("spark.sql.execution.pythonUDTF.arrow.enabled")
-        finally:
-            super(UDTFParityTests, cls).tearDownClass()
-
-    # TODO: use PySpark error classes instead of SparkConnectGrpcException
-
-    def test_struct_output_type_casting_row(self):
-        self.check_struct_output_type_casting_row(SparkConnectGrpcException)
-
-    def test_udtf_with_invalid_return_type(self):
-        @udtf(returnType="int")
-        class TestUDTF:
-            def eval(self, a: int):
-                yield a + 1,
-
-        with self.assertRaisesRegex(
-            SparkConnectGrpcException, "Invalid Python user-defined table function return type."
-        ):
-            TestUDTF(lit(1)).collect()
-
-    @unittest.skipIf(
-        "SPARK_SKIP_CONNECT_COMPAT_TESTS" in os.environ, "Failed with different Client <> Server"
-    )
-    def test_udtf_init_with_additional_args(self):
-        super(UDTFParityTests, self).test_udtf_init_with_additional_args()
-
-    @unittest.skipIf(
-        "SPARK_SKIP_CONNECT_COMPAT_TESTS" in os.environ, "Failed with different Client <> Server"
-    )
-    def test_udtf_with_wrong_num_input(self):
-        super(UDTFParityTests, self).test_udtf_with_wrong_num_input()
-
-    @unittest.skipIf(
-        "SPARK_SKIP_CONNECT_COMPAT_TESTS" in os.environ, "Failed with different Client <> Server"
-    )
-    def test_array_output_type_casting(self):
-        super(UDTFParityTests, self).test_array_output_type_casting()
-
-    @unittest.skipIf(
-        "SPARK_SKIP_CONNECT_COMPAT_TESTS" in os.environ, "Failed with different Client <> Server"
-    )
-    def test_map_output_type_casting(self):
-        super(UDTFParityTests, self).test_map_output_type_casting()
-
-    @unittest.skipIf(
-        "SPARK_SKIP_CONNECT_COMPAT_TESTS" in os.environ, "Failed with different Client <> Server"
-    )
-    def test_numeric_output_type_casting(self):
-        super(UDTFParityTests, self).test_numeric_output_type_casting()
-
-    @unittest.skipIf(
-        "SPARK_SKIP_CONNECT_COMPAT_TESTS" in os.environ, "Failed with different Client <> Server"
-    )
-    def test_numeric_output_type_casting(self):
-        super(UDTFParityTests, self).test_numeric_output_type_casting()
-
-    @unittest.skipIf(
-        "SPARK_SKIP_CONNECT_COMPAT_TESTS" in os.environ, "Failed with different Client <> Server"
-    )
-    def test_numeric_string_output_type_casting(self):
-        super(UDTFParityTests, self).test_numeric_string_output_type_casting()
-
-    @unittest.skipIf(
-        "SPARK_SKIP_CONNECT_COMPAT_TESTS" in os.environ, "Failed with different Client <> Server"
-    )
-    def test_string_output_type_casting(self):
-        super(UDTFParityTests, self).test_string_output_type_casting()
-
-    @unittest.skipIf(
-        "SPARK_SKIP_CONNECT_COMPAT_TESTS" in os.environ, "Failed with different Client <> Server"
-    )
-    def test_string_output_type_casting(self):
-        super(UDTFParityTests, self).test_string_output_type_casting()
-
-    @unittest.skipIf(
-        "SPARK_SKIP_CONNECT_COMPAT_TESTS" in os.environ, "Failed with different Client <> Server"
-    )
-    def test_struct_output_type_casting_dict(self):
-        super(UDTFParityTests, self).test_struct_output_type_casting_dict()
-
-    @unittest.skipIf(
-        "SPARK_SKIP_CONNECT_COMPAT_TESTS" in os.environ, "Failed with different Client <> Server"
-    )
-    def test_udtf_init_with_additional_args(self):
-        super(UDTFParityTests, self).test_udtf_init_with_additional_args()
-
-    @unittest.skipIf(
-        "SPARK_SKIP_CONNECT_COMPAT_TESTS" in os.environ, "Failed with different Client <> Server"
-    )
-    def test_udtf_with_wrong_num_input(self):
-        super(UDTFParityTests, self).test_udtf_with_wrong_num_input()
-
-
-@unittest.skipIf(
-    os.environ.get("SPARK_SKIP_CONNECT_COMPAT_TESTS") == "1",
-    "Python UDTF with Arrow is still under development.",
-)
-class ArrowUDTFParityTests(UDTFArrowTestsMixin, UDTFParityTests):
-    @classmethod
-    def setUpClass(cls):
-        super(ArrowUDTFParityTests, cls).setUpClass()
-        cls.spark.conf.set("spark.sql.execution.pythonUDTF.arrow.enabled", "true")
-
-    @classmethod
-    def tearDownClass(cls):
-        try:
-            cls.spark.conf.unset("spark.sql.execution.pythonUDTF.arrow.enabled")
-        finally:
-            super(ArrowUDTFParityTests, cls).tearDownClass()
-
-
-if __name__ == "__main__":
-    import unittest
-    from pyspark.sql.tests.connect.test_parity_udtf import *  # noqa: F401
-
-    try:
-        import xmlrunner  # type: ignore[import]
-
-        testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
-    except ImportError:
-        testRunner = None
-    unittest.main(testRunner=testRunner, verbosity=2)

snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_session.py

@@ -1,181 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-import threading
-import time
-import unittest
-from typing import Optional
-
-from pyspark.sql.connect.client import ChannelBuilder
-from pyspark.sql.connect.session import SparkSession as RemoteSparkSession
-from pyspark.testing.connectutils import should_test_connect
-
-if should_test_connect:
-    from pyspark.testing.connectutils import ReusedConnectTestCase
-
-
-class CustomChannelBuilder(ChannelBuilder):
-    @property
-    def userId(self) -> Optional[str]:
-        return "abc"
-
-
-class SparkSessionTestCase(unittest.TestCase):
-    def test_fails_to_create_session_without_remote_and_channel_builder(self):
-        with self.assertRaises(ValueError):
-            RemoteSparkSession.builder.getOrCreate()
-
-    def test_fails_to_create_when_both_remote_and_channel_builder_are_specified(self):
-        with self.assertRaises(ValueError):
-            (
-                RemoteSparkSession.builder.channelBuilder(CustomChannelBuilder("sc://localhost"))
-                .remote("sc://localhost")
-                .getOrCreate()
-            )
-
-    def test_creates_session_with_channel_builder(self):
-        test_session = RemoteSparkSession.builder.channelBuilder(
-            CustomChannelBuilder("sc://other")
-        ).getOrCreate()
-        host = test_session.client.host
-        test_session.stop()
-
-        self.assertEqual("other", host)
-
-    def test_creates_session_with_remote(self):
-        test_session = RemoteSparkSession.builder.remote("sc://other").getOrCreate()
-        host = test_session.client.host
-        test_session.stop()
-
-        self.assertEqual("other", host)
-
-    def test_session_stop(self):
-        session = RemoteSparkSession.builder.remote("sc://other").getOrCreate()
-
-        self.assertFalse(session.is_stopped)
-        session.stop()
-        self.assertTrue(session.is_stopped)
-
-    def test_session_create_sets_active_session(self):
-        session = RemoteSparkSession.builder.remote("sc://abc").create()
-        session2 = RemoteSparkSession.builder.remote("sc://other").getOrCreate()
-
-        self.assertIs(session, session2)
-        session.stop()
-
-
-class JobCancellationTests(ReusedConnectTestCase):
-    def test_tags(self):
-        self.spark.clearTags()
-        self.spark.addTag("a")
-        self.assertEqual(self.spark.getTags(), {"a"})
-        self.spark.addTag("b")
-        self.spark.removeTag("a")
-        self.assertEqual(self.spark.getTags(), {"b"})
-        self.spark.addTag("c")
-        self.spark.clearTags()
-        self.assertEqual(self.spark.getTags(), set())
-        self.spark.clearTags()
-
-    def test_interrupt_tag(self):
-        thread_ids = range(4)
-        self.check_job_cancellation(
-            lambda job_group: self.spark.addTag(job_group),
-            lambda job_group: self.spark.interruptTag(job_group),
-            thread_ids,
-            [i for i in thread_ids if i % 2 == 0],
-            [i for i in thread_ids if i % 2 != 0],
-        )
-        self.spark.clearTags()
-
-    def test_interrupt_all(self):
-        thread_ids = range(4)
-        self.check_job_cancellation(
-            lambda job_group: None,
-            lambda job_group: self.spark.interruptAll(),
-            thread_ids,
-            thread_ids,
-            [],
-        )
-        self.spark.clearTags()
-
-    def check_job_cancellation(
-        self, setter, canceller, thread_ids, thread_ids_to_cancel, thread_ids_to_run
-    ):
-
-        job_id_a = "job_ids_to_cancel"
-        job_id_b = "job_ids_to_run"
-        threads = []
-
-        # A list which records whether job is cancelled.
-        # The index of the array is the thread index which job run in.
-        is_job_cancelled = [False for _ in thread_ids]
-
-        def run_job(job_id, index):
-            """
-            Executes a job with the group ``job_group``. Each job waits for 3 seconds
-            and then exits.
-            """
-            try:
-                setter(job_id)
-
-                def func(itr):
-                    for pdf in itr:
-                        time.sleep(pdf._1.iloc[0])
-                        yield pdf
-
-                self.spark.createDataFrame([[20]]).repartition(1).mapInPandas(
-                    func, schema="_1 LONG"
-                ).collect()
-                is_job_cancelled[index] = False
-            except Exception:
-                # Assume that exception means job cancellation.
-                is_job_cancelled[index] = True
-
-        # Test if job succeeded when not cancelled.
-        run_job(job_id_a, 0)
-        self.assertFalse(is_job_cancelled[0])
-        self.spark.clearTags()
-
-        # Run jobs
-        for i in thread_ids_to_cancel:
-            t = threading.Thread(target=run_job, args=(job_id_a, i))
-            t.start()
-            threads.append(t)
-
-        for i in thread_ids_to_run:
-            t = threading.Thread(target=run_job, args=(job_id_b, i))
-            t.start()
-            threads.append(t)
-
-        # Wait to make sure all jobs are executed.
-        time.sleep(10)
-        # And then, cancel one job group.
-        canceller(job_id_a)
-
-        # Wait until all threads launching jobs are finished.
-        for t in threads:
-            t.join()
-
-        for i in thread_ids_to_cancel:
-            self.assertTrue(
-                is_job_cancelled[i], "Thread {i}: Job in group A was not cancelled.".format(i=i)
-            )
-
-        for i in thread_ids_to_run:
-            self.assertFalse(
-                is_job_cancelled[i], "Thread {i}: Job in group B did not succeeded.".format(i=i)
-            )

snowflake/snowpark_connect/includes/python/pyspark/sql/tests/connect/test_utils.py

@@ -1,42 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-import os
-import unittest
-
-from pyspark.testing.connectutils import ReusedConnectTestCase
-from pyspark.sql.tests.test_utils import UtilsTestsMixin
-
-
-class ConnectUtilsTests(ReusedConnectTestCase, UtilsTestsMixin):
-    @unittest.skipIf(
-        "SPARK_SKIP_CONNECT_COMPAT_TESTS" in os.environ, "Failed with different Client <> Server"
-    )
-    def test_assert_approx_equal_decimaltype_custom_rtol_pass(self):
-        super(ConnectUtilsTests, self).test_assert_approx_equal_decimaltype_custom_rtol_pass()
-
-
-if __name__ == "__main__":
-    import unittest
-    from pyspark.sql.tests.connect.test_utils import *  # noqa: F401
-
-    try:
-        import xmlrunner
-
-        testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
-    except ImportError:
-        testRunner = None
-    unittest.main(testRunner=testRunner, verbosity=2)

snowflake/snowpark_connect/includes/python/pyspark/sql/tests/pandas/__init__.py

@@ -1,16 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#