snowpark-connect 0.24.0__py3-none-any.whl → 0.25.0__py3-none-any.whl

snowflake/snowpark_connect/includes/python/pyspark/tests/test_install_spark.py

@@ -1,124 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-import os
-import tempfile
-import unittest
-
-from pyspark.install import (
-    install_spark,
-    DEFAULT_HADOOP,
-    DEFAULT_HIVE,
-    UNSUPPORTED_COMBINATIONS,
-    checked_versions,
-    checked_package_name,
-)
-
-
-class SparkInstallationTestCase(unittest.TestCase):
-    def test_install_spark(self):
-        # Test only one case. Testing this is expensive because it needs to download
-        # the Spark distribution.
-        spark_version, hadoop_version, hive_version = checked_versions("3.4.4", "3", "2.3")
-
-        with tempfile.TemporaryDirectory() as tmp_dir:
-            install_spark(
-                dest=tmp_dir,
-                spark_version=spark_version,
-                hadoop_version=hadoop_version,
-                hive_version=hive_version,
-            )
-
-            self.assertTrue(os.path.isdir("%s/jars" % tmp_dir))
-            self.assertTrue(os.path.exists("%s/bin/spark-submit" % tmp_dir))
-            self.assertTrue(os.path.exists("%s/RELEASE" % tmp_dir))
-
-    def test_package_name(self):
-        self.assertEqual(
-            "spark-3.0.0-bin-hadoop3.2", checked_package_name("spark-3.0.0", "hadoop3.2", "hive2.3")
-        )
-
-        spark_version, hadoop_version, hive_version = checked_versions("3.2.0", "3", "2.3")
-        self.assertEqual(
-            "spark-3.2.0-bin-hadoop3.2",
-            checked_package_name(spark_version, hadoop_version, hive_version),
-        )
-
-        spark_version, hadoop_version, hive_version = checked_versions("3.3.0", "3", "2.3")
-        self.assertEqual(
-            "spark-3.3.0-bin-hadoop3",
-            checked_package_name(spark_version, hadoop_version, hive_version),
-        )
-
-    def test_checked_versions(self):
-        test_version = "3.0.1"  # Just pick one version to test.
-
-        # Positive test cases
-        self.assertEqual(
-            ("spark-2.4.1", "without-hadoop", "hive2.3"),
-            checked_versions("2.4.1", "without", "2.3"),
-        )
-
-        self.assertEqual(
-            ("spark-3.0.1", "without-hadoop", "hive2.3"),
-            checked_versions("spark-3.0.1", "without-hadoop", "hive2.3"),
-        )
-
-        self.assertEqual(
-            ("spark-3.3.0", "hadoop3", "hive2.3"),
-            checked_versions("spark-3.3.0", "hadoop3", "hive2.3"),
-        )
-
-        # Negative test cases
-        for (hadoop_version, hive_version) in UNSUPPORTED_COMBINATIONS:
-            with self.assertRaisesRegex(RuntimeError, "Hive.*should.*Hadoop"):
-                checked_versions(
-                    spark_version=test_version,
-                    hadoop_version=hadoop_version,
-                    hive_version=hive_version,
-                )
-
-        with self.assertRaisesRegex(RuntimeError, "Spark version should start with 'spark-'"):
-            checked_versions(
-                spark_version="malformed", hadoop_version=DEFAULT_HADOOP, hive_version=DEFAULT_HIVE
-            )
-
-        with self.assertRaisesRegex(RuntimeError, "Spark distribution.*malformed.*"):
-            checked_versions(
-                spark_version=test_version, hadoop_version="malformed", hive_version=DEFAULT_HIVE
-            )
-
-        with self.assertRaisesRegex(RuntimeError, "Spark distribution.*malformed.*"):
-            checked_versions(
-                spark_version=test_version, hadoop_version=DEFAULT_HADOOP, hive_version="malformed"
-            )
-
-        with self.assertRaisesRegex(RuntimeError, "Spark distribution of hive1.2 is not supported"):
-            checked_versions(
-                spark_version=test_version, hadoop_version="hadoop3", hive_version="hive1.2"
-            )
-
-
-if __name__ == "__main__":
-    from pyspark.tests.test_install_spark import *  # noqa: F401
-
-    try:
-        import xmlrunner
-
-        testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
-    except ImportError:
-        testRunner = None
-    unittest.main(testRunner=testRunner, verbosity=2)

snowflake/snowpark_connect/includes/python/pyspark/tests/test_join.py

@@ -1,69 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-from pyspark.testing.utils import ReusedPySparkTestCase
-
-
-class JoinTests(ReusedPySparkTestCase):
-    def test_narrow_dependency_in_join(self):
-        rdd = self.sc.parallelize(range(10)).map(lambda x: (x, x))
-        parted = rdd.partitionBy(2)
-        self.assertEqual(2, parted.union(parted).getNumPartitions())
-        self.assertEqual(rdd.getNumPartitions() + 2, parted.union(rdd).getNumPartitions())
-        self.assertEqual(rdd.getNumPartitions() + 2, rdd.union(parted).getNumPartitions())
-
-        tracker = self.sc.statusTracker()
-
-        self.sc.setJobGroup("test1", "test", True)
-        d = sorted(parted.join(parted).collect())
-        self.assertEqual(10, len(d))
-        self.assertEqual((0, (0, 0)), d[0])
-        jobId = tracker.getJobIdsForGroup("test1")[0]
-        self.assertEqual(2, len(tracker.getJobInfo(jobId).stageIds))
-
-        self.sc.setJobGroup("test2", "test", True)
-        d = sorted(parted.join(rdd).collect())
-        self.assertEqual(10, len(d))
-        self.assertEqual((0, (0, 0)), d[0])
-        jobId = tracker.getJobIdsForGroup("test2")[0]
-        self.assertEqual(3, len(tracker.getJobInfo(jobId).stageIds))
-
-        self.sc.setJobGroup("test3", "test", True)
-        d = sorted(parted.cogroup(parted).collect())
-        self.assertEqual(10, len(d))
-        self.assertEqual([[0], [0]], list(map(list, d[0][1])))
-        jobId = tracker.getJobIdsForGroup("test3")[0]
-        self.assertEqual(2, len(tracker.getJobInfo(jobId).stageIds))
-
-        self.sc.setJobGroup("test4", "test", True)
-        d = sorted(parted.cogroup(rdd).collect())
-        self.assertEqual(10, len(d))
-        self.assertEqual([[0], [0]], list(map(list, d[0][1])))
-        jobId = tracker.getJobIdsForGroup("test4")[0]
-        self.assertEqual(3, len(tracker.getJobInfo(jobId).stageIds))
-
-
-if __name__ == "__main__":
-    import unittest
-    from pyspark.tests.test_join import *  # noqa: F401
-
-    try:
-        import xmlrunner
-
-        testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
-    except ImportError:
-        testRunner = None
-    unittest.main(testRunner=testRunner, verbosity=2)

snowflake/snowpark_connect/includes/python/pyspark/tests/test_memory_profiler.py

@@ -1,167 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import os
-import sys
-import tempfile
-import unittest
-import warnings
-from io import StringIO
-from typing import Iterator
-from unittest import mock
-
-from pyspark import SparkConf, SparkContext
-from pyspark.profiler import has_memory_profiler
-from pyspark.sql import SparkSession
-from pyspark.sql.functions import pandas_udf, udf
-from pyspark.testing.sqlutils import have_pandas, pandas_requirement_message
-from pyspark.testing.utils import PySparkTestCase
-
-
-@unittest.skipIf(
-    "COVERAGE_PROCESS_START" in os.environ, "Flaky with coverage enabled, skipping for now."
-)
-@unittest.skipIf(not has_memory_profiler, "Must have memory-profiler installed.")
-@unittest.skipIf(not have_pandas, pandas_requirement_message)
-class MemoryProfilerTests(PySparkTestCase):
-    def setUp(self):
-        self._old_sys_path = list(sys.path)
-        class_name = self.__class__.__name__
-        conf = SparkConf().set("spark.python.profile.memory", "true")
-        self.sc = SparkContext("local[4]", class_name, conf=conf)
-        self.spark = SparkSession(sparkContext=self.sc)
-
-    def test_memory_profiler(self):
-        self.exec_python_udf()
-
-        profilers = self.sc.profiler_collector.profilers
-        self.assertEqual(1, len(profilers))
-        id, profiler, _ = profilers[0]
-        stats = profiler.stats()
-        self.assertTrue(stats is not None)
-
-        with mock.patch("sys.stdout", new=StringIO()) as fake_out:
-            self.sc.show_profiles()
-        self.assertTrue("plus_one" in fake_out.getvalue())
-
-        d = tempfile.gettempdir()
-        self.sc.dump_profiles(d)
-        self.assertTrue("udf_%d_memory.txt" % id in os.listdir(d))
-
-    def test_profile_pandas_udf(self):
-        udfs = [self.exec_pandas_udf_ser_to_ser, self.exec_pandas_udf_ser_to_scalar]
-        udf_names = ["ser_to_ser", "ser_to_scalar"]
-        for f, f_name in zip(udfs, udf_names):
-            f()
-            with mock.patch("sys.stdout", new=StringIO()) as fake_out:
-                self.sc.show_profiles()
-            self.assertTrue(f_name in fake_out.getvalue())
-
-        with warnings.catch_warnings(record=True) as warns:
-            warnings.simplefilter("always")
-            self.exec_pandas_udf_iter_to_iter()
-            user_warns = [warn.message for warn in warns if isinstance(warn.message, UserWarning)]
-            self.assertTrue(len(user_warns) > 0)
-            self.assertTrue(
-                "Profiling UDFs with iterators input/output is not supported" in str(user_warns[0])
-            )
-
-    def test_profile_pandas_function_api(self):
-        apis = [self.exec_grouped_map]
-        f_names = ["grouped_map"]
-        for api, f_name in zip(apis, f_names):
-            api()
-            with mock.patch("sys.stdout", new=StringIO()) as fake_out:
-                self.sc.show_profiles()
-            self.assertTrue(f_name in fake_out.getvalue())
-
-        with warnings.catch_warnings(record=True) as warns:
-            warnings.simplefilter("always")
-            self.exec_map()
-            user_warns = [warn.message for warn in warns if isinstance(warn.message, UserWarning)]
-            self.assertTrue(len(user_warns) > 0)
-            self.assertTrue(
-                "Profiling UDFs with iterators input/output is not supported" in str(user_warns[0])
-            )
-
-    def exec_python_udf(self):
-        @udf("int")
-        def plus_one(v):
-            return v + 1
-
-        self.spark.range(10).select(plus_one("id")).collect()
-
-    def exec_pandas_udf_ser_to_ser(self):
-        import pandas as pd
-
-        @pandas_udf("int")
-        def ser_to_ser(ser: pd.Series) -> pd.Series:
-            return ser + 1
-
-        self.spark.range(10).select(ser_to_ser("id")).collect()
-
-    def exec_pandas_udf_ser_to_scalar(self):
-        import pandas as pd
-
-        @pandas_udf("int")
-        def ser_to_scalar(ser: pd.Series) -> float:
-            return ser.median()
-
-        self.spark.range(10).select(ser_to_scalar("id")).collect()
-
-    # Unsupported
-    def exec_pandas_udf_iter_to_iter(self):
-        import pandas as pd
-
-        @pandas_udf("int")
-        def iter_to_iter(batch_ser: Iterator[pd.Series]) -> Iterator[pd.Series]:
-            for ser in batch_ser:
-                yield ser + 1
-
-        self.spark.range(10).select(iter_to_iter("id")).collect()
-
-    def exec_grouped_map(self):
-        import pandas as pd
-
-        def grouped_map(pdf: pd.DataFrame) -> pd.DataFrame:
-            return pdf.assign(v=pdf.v - pdf.v.mean())
-
-        df = self.spark.createDataFrame([(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0)], ("id", "v"))
-        df.groupby("id").applyInPandas(grouped_map, schema="id long, v double").collect()
-
-    # Unsupported
-    def exec_map(self):
-        import pandas as pd
-
-        def map(pdfs: Iterator[pd.DataFrame]) -> Iterator[pd.DataFrame]:
-            for pdf in pdfs:
-                yield pdf[pdf.id == 1]
-
-        df = self.spark.createDataFrame([(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0)], ("id", "v"))
-        df.mapInPandas(map, schema=df.schema).collect()
-
-
-if __name__ == "__main__":
-    from pyspark.tests.test_memory_profiler import *  # noqa: F401
-
-    try:
-        import xmlrunner
-
-        testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
-    except ImportError:
-        testRunner = None
-    unittest.main(testRunner=testRunner, verbosity=2)

snowflake/snowpark_connect/includes/python/pyspark/tests/test_pin_thread.py

@@ -1,194 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-import os
-import time
-import threading
-import unittest
-
-from pyspark import SparkContext, SparkConf, InheritableThread
-
-
-class PinThreadTests(unittest.TestCase):
-    # These tests are in a separate class because it uses
-    # 'PYSPARK_PIN_THREAD' environment variable to test thread pin feature.
-
-    @classmethod
-    def setUpClass(cls):
-        cls.old_pin_thread = os.environ.get("PYSPARK_PIN_THREAD")
-        os.environ["PYSPARK_PIN_THREAD"] = "true"
-        cls.sc = SparkContext("local[4]", cls.__name__, conf=SparkConf())
-
-    @classmethod
-    def tearDownClass(cls):
-        cls.sc.stop()
-        if cls.old_pin_thread is not None:
-            os.environ["PYSPARK_PIN_THREAD"] = cls.old_pin_thread
-        else:
-            del os.environ["PYSPARK_PIN_THREAD"]
-
-    def test_pinned_thread(self):
-        threads = []
-        exceptions = []
-        property_name = "test_property_%s" % PinThreadTests.__name__
-        jvm_thread_ids = []
-
-        for i in range(10):
-
-            def test_local_property():
-                jvm_thread_id = self.sc._jvm.java.lang.Thread.currentThread().getId()
-                jvm_thread_ids.append(jvm_thread_id)
-
-                # If a property is set in this thread, later it should get the same property
-                # within this thread.
-                self.sc.setLocalProperty(property_name, str(i))
-
-                # 5 threads, 1 second sleep. 5 threads without a sleep.
-                time.sleep(i % 2)
-
-                try:
-                    assert self.sc.getLocalProperty(property_name) == str(i)
-
-                    # Each command might create a thread in multi-threading mode in Py4J.
-                    # This assert makes sure that the created thread is being reused.
-                    assert jvm_thread_id == self.sc._jvm.java.lang.Thread.currentThread().getId()
-                except Exception as e:
-                    exceptions.append(e)
-
-            threads.append(threading.Thread(target=test_local_property))
-
-        for t in threads:
-            t.start()
-
-        for t in threads:
-            t.join()
-
-        for e in exceptions:
-            raise e
-
-        # Created JVM threads should be 10 because Python thread are 10.
-        assert len(set(jvm_thread_ids)) == 10
-
-    def test_multiple_group_jobs(self):
-        # SPARK-22340: Add a mode to pin Python thread into JVM's
-        self.check_job_cancellation(
-            lambda job_group: self.sc.setJobGroup(
-                job_group, "test rdd collect with setting job group"
-            ),
-            lambda job_group: self.sc.cancelJobGroup(job_group),
-        )
-
-    def test_multiple_group_tags(self):
-        # SPARK-44194: Test pinned thread mode with job tags.
-        self.check_job_cancellation(
-            lambda job_tag: self.sc.addJobTag(job_tag),
-            lambda job_tag: self.sc.cancelJobsWithTag(job_tag),
-        )
-
-    def check_job_cancellation(self, setter, canceller):
-
-        job_id_a = "job_ids_to_cancel"
-        job_id_b = "job_ids_to_run"
-
-        threads = []
-        thread_ids = range(4)
-        thread_ids_to_cancel = [i for i in thread_ids if i % 2 == 0]
-        thread_ids_to_run = [i for i in thread_ids if i % 2 != 0]
-
-        # A list which records whether job is cancelled.
-        # The index of the array is the thread index which job run in.
-        is_job_cancelled = [False for _ in thread_ids]
-
-        def run_job(job_id, index):
-            """
-            Executes a job with the group ``job_group``. Each job waits for 3 seconds
-            and then exits.
-            """
-            try:
-                setter(job_id)
-                self.sc.parallelize([15]).map(lambda x: time.sleep(x)).collect()
-                is_job_cancelled[index] = False
-            except Exception:
-                # Assume that exception means job cancellation.
-                is_job_cancelled[index] = True
-
-        # Test if job succeeded when not cancelled.
-        run_job(job_id_a, 0)
-        self.assertFalse(is_job_cancelled[0])
-
-        # Run jobs
-        for i in thread_ids_to_cancel:
-            t = threading.Thread(target=run_job, args=(job_id_a, i))
-            t.start()
-            threads.append(t)
-
-        for i in thread_ids_to_run:
-            t = threading.Thread(target=run_job, args=(job_id_b, i))
-            t.start()
-            threads.append(t)
-
-        # Wait to make sure all jobs are executed.
-        time.sleep(3)
-        # And then, cancel one job group.
-        canceller(job_id_a)
-
-        # Wait until all threads launching jobs are finished.
-        for t in threads:
-            t.join()
-
-        for i in thread_ids_to_cancel:
-            self.assertTrue(
-                is_job_cancelled[i], "Thread {i}: Job in group A was not cancelled.".format(i=i)
-            )
-
-        for i in thread_ids_to_run:
-            self.assertFalse(
-                is_job_cancelled[i], "Thread {i}: Job in group B did not succeeded.".format(i=i)
-            )
-
-    def test_inheritable_local_property(self):
-        self.sc.setLocalProperty("a", "hi")
-        expected = []
-
-        def get_inner_local_prop():
-            expected.append(self.sc.getLocalProperty("b"))
-
-        def get_outer_local_prop():
-            expected.append(self.sc.getLocalProperty("a"))
-            self.sc.setLocalProperty("b", "hello")
-            t2 = InheritableThread(target=get_inner_local_prop)
-            t2.start()
-            t2.join()
-
-        t1 = InheritableThread(target=get_outer_local_prop)
-        t1.start()
-        t1.join()
-
-        self.assertEqual(self.sc.getLocalProperty("b"), None)
-        self.assertEqual(expected, ["hi", "hello"])
-
-
-if __name__ == "__main__":
-    import unittest
-    from pyspark.tests.test_pin_thread import *  # noqa: F401
-
-    try:
-        import xmlrunner
-
-        testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
-    except ImportError:
-        testRunner = None
-    unittest.main(testRunner=testRunner, verbosity=2)