snowpark-connect 0.20.2__py3-none-any.whl

snowflake/snowpark_connect/includes/python/pyspark/tests/test_statcounter.py

@@ -0,0 +1,130 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from pyspark.statcounter import StatCounter
+from pyspark.testing.utils import ReusedPySparkTestCase
+import math
+
+
+class StatCounterTests(ReusedPySparkTestCase):
+    def test_base(self):
+        stats = self.sc.parallelize([1.0, 2.0, 3.0, 4.0]).stats()
+        self.assertEqual(stats.count(), 4)
+        self.assertEqual(stats.max(), 4.0)
+        self.assertEqual(stats.mean(), 2.5)
+        self.assertEqual(stats.min(), 1.0)
+        self.assertAlmostEqual(stats.stdev(), 1.118033988749895)
+        self.assertAlmostEqual(stats.sampleStdev(), 1.2909944487358056)
+        self.assertEqual(stats.sum(), 10.0)
+        self.assertAlmostEqual(stats.variance(), 1.25)
+        self.assertAlmostEqual(stats.sampleVariance(), 1.6666666666666667)
+
+    def test_as_dict(self):
+        stats = self.sc.parallelize([1.0, 2.0, 3.0, 4.0]).stats().asDict()
+        self.assertEqual(stats["count"], 4)
+        self.assertEqual(stats["max"], 4.0)
+        self.assertEqual(stats["mean"], 2.5)
+        self.assertEqual(stats["min"], 1.0)
+        self.assertAlmostEqual(stats["stdev"], 1.2909944487358056)
+        self.assertEqual(stats["sum"], 10.0)
+        self.assertAlmostEqual(stats["variance"], 1.6666666666666667)
+
+        stats = self.sc.parallelize([1.0, 2.0, 3.0, 4.0]).stats().asDict(sample=True)
+        self.assertEqual(stats["count"], 4)
+        self.assertEqual(stats["max"], 4.0)
+        self.assertEqual(stats["mean"], 2.5)
+        self.assertEqual(stats["min"], 1.0)
+        self.assertAlmostEqual(stats["stdev"], 1.118033988749895)
+        self.assertEqual(stats["sum"], 10.0)
+        self.assertAlmostEqual(stats["variance"], 1.25)
+
+    def test_merge(self):
+        stats = StatCounter([1.0, 2.0, 3.0, 4.0])
+        stats.merge(5.0)
+        self.assertEqual(stats.count(), 5)
+        self.assertEqual(stats.max(), 5.0)
+        self.assertEqual(stats.mean(), 3.0)
+        self.assertEqual(stats.min(), 1.0)
+        self.assertAlmostEqual(stats.stdev(), 1.414213562373095)
+        self.assertAlmostEqual(stats.sampleStdev(), 1.5811388300841898)
+        self.assertEqual(stats.sum(), 15.0)
+        self.assertAlmostEqual(stats.variance(), 2.0)
+        self.assertAlmostEqual(stats.sampleVariance(), 2.5)
+
+    def test_merge_stats(self):
+        stats1 = StatCounter([1.0, 2.0, 3.0, 4.0])
+        stats2 = StatCounter([1.0, 2.0, 3.0, 4.0])
+        stats = stats1.mergeStats(stats2)
+        self.assertEqual(stats.count(), 8)
+        self.assertEqual(stats.max(), 4.0)
+        self.assertEqual(stats.mean(), 2.5)
+        self.assertEqual(stats.min(), 1.0)
+        self.assertAlmostEqual(stats.stdev(), 1.118033988749895)
+        self.assertAlmostEqual(stats.sampleStdev(), 1.1952286093343936)
+        self.assertEqual(stats.sum(), 20.0)
+        self.assertAlmostEqual(stats.variance(), 1.25)
+        self.assertAlmostEqual(stats.sampleVariance(), 1.4285714285714286)
+        execution_statements = [
+            StatCounter([1.0, 2.0]).mergeStats(StatCounter(range(1, 301))),
+            StatCounter(range(1, 301)).mergeStats(StatCounter([1.0, 2.0])),
+        ]
+        for stats in execution_statements:
+            self.assertEqual(stats.count(), 302)
+            self.assertEqual(stats.max(), 300.0)
+            self.assertEqual(stats.min(), 1.0)
+            self.assertAlmostEqual(stats.mean(), 149.51324503311)
+            self.assertAlmostEqual(stats.variance(), 7596.302804701549)
+            self.assertAlmostEqual(stats.sampleVariance(), 7621.539691095905)
+
+    def test_variance_when_size_zero(self):
+        # SPARK-38854: Test case to improve test coverage when
+        # StatCounter argument is empty list or None
+        arguments = [[], None]
+
+        for arg in arguments:
+            stats = StatCounter(arg)
+            self.assertTrue(math.isnan(stats.variance()))
+            self.assertTrue(math.isnan(stats.sampleVariance()))
+            self.assertEqual(stats.count(), 0)
+            self.assertTrue(math.isinf(stats.max()))
+            self.assertTrue(math.isinf(stats.min()))
+            self.assertEqual(stats.mean(), 0.0)
+
+    def test_merge_stats_with_self(self):
+        stats = StatCounter([1.0, 2.0, 3.0, 4.0])
+        stats.mergeStats(stats)
+        self.assertEqual(stats.count(), 8)
+        self.assertEqual(stats.max(), 4.0)
+        self.assertEqual(stats.mean(), 2.5)
+        self.assertEqual(stats.min(), 1.0)
+        self.assertAlmostEqual(stats.stdev(), 1.118033988749895)
+        self.assertAlmostEqual(stats.sampleStdev(), 1.1952286093343936)
+        self.assertEqual(stats.sum(), 20.0)
+        self.assertAlmostEqual(stats.variance(), 1.25)
+        self.assertAlmostEqual(stats.sampleVariance(), 1.4285714285714286)
+
+
+if __name__ == "__main__":
+    import unittest
+    from pyspark.tests.test_statcounter import *  # noqa: F401
+
+    try:
+        import xmlrunner
+
+        testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)

snowflake/snowpark_connect/includes/python/pyspark/tests/test_taskcontext.py

@@ -0,0 +1,350 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import os
+import random
+import stat
+import sys
+import tempfile
+import time
+import unittest
+
+from pyspark import SparkConf, SparkContext, TaskContext, BarrierTaskContext
+from pyspark.testing.utils import PySparkTestCase, SPARK_HOME, eventually
+
+
+class TaskContextTests(PySparkTestCase):
+    def setUp(self):
+        self._old_sys_path = list(sys.path)
+        class_name = self.__class__.__name__
+        # Allow retries even though they are normally disabled in local mode
+        self.sc = SparkContext("local[4, 2]", class_name)
+
+    def test_stage_id(self):
+        """Test the stage ids are available and incrementing as expected."""
+        rdd = self.sc.parallelize(range(10))
+        stage1 = rdd.map(lambda x: TaskContext.get().stageId()).take(1)[0]
+        stage2 = rdd.map(lambda x: TaskContext.get().stageId()).take(1)[0]
+        # Test using the constructor directly rather than the get()
+        stage3 = rdd.map(lambda x: TaskContext().stageId()).take(1)[0]
+        self.assertEqual(stage1 + 1, stage2)
+        self.assertEqual(stage1 + 2, stage3)
+        self.assertEqual(stage2 + 1, stage3)
+
+    def test_resources(self):
+        """Test the resources are empty by default."""
+        rdd = self.sc.parallelize(range(10))
+        resources1 = rdd.map(lambda x: TaskContext.get().resources()).take(1)[0]
+        # Test using the constructor directly rather than the get()
+        resources2 = rdd.map(lambda x: TaskContext().resources()).take(1)[0]
+        self.assertEqual(len(resources1), 0)
+        self.assertEqual(len(resources2), 0)
+
+    def test_partition_id(self):
+        """Test the partition id."""
+        rdd1 = self.sc.parallelize(range(10), 1)
+        rdd2 = self.sc.parallelize(range(10), 2)
+        pids1 = rdd1.map(lambda x: TaskContext.get().partitionId()).collect()
+        pids2 = rdd2.map(lambda x: TaskContext.get().partitionId()).collect()
+        self.assertEqual(0, pids1[0])
+        self.assertEqual(0, pids1[9])
+        self.assertEqual(0, pids2[0])
+        self.assertEqual(1, pids2[9])
+
+    def test_attempt_number(self):
+        """Verify the attempt numbers are correctly reported."""
+        rdd = self.sc.parallelize(range(10))
+        # Verify a simple job with no failures
+        attempt_numbers = rdd.map(lambda x: TaskContext.get().attemptNumber()).collect()
+        map(lambda attempt: self.assertEqual(0, attempt), attempt_numbers)
+
+        def fail_on_first(x):
+            """Fail on the first attempt so we get a positive attempt number"""
+            tc = TaskContext.get()
+            attempt_number = tc.attemptNumber()
+            partition_id = tc.partitionId()
+            attempt_id = tc.taskAttemptId()
+            if attempt_number == 0 and partition_id == 0:
+                raise RuntimeError("Failing on first attempt")
+            else:
+                return [x, partition_id, attempt_number, attempt_id]
+
+        result = rdd.map(fail_on_first).collect()
+        # We should re-submit the first partition to it but other partitions should be attempt 0
+        self.assertEqual([0, 0, 1], result[0][0:3])
+        self.assertEqual([9, 3, 0], result[9][0:3])
+        first_partition = filter(lambda x: x[1] == 0, result)
+        map(lambda x: self.assertEqual(1, x[2]), first_partition)
+        other_partitions = filter(lambda x: x[1] != 0, result)
+        map(lambda x: self.assertEqual(0, x[2]), other_partitions)
+        # The task attempt id should be different
+        self.assertTrue(result[0][3] != result[9][3])
+
+    def test_tc_on_driver(self):
+        """Verify that getting the TaskContext on the driver returns None."""
+        tc = TaskContext.get()
+        self.assertTrue(tc is None)
+
+    def test_get_local_property(self):
+        """Verify that local properties set on the driver are available in TaskContext."""
+        key = "testkey"
+        value = "testvalue"
+        self.sc.setLocalProperty(key, value)
+        try:
+            rdd = self.sc.parallelize(range(1), 1)
+            prop1 = rdd.map(lambda _: TaskContext.get().getLocalProperty(key)).collect()[0]
+            self.assertEqual(prop1, value)
+            prop2 = rdd.map(lambda _: TaskContext.get().getLocalProperty("otherkey")).collect()[0]
+            self.assertTrue(prop2 is None)
+        finally:
+            self.sc.setLocalProperty(key, None)
+
+    def test_barrier(self):
+        """
+        Verify that BarrierTaskContext.barrier() performs global sync among all barrier tasks
+        within a stage.
+        """
+        rdd = self.sc.parallelize(range(10), 4)
+
+        def f(iterator):
+            yield sum(iterator)
+
+        def context_barrier(x):
+            tc = BarrierTaskContext.get()
+            time.sleep(random.randint(1, 5) * 2)
+            tc.barrier()
+            return time.time()
+
+        times = rdd.barrier().mapPartitions(f).map(context_barrier).collect()
+        self.assertTrue(max(times) - min(times) < 2)
+
+    def test_all_gather(self):
+        """
+        Verify that BarrierTaskContext.allGather() performs global sync among all barrier tasks
+        within a stage and passes messages properly.
+        """
+        rdd = self.sc.parallelize(range(10), 4)
+
+        def f(iterator):
+            yield sum(iterator)
+
+        def context_barrier(x):
+            tc = BarrierTaskContext.get()
+            time.sleep(random.randint(1, 10))
+            out = tc.allGather(str(tc.partitionId()))
+            pids = [int(e) for e in out]
+            return pids
+
+        pids = rdd.barrier().mapPartitions(f).map(context_barrier).collect()[0]
+        self.assertEqual(pids, [0, 1, 2, 3])
+
+    def test_barrier_infos(self):
+        """
+        Verify that BarrierTaskContext.getTaskInfos() returns a list of all task infos in the
+        barrier stage.
+        """
+        rdd = self.sc.parallelize(range(10), 4)
+
+        def f(iterator):
+            yield sum(iterator)
+
+        taskInfos = (
+            rdd.barrier()
+            .mapPartitions(f)
+            .map(lambda x: BarrierTaskContext.get().getTaskInfos())
+            .collect()
+        )
+        self.assertTrue(len(taskInfos) == 4)
+        self.assertTrue(len(taskInfos[0]) == 4)
+
+    def test_context_get(self):
+        """
+        Verify that TaskContext.get() works both in or not in a barrier stage.
+        """
+        rdd = self.sc.parallelize(range(10), 4)
+
+        def f(iterator):
+            taskContext = TaskContext.get()
+            if isinstance(taskContext, BarrierTaskContext):
+                yield taskContext.partitionId() + 1
+            elif isinstance(taskContext, TaskContext):
+                yield taskContext.partitionId() + 2
+            else:
+                yield -1
+
+        # for normal stage
+        result1 = rdd.mapPartitions(f).collect()
+        self.assertTrue(result1 == [2, 3, 4, 5])
+        # for barrier stage
+        result2 = rdd.barrier().mapPartitions(f).collect()
+        self.assertTrue(result2 == [1, 2, 3, 4])
+
+    def test_barrier_context_get(self):
+        """
+        Verify that BarrierTaskContext.get() should only works in a barrier stage.
+        """
+        rdd = self.sc.parallelize(range(10), 4)
+
+        def f(iterator):
+            try:
+                taskContext = BarrierTaskContext.get()
+            except Exception:
+                yield -1
+            else:
+                yield taskContext.partitionId()
+
+        # for normal stage
+        result1 = rdd.mapPartitions(f).collect()
+        self.assertTrue(result1 == [-1, -1, -1, -1])
+        # for barrier stage
+        result2 = rdd.barrier().mapPartitions(f).collect()
+        self.assertTrue(result2 == [0, 1, 2, 3])
+
+
+class TaskContextTestsWithWorkerReuse(unittest.TestCase):
+    def setUp(self):
+        class_name = self.__class__.__name__
+        conf = SparkConf().set("spark.python.worker.reuse", "true")
+        self.sc = SparkContext("local[2]", class_name, conf=conf)
+
+    def test_barrier_with_python_worker_reuse(self):
+        """
+        Regression test for SPARK-25921: verify that BarrierTaskContext.barrier() with
+        reused python worker.
+        """
+        # start a normal job first to start all workers and get all worker pids
+        worker_pids = self.sc.parallelize(range(2), 2).map(lambda x: os.getpid()).collect()
+        # the worker will reuse in this barrier job
+        rdd = self.sc.parallelize(range(10), 2)
+
+        def f(iterator):
+            yield sum(iterator)
+
+        def context_barrier(x):
+            tc = BarrierTaskContext.get()
+            time.sleep(random.randint(1, 5) * 2)
+            tc.barrier()
+            return (time.time(), os.getpid())
+
+        result = rdd.barrier().mapPartitions(f).map(context_barrier).collect()
+        times = list(map(lambda x: x[0], result))
+        pids = list(map(lambda x: x[1], result))
+        # check both barrier and worker reuse effect
+        self.assertTrue(max(times) - min(times) < 2)
+        for pid in pids:
+            self.assertTrue(pid in worker_pids)
+
+    def check_task_context_correct_with_python_worker_reuse(self):
+        """Verify the task context correct when reused python worker"""
+        # start a normal job first to start all workers and get all worker pids
+        worker_pids = self.sc.parallelize(range(2), 2).map(lambda x: os.getpid()).collect()
+        # the worker will reuse in this barrier job
+        rdd = self.sc.parallelize(range(10), 2)
+
+        def context(iterator):
+            tp = TaskContext.get().partitionId()
+            try:
+                bp = BarrierTaskContext.get().partitionId()
+            except Exception:
+                bp = -1
+
+            yield (tp, bp, os.getpid())
+
+        # normal stage after normal stage
+        normal_result = rdd.mapPartitions(context).collect()
+        tps, bps, pids = zip(*normal_result)
+        self.assertTrue(tps == (0, 1))
+        self.assertTrue(bps == (-1, -1))
+        for pid in pids:
+            self.assertTrue(pid in worker_pids)
+        # barrier stage after normal stage
+        barrier_result = rdd.barrier().mapPartitions(context).collect()
+        tps, bps, pids = zip(*barrier_result)
+        self.assertTrue(tps == (0, 1))
+        self.assertTrue(bps == (0, 1))
+        for pid in pids:
+            self.assertTrue(pid in worker_pids)
+        # normal stage after barrier stage
+        normal_result2 = rdd.mapPartitions(context).collect()
+        tps, bps, pids = zip(*normal_result2)
+        self.assertTrue(tps == (0, 1))
+        self.assertTrue(bps == (-1, -1))
+        for pid in pids:
+            self.assertTrue(pid in worker_pids)
+        return True
+
+    def test_task_context_correct_with_python_worker_reuse(self):
+        # Retrying the check as the PIDs from Python workers might be different even
+        # when reusing Python workers is enabled if a Python worker is dead for some reasons
+        # (e.g., socket connection failure) and new Python worker is created.
+        eventually(self.check_task_context_correct_with_python_worker_reuse, catch_assertions=True)
+
+    def tearDown(self):
+        self.sc.stop()
+
+
+class TaskContextTestsWithResources(unittest.TestCase):
+    def setUp(self):
+        class_name = self.__class__.__name__
+        self.tempFile = tempfile.NamedTemporaryFile(delete=False)
+        self.tempFile.write(b'echo {\\"name\\": \\"gpu\\", \\"addresses\\": [\\"0\\"]}')
+        self.tempFile.close()
+        # create temporary directory for Worker resources coordination
+        self.tempdir = tempfile.NamedTemporaryFile(delete=False)
+        os.unlink(self.tempdir.name)
+        os.chmod(
+            self.tempFile.name,
+            stat.S_IRWXU | stat.S_IXGRP | stat.S_IRGRP | stat.S_IROTH | stat.S_IXOTH,
+        )
+        conf = SparkConf().set("spark.test.home", SPARK_HOME)
+        conf = conf.set("spark.worker.resource.gpu.discoveryScript", self.tempFile.name)
+        conf = conf.set("spark.worker.resource.gpu.amount", 1)
+        conf = conf.set("spark.task.cpus", 2)
+        conf = conf.set("spark.task.resource.gpu.amount", "1")
+        conf = conf.set("spark.executor.resource.gpu.amount", "1")
+        self.sc = SparkContext("local-cluster[2,2,1024]", class_name, conf=conf)
+
+    def test_cpus(self):
+        """Test the cpus are available."""
+        rdd = self.sc.parallelize(range(10))
+        cpus = rdd.map(lambda x: TaskContext.get().cpus()).take(1)[0]
+        self.assertEqual(cpus, 2)
+
+    def test_resources(self):
+        """Test the resources are available."""
+        rdd = self.sc.parallelize(range(10))
+        resources = rdd.map(lambda x: TaskContext.get().resources()).take(1)[0]
+        self.assertEqual(len(resources), 1)
+        self.assertTrue("gpu" in resources)
+        self.assertEqual(resources["gpu"].name, "gpu")
+        self.assertEqual(resources["gpu"].addresses, ["0"])
+
+    def tearDown(self):
+        os.unlink(self.tempFile.name)
+        self.sc.stop()
+
+
+if __name__ == "__main__":
+    import unittest
+    from pyspark.tests.test_taskcontext import *  # noqa: F401
+
+    try:
+        import xmlrunner
+
+        testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)

snowflake/snowpark_connect/includes/python/pyspark/tests/test_util.py

@@ -0,0 +1,97 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import os
+import unittest
+
+from py4j.protocol import Py4JJavaError
+
+from pyspark import keyword_only
+from pyspark.testing.utils import PySparkTestCase
+from pyspark.find_spark_home import _find_spark_home
+
+
+class KeywordOnlyTests(unittest.TestCase):
+    class Wrapped:
+        @keyword_only
+        def set(self, x=None, y=None):
+            if "x" in self._input_kwargs:
+                self._x = self._input_kwargs["x"]
+            if "y" in self._input_kwargs:
+                self._y = self._input_kwargs["y"]
+            return x, y
+
+    def test_keywords(self):
+        w = self.Wrapped()
+        x, y = w.set(y=1)
+        self.assertEqual(y, 1)
+        self.assertEqual(y, w._y)
+        self.assertIsNone(x)
+        self.assertFalse(hasattr(w, "_x"))
+
+    def test_non_keywords(self):
+        w = self.Wrapped()
+        self.assertRaises(TypeError, lambda: w.set(0, y=1))
+
+    def test_kwarg_ownership(self):
+        # test _input_kwargs is owned by each class instance and not a shared static variable
+        class Setter:
+            @keyword_only
+            def set(self, x=None, other=None, other_x=None):
+                if "other" in self._input_kwargs:
+                    self._input_kwargs["other"].set(x=self._input_kwargs["other_x"])
+                self._x = self._input_kwargs["x"]
+
+        a = Setter()
+        b = Setter()
+        a.set(x=1, other=b, other_x=2)
+        self.assertEqual(a._x, 1)
+        self.assertEqual(b._x, 2)
+
+
+class UtilTests(PySparkTestCase):
+    def test_py4j_str(self):
+        with self.assertRaises(Py4JJavaError) as context:
+            # This attempts java.lang.String(null) which throws an NPE.
+            self.sc._jvm.java.lang.String(None)
+
+        self.assertTrue("NullPointerException" in str(context.exception))
+
+    def test_parsing_version_string(self):
+        from pyspark.util import VersionUtils
+
+        self.assertRaises(ValueError, lambda: VersionUtils.majorMinorVersion("abced"))
+
+    def test_find_spark_home(self):
+        # SPARK-38827: Test find_spark_home without `SPARK_HOME` environment variable set.
+        origin = os.environ["SPARK_HOME"]
+        try:
+            del os.environ["SPARK_HOME"]
+            self.assertEquals(origin, _find_spark_home())
+        finally:
+            os.environ["SPARK_HOME"] = origin
+
+
+if __name__ == "__main__":
+    from pyspark.tests.test_util import *  # noqa: F401
+
+    try:
+        import xmlrunner
+
+        testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
+    except ImportError:
+        testRunner = None
+    unittest.main(testRunner=testRunner, verbosity=2)