snowpark-connect 0.23.0__py3-none-any.whl → 0.25.0__py3-none-any.whl

snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_connect_pipeline.py

@@ -1,47 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-import os
-import unittest
-
-from pyspark.sql import SparkSession
-from pyspark.ml.tests.connect.test_legacy_mode_pipeline import PipelineTestsMixin
-
-
-@unittest.skipIf("SPARK_SKIP_CONNECT_COMPAT_TESTS" in os.environ, "Requires JVM access")
-class PipelineTestsOnConnect(PipelineTestsMixin, unittest.TestCase):
-    def setUp(self) -> None:
-        self.spark = (
-            SparkSession.builder.remote(os.environ.get("SPARK_CONNECT_TESTING_REMOTE", "local[2]"))
-            .config("spark.connect.copyFromLocalToFs.allowDestLocal", "true")
-            .getOrCreate()
-        )
-
-    def tearDown(self) -> None:
-        self.spark.stop()
-
-
-if __name__ == "__main__":
-    from pyspark.ml.tests.connect.test_connect_pipeline import *  # noqa: F401,F403
-
-    try:
-        import xmlrunner  # type: ignore[import]
-
-        testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
-    except ImportError:
-        testRunner = None
-    unittest.main(testRunner=testRunner, verbosity=2)

snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_connect_summarizer.py

@@ -1,43 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-import os
-import unittest
-
-from pyspark.sql import SparkSession
-from pyspark.ml.tests.connect.test_legacy_mode_summarizer import SummarizerTestsMixin
-
-
-class SummarizerTestsOnConnect(SummarizerTestsMixin, unittest.TestCase):
-    def setUp(self) -> None:
-        self.spark = SparkSession.builder.remote(
-            os.environ.get("SPARK_CONNECT_TESTING_REMOTE", "local[2]")
-        ).getOrCreate()
-
-    def tearDown(self) -> None:
-        self.spark.stop()
-
-
-if __name__ == "__main__":
-    from pyspark.ml.tests.connect.test_connect_summarizer import *  # noqa: F401,F403
-
-    try:
-        import xmlrunner  # type: ignore[import]
-
-        testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
-    except ImportError:
-        testRunner = None
-    unittest.main(testRunner=testRunner, verbosity=2)

snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_connect_tuning.py

@@ -1,46 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-import os
-import unittest
-from pyspark.sql import SparkSession
-from pyspark.ml.tests.connect.test_legacy_mode_tuning import CrossValidatorTestsMixin
-
-
-@unittest.skipIf("SPARK_SKIP_CONNECT_COMPAT_TESTS" in os.environ, "Requires JVM access")
-class CrossValidatorTestsOnConnect(CrossValidatorTestsMixin, unittest.TestCase):
-    def setUp(self) -> None:
-        self.spark = (
-            SparkSession.builder.remote(os.environ.get("SPARK_CONNECT_TESTING_REMOTE", "local[2]"))
-            .config("spark.connect.copyFromLocalToFs.allowDestLocal", "true")
-            .getOrCreate()
-        )
-
-    def tearDown(self) -> None:
-        self.spark.stop()
-
-
-if __name__ == "__main__":
-    from pyspark.ml.tests.connect.test_connect_tuning import *  # noqa: F401,F403
-
-    try:
-        import xmlrunner  # type: ignore[import]
-
-        testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
-    except ImportError:
-        testRunner = None
-    unittest.main(testRunner=testRunner, verbosity=2)

snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_legacy_mode_classification.py

@@ -1,238 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-import os
-import tempfile
-import unittest
-import numpy as np
-from pyspark.ml.connect.classification import (
-    LogisticRegression as LORV2,
-    LogisticRegressionModel as LORV2Model,
-)
-from pyspark.sql import SparkSession
-
-
-have_torch = True
-try:
-    import torch  # noqa: F401
-except ImportError:
-    have_torch = False
-
-
-class ClassificationTestsMixin:
-    @staticmethod
-    def _check_result(result_dataframe, expected_predictions, expected_probabilities=None):
-        np.testing.assert_array_equal(list(result_dataframe.prediction), expected_predictions)
-        if "probability" in result_dataframe.columns:
-            np.testing.assert_allclose(
-                list(result_dataframe.probability),
-                expected_probabilities,
-                rtol=1e-1,
-            )
-
-    def test_binary_classes_logistic_regression(self):
-        df1 = self.spark.createDataFrame(
-            [
-                (1.0, [0.0, 5.0]),
-                (0.0, [1.0, 2.0]),
-                (1.0, [2.0, 1.0]),
-                (0.0, [3.0, 3.0]),
-            ]
-            * 100,
-            ["label", "features"],
-        )
-        eval_df1 = self.spark.createDataFrame(
-            [
-                ([0.0, 2.0],),
-                ([3.5, 3.0],),
-            ],
-            ["features"],
-        )
-
-        lorv2 = LORV2(maxIter=200, numTrainWorkers=2, learningRate=0.001)
-        assert lorv2.getMaxIter() == 200
-        assert lorv2.getNumTrainWorkers() == 2
-        assert lorv2.getOrDefault(lorv2.learningRate) == 0.001
-
-        model = lorv2.fit(df1)
-        assert model.uid == lorv2.uid
-
-        expected_predictions = [1, 0]
-        expected_probabilities = [
-            [0.217875, 0.782125],
-            [0.839615, 0.160385],
-        ]
-
-        result = model.transform(eval_df1).toPandas()
-        self._check_result(result, expected_predictions, expected_probabilities)
-        local_transform_result = model.transform(eval_df1.toPandas())
-        self._check_result(local_transform_result, expected_predictions, expected_probabilities)
-
-        model.set(model.probabilityCol, "")
-        result_without_prob = model.transform(eval_df1).toPandas()
-        assert "probability" not in result_without_prob.columns
-        self._check_result(result_without_prob, expected_predictions, None)
-
-    def test_multi_classes_logistic_regression(self):
-        df1 = self.spark.createDataFrame(
-            [
-                (1.0, [1.0, 5.0]),
-                (2.0, [1.0, -2.0]),
-                (0.0, [-2.0, 1.5]),
-            ]
-            * 100,
-            ["label", "features"],
-        )
-        eval_df1 = self.spark.createDataFrame(
-            [
-                ([1.5, 5.0],),
-                ([1.0, -2.5],),
-                ([-2.0, 1.0],),
-            ],
-            ["features"],
-        )
-
-        lorv2 = LORV2(maxIter=200, numTrainWorkers=2, learningRate=0.001)
-
-        model = lorv2.fit(df1)
-
-        expected_predictions = [1, 2, 0]
-        expected_probabilities = [
-            [5.526459e-03, 9.943553e-01, 1.183146e-04],
-            [4.629959e-03, 8.141352e-03, 9.872288e-01],
-            [9.624363e-01, 3.080821e-02, 6.755549e-03],
-        ]
-
-        result = model.transform(eval_df1).toPandas()
-        self._check_result(result, expected_predictions, expected_probabilities)
-        local_transform_result = model.transform(eval_df1.toPandas())
-        self._check_result(local_transform_result, expected_predictions, expected_probabilities)
-
-    def test_save_load(self):
-        with tempfile.TemporaryDirectory() as tmp_dir:
-            estimator = LORV2(maxIter=2, numTrainWorkers=2, learningRate=0.001)
-            local_path = os.path.join(tmp_dir, "estimator")
-            estimator.saveToLocal(local_path)
-            loaded_estimator = LORV2.loadFromLocal(local_path)
-            assert loaded_estimator.uid == estimator.uid
-            assert loaded_estimator.getOrDefault(loaded_estimator.maxIter) == 2
-            assert loaded_estimator.getOrDefault(loaded_estimator.numTrainWorkers) == 2
-            assert loaded_estimator.getOrDefault(loaded_estimator.learningRate) == 0.001
-
-            # test overwriting
-            estimator2 = estimator.copy()
-            estimator2.set(estimator2.maxIter, 10)
-            estimator2.saveToLocal(local_path, overwrite=True)
-            loaded_estimator2 = LORV2.loadFromLocal(local_path)
-            assert loaded_estimator2.getOrDefault(loaded_estimator2.maxIter) == 10
-
-            fs_path = os.path.join(tmp_dir, "fs", "estimator")
-            estimator.save(fs_path)
-            loaded_estimator = LORV2.load(fs_path)
-            assert loaded_estimator.uid == estimator.uid
-            assert loaded_estimator.getOrDefault(loaded_estimator.maxIter) == 2
-            assert loaded_estimator.getOrDefault(loaded_estimator.numTrainWorkers) == 2
-            assert loaded_estimator.getOrDefault(loaded_estimator.learningRate) == 0.001
-
-            training_dataset = self.spark.createDataFrame(
-                [
-                    (1.0, [0.0, 5.0]),
-                    (0.0, [1.0, 2.0]),
-                    (1.0, [2.0, 1.0]),
-                    (0.0, [3.0, 3.0]),
-                ]
-                * 100,
-                ["label", "features"],
-            )
-            eval_df1 = self.spark.createDataFrame(
-                [
-                    ([0.0, 2.0],),
-                    ([3.5, 3.0],),
-                ],
-                ["features"],
-            )
-
-            model = estimator.fit(training_dataset)
-            model_predictions = model.transform(eval_df1.toPandas())
-
-            assert model.uid == estimator.uid
-
-            local_model_path = os.path.join(tmp_dir, "model")
-            model.saveToLocal(local_model_path)
-
-            # test saved torch model can be loaded by pytorch solely
-            lor_torch_model = torch.load(
-                os.path.join(local_model_path, "LogisticRegressionModel.torch")
-            )
-
-            with torch.inference_mode():
-                torch_infer_result = lor_torch_model(
-                    torch.tensor(np.stack(list(eval_df1.toPandas().features)), dtype=torch.float32)
-                ).numpy()
-
-            np.testing.assert_allclose(
-                np.stack(list(model_predictions.probability)),
-                torch_infer_result,
-                rtol=1e-4,
-            )
-
-            loaded_model = LORV2Model.loadFromLocal(local_model_path)
-            assert loaded_model.numFeatures == 2
-            assert loaded_model.numClasses == 2
-            assert loaded_model.getOrDefault(loaded_model.maxIter) == 2
-            assert loaded_model.torch_model is not None
-            np.testing.assert_allclose(
-                loaded_model.torch_model.weight.detach().numpy(),
-                model.torch_model.weight.detach().numpy(),
-            )
-            np.testing.assert_allclose(
-                loaded_model.torch_model.bias.detach().numpy(),
-                model.torch_model.bias.detach().numpy(),
-            )
-
-            # Test loaded model transformation.
-            loaded_model.transform(eval_df1.toPandas())
-
-            fs_model_path = os.path.join(tmp_dir, "fs", "model")
-            model.save(fs_model_path)
-            loaded_model = LORV2Model.load(fs_model_path)
-            assert loaded_model.numFeatures == 2
-            assert loaded_model.numClasses == 2
-            assert loaded_model.getOrDefault(loaded_model.maxIter) == 2
-            assert loaded_model.torch_model is not None
-            # Test loaded model transformation works.
-            loaded_model.transform(eval_df1.toPandas())
-
-
-class ClassificationTests(ClassificationTestsMixin, unittest.TestCase):
-    def setUp(self) -> None:
-        self.spark = SparkSession.builder.master("local[2]").getOrCreate()
-
-    def tearDown(self) -> None:
-        self.spark.stop()
-
-
-if __name__ == "__main__":
-    from pyspark.ml.tests.connect.test_legacy_mode_classification import *  # noqa: F401,F403
-
-    try:
-        import xmlrunner  # type: ignore[import]
-
-        testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
-    except ImportError:
-        testRunner = None
-    unittest.main(testRunner=testRunner, verbosity=2)

snowflake/snowpark_connect/includes/python/pyspark/ml/tests/connect/test_legacy_mode_evaluation.py

@@ -1,194 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-import unittest
-import numpy as np
-import tempfile
-
-from pyspark.ml.connect.evaluation import (
-    RegressionEvaluator,
-    BinaryClassificationEvaluator,
-    MulticlassClassificationEvaluator,
-)
-from pyspark.sql import SparkSession
-
-
-have_torcheval = True
-try:
-    import torcheval  # noqa: F401
-except ImportError:
-    have_torcheval = False
-
-
-class EvaluationTestsMixin:
-    def test_regressor_evaluator(self):
-        df1 = self.spark.createDataFrame(
-            [
-                (0.5, 1.0),
-                (-0.5, -0.8),
-                (2.0, 3.0),
-            ],
-            schema=["label", "prediction"],
-        )
-
-        local_df1 = df1.toPandas()
-
-        mse_evaluator = RegressionEvaluator(
-            metricName="mse",
-            labelCol="label",
-            predictionCol="prediction",
-        )
-
-        expected_mse = 0.4466666877269745
-        mse = mse_evaluator.evaluate(df1)
-        mse_local = mse_evaluator.evaluate(local_df1)
-        np.testing.assert_almost_equal(mse, expected_mse)
-        np.testing.assert_almost_equal(mse_local, expected_mse)
-
-        rmse_evaluator = RegressionEvaluator(
-            metricName="rmse",
-            labelCol="label",
-            predictionCol="prediction",
-        )
-
-        expected_rmse = 0.6683312709480042
-        rmse = rmse_evaluator.evaluate(df1)
-        rmse_local = rmse_evaluator.evaluate(local_df1)
-        np.testing.assert_almost_equal(rmse, expected_rmse)
-        np.testing.assert_almost_equal(rmse_local, expected_rmse)
-
-        r2_evaluator = RegressionEvaluator(
-            metricName="r2",
-            labelCol="label",
-            predictionCol="prediction",
-        )
-
-        expected_r2 = 0.5768420696258545
-        r2 = r2_evaluator.evaluate(df1)
-        r2_local = r2_evaluator.evaluate(local_df1)
-        np.testing.assert_almost_equal(r2, expected_r2)
-        np.testing.assert_almost_equal(r2_local, expected_r2)
-
-        # Test save / load
-        with tempfile.TemporaryDirectory() as tmp_dir:
-            r2_evaluator.saveToLocal(f"{tmp_dir}/ev")
-            loaded_evaluator = RegressionEvaluator.loadFromLocal(f"{tmp_dir}/ev")
-            assert loaded_evaluator.getMetricName() == "r2"
-
-    def test_binary_classifier_evaluator(self):
-        df1 = self.spark.createDataFrame(
-            [
-                (1, 0.2, [0.8, 0.2]),
-                (0, 0.6, [0.4, 0.6]),
-                (1, 0.8, [0.2, 0.8]),
-                (1, 0.7, [0.3, 0.7]),
-                (0, 0.4, [0.6, 0.4]),
-                (0, 0.3, [0.7, 0.3]),
-            ],
-            schema=["label", "prob", "prob2"],
-        )
-
-        local_df1 = df1.toPandas()
-
-        for prob_col in ["prob", "prob2"]:
-            auroc_evaluator = BinaryClassificationEvaluator(
-                metricName="areaUnderROC",
-                labelCol="label",
-                probabilityCol=prob_col,
-            )
-
-            expected_auroc = 0.6667
-            auroc = auroc_evaluator.evaluate(df1)
-            auroc_local = auroc_evaluator.evaluate(local_df1)
-            np.testing.assert_almost_equal(auroc, expected_auroc, decimal=2)
-            np.testing.assert_almost_equal(auroc_local, expected_auroc, decimal=2)
-
-            auprc_evaluator = BinaryClassificationEvaluator(
-                metricName="areaUnderPR",
-                labelCol="label",
-                probabilityCol=prob_col,
-            )
-
-            expected_auprc = 0.8333
-            auprc = auprc_evaluator.evaluate(df1)
-            auprc_local = auprc_evaluator.evaluate(local_df1)
-            np.testing.assert_almost_equal(auprc, expected_auprc, decimal=2)
-            np.testing.assert_almost_equal(auprc_local, expected_auprc, decimal=2)
-
-        # Test save / load
-        with tempfile.TemporaryDirectory() as tmp_dir:
-            auprc_evaluator.saveToLocal(f"{tmp_dir}/ev")
-            loaded_evaluator = RegressionEvaluator.loadFromLocal(f"{tmp_dir}/ev")
-            assert loaded_evaluator.getMetricName() == "areaUnderPR"
-
-    def test_multiclass_classifier_evaluator(self):
-        df1 = self.spark.createDataFrame(
-            [
-                (1, 1),
-                (1, 1),
-                (2, 3),
-                (0, 0),
-                (0, 1),
-                (3, 1),
-                (3, 3),
-                (2, 2),
-                (1, 0),
-                (2, 2),
-            ],
-            schema=["label", "prediction"],
-        )
-
-        local_df1 = df1.toPandas()
-
-        accuracy_evaluator = MulticlassClassificationEvaluator(
-            metricName="accuracy",
-            labelCol="label",
-            predictionCol="prediction",
-        )
-
-        expected_accuracy = 0.600
-        accuracy = accuracy_evaluator.evaluate(df1)
-        accuracy_local = accuracy_evaluator.evaluate(local_df1)
-        np.testing.assert_almost_equal(accuracy, expected_accuracy, decimal=2)
-        np.testing.assert_almost_equal(accuracy_local, expected_accuracy, decimal=2)
-
-        # Test save / load
-        with tempfile.TemporaryDirectory() as tmp_dir:
-            accuracy_evaluator.saveToLocal(f"{tmp_dir}/ev")
-            loaded_evaluator = RegressionEvaluator.loadFromLocal(f"{tmp_dir}/ev")
-            assert loaded_evaluator.getMetricName() == "accuracy"
-
-
-@unittest.skipIf(not have_torcheval, "torcheval is required")
-class EvaluationTests(EvaluationTestsMixin, unittest.TestCase):
-    def setUp(self) -> None:
-        self.spark = SparkSession.builder.master("local[2]").getOrCreate()
-
-    def tearDown(self) -> None:
-        self.spark.stop()
-
-
-if __name__ == "__main__":
-    from pyspark.ml.tests.connect.test_legacy_mode_evaluation import *  # noqa: F401,F403
-
-    try:
-        import xmlrunner  # type: ignore[import]
-
-        testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
-    except ImportError:
-        testRunner = None
-    unittest.main(testRunner=testRunner, verbosity=2)