snowpark-connect 0.23.0__py3-none-any.whl → 0.25.0__py3-none-any.whl

snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/test_tvs_io_nested.py

@@ -1,96 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import tempfile
-import unittest
-
-from pyspark.ml.classification import LogisticRegression, OneVsRest
-from pyspark.ml.evaluation import MulticlassClassificationEvaluator
-from pyspark.ml.linalg import Vectors
-from pyspark.ml.tuning import (
-    ParamGridBuilder,
-    TrainValidationSplit,
-    TrainValidationSplitModel,
-)
-from pyspark.testing.mlutils import (
-    DummyLogisticRegression,
-    SparkSessionTestCase,
-)
-from pyspark.ml.tests.tuning.test_tuning import ValidatorTestUtilsMixin
-
-
-class TrainValidationSplitIONestedTests(SparkSessionTestCase, ValidatorTestUtilsMixin):
-    def _run_test_save_load_nested_estimator(self, LogisticRegressionCls):
-        # This tests saving and loading the trained model only.
-        # Save/load for TrainValidationSplit will be added later: SPARK-13786
-        temp_path = tempfile.mkdtemp()
-        dataset = self.spark.createDataFrame(
-            [
-                (Vectors.dense([0.0]), 0.0),
-                (Vectors.dense([0.4]), 1.0),
-                (Vectors.dense([0.5]), 0.0),
-                (Vectors.dense([0.6]), 1.0),
-                (Vectors.dense([1.0]), 1.0),
-            ]
-            * 10,
-            ["features", "label"],
-        )
-        ova = OneVsRest(classifier=LogisticRegressionCls())
-        lr1 = LogisticRegressionCls().setMaxIter(100)
-        lr2 = LogisticRegressionCls().setMaxIter(150)
-        grid = ParamGridBuilder().addGrid(ova.classifier, [lr1, lr2]).build()
-        evaluator = MulticlassClassificationEvaluator()
-
-        tvs = TrainValidationSplit(estimator=ova, estimatorParamMaps=grid, evaluator=evaluator)
-        tvsModel = tvs.fit(dataset)
-        tvsPath = temp_path + "/tvs"
-        tvs.save(tvsPath)
-        loadedTvs = TrainValidationSplit.load(tvsPath)
-        self.assert_param_maps_equal(loadedTvs.getEstimatorParamMaps(), grid)
-        self.assertEqual(loadedTvs.getEstimator().uid, tvs.getEstimator().uid)
-        self.assertEqual(loadedTvs.getEvaluator().uid, tvs.getEvaluator().uid)
-
-        originalParamMap = tvs.getEstimatorParamMaps()
-        loadedParamMap = loadedTvs.getEstimatorParamMaps()
-        for i, param in enumerate(loadedParamMap):
-            for p in param:
-                if p.name == "classifier":
-                    self.assertEqual(param[p].uid, originalParamMap[i][p].uid)
-                else:
-                    self.assertEqual(param[p], originalParamMap[i][p])
-
-        tvsModelPath = temp_path + "/tvsModel"
-        tvsModel.save(tvsModelPath)
-        loadedModel = TrainValidationSplitModel.load(tvsModelPath)
-        self.assert_param_maps_equal(loadedModel.getEstimatorParamMaps(), grid)
-        self.assertEqual(loadedModel.bestModel.uid, tvsModel.bestModel.uid)
-
-    def test_save_load_nested_estimator(self):
-        self._run_test_save_load_nested_estimator(LogisticRegression)
-        self._run_test_save_load_nested_estimator(DummyLogisticRegression)
-
-
-if __name__ == "__main__":
-    from pyspark.ml.tests.tuning.test_tvs_io_nested import *  # noqa: F401
-
-    try:
-        import xmlrunner
-
-        testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
-    except ImportError:
-        testRunner = None
-    unittest.main(testRunner=testRunner, verbosity=2)

snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/test_tvs_io_pipeline.py

@@ -1,142 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import tempfile
-import unittest
-
-from pyspark.ml.feature import HashingTF, Tokenizer
-from pyspark.ml import Pipeline
-from pyspark.ml.classification import LogisticRegression, OneVsRest
-from pyspark.ml.evaluation import MulticlassClassificationEvaluator
-from pyspark.ml.tuning import (
-    ParamGridBuilder,
-    TrainValidationSplit,
-    TrainValidationSplitModel,
-)
-from pyspark.testing.mlutils import (
-    DummyLogisticRegression,
-    SparkSessionTestCase,
-)
-
-from pyspark.ml.tests.tuning.test_tuning import ValidatorTestUtilsMixin
-
-
-class TrainValidationSplitIONestedTests(SparkSessionTestCase, ValidatorTestUtilsMixin):
-    def _run_test_save_load_pipeline_estimator(self, LogisticRegressionCls):
-        temp_path = tempfile.mkdtemp()
-        training = self.spark.createDataFrame(
-            [
-                (0, "a b c d e spark", 1.0),
-                (1, "b d", 0.0),
-                (2, "spark f g h", 1.0),
-                (3, "hadoop mapreduce", 0.0),
-                (4, "b spark who", 1.0),
-                (5, "g d a y", 0.0),
-                (6, "spark fly", 1.0),
-                (7, "was mapreduce", 0.0),
-            ],
-            ["id", "text", "label"],
-        )
-
-        # Configure an ML pipeline, which consists of tree stages: tokenizer, hashingTF, and lr.
-        tokenizer = Tokenizer(inputCol="text", outputCol="words")
-        hashingTF = HashingTF(inputCol=tokenizer.getOutputCol(), outputCol="features")
-
-        ova = OneVsRest(classifier=LogisticRegressionCls())
-        lr1 = LogisticRegressionCls().setMaxIter(5)
-        lr2 = LogisticRegressionCls().setMaxIter(10)
-
-        pipeline = Pipeline(stages=[tokenizer, hashingTF, ova])
-
-        paramGrid = (
-            ParamGridBuilder()
-            .addGrid(hashingTF.numFeatures, [10, 100])
-            .addGrid(ova.classifier, [lr1, lr2])
-            .build()
-        )
-
-        tvs = TrainValidationSplit(
-            estimator=pipeline,
-            estimatorParamMaps=paramGrid,
-            evaluator=MulticlassClassificationEvaluator(),
-        )
-        tvsPath = temp_path + "/tvs"
-        tvs.save(tvsPath)
-        loadedTvs = TrainValidationSplit.load(tvsPath)
-        self.assert_param_maps_equal(loadedTvs.getEstimatorParamMaps(), paramGrid)
-        self.assertEqual(loadedTvs.getEstimator().uid, tvs.getEstimator().uid)
-
-        # Run train validation split, and choose the best set of parameters.
-        tvsModel = tvs.fit(training)
-
-        # test save/load of CrossValidatorModel
-        tvsModelPath = temp_path + "/tvsModel"
-        tvsModel.save(tvsModelPath)
-        loadedModel = TrainValidationSplitModel.load(tvsModelPath)
-        self.assertEqual(loadedModel.bestModel.uid, tvsModel.bestModel.uid)
-        self.assertEqual(len(loadedModel.bestModel.stages), len(tvsModel.bestModel.stages))
-        for loadedStage, originalStage in zip(
-            loadedModel.bestModel.stages, tvsModel.bestModel.stages
-        ):
-            self.assertEqual(loadedStage.uid, originalStage.uid)
-
-        # Test nested pipeline
-        nested_pipeline = Pipeline(stages=[tokenizer, Pipeline(stages=[hashingTF, ova])])
-        tvs2 = TrainValidationSplit(
-            estimator=nested_pipeline,
-            estimatorParamMaps=paramGrid,
-            evaluator=MulticlassClassificationEvaluator(),
-        )
-        tvs2Path = temp_path + "/tvs2"
-        tvs2.save(tvs2Path)
-        loadedTvs2 = TrainValidationSplit.load(tvs2Path)
-        self.assert_param_maps_equal(loadedTvs2.getEstimatorParamMaps(), paramGrid)
-        self.assertEqual(loadedTvs2.getEstimator().uid, tvs2.getEstimator().uid)
-
-        # Run train validation split, and choose the best set of parameters.
-        tvsModel2 = tvs2.fit(training)
-        # test save/load of CrossValidatorModel
-        tvsModelPath2 = temp_path + "/tvsModel2"
-        tvsModel2.save(tvsModelPath2)
-        loadedModel2 = TrainValidationSplitModel.load(tvsModelPath2)
-        self.assertEqual(loadedModel2.bestModel.uid, tvsModel2.bestModel.uid)
-        loaded_nested_pipeline_model = loadedModel2.bestModel.stages[1]
-        original_nested_pipeline_model = tvsModel2.bestModel.stages[1]
-        self.assertEqual(loaded_nested_pipeline_model.uid, original_nested_pipeline_model.uid)
-        self.assertEqual(
-            len(loaded_nested_pipeline_model.stages), len(original_nested_pipeline_model.stages)
-        )
-        for loadedStage, originalStage in zip(
-            loaded_nested_pipeline_model.stages, original_nested_pipeline_model.stages
-        ):
-            self.assertEqual(loadedStage.uid, originalStage.uid)
-
-    def test_save_load_pipeline_estimator(self):
-        self._run_test_save_load_pipeline_estimator(LogisticRegression)
-        self._run_test_save_load_pipeline_estimator(DummyLogisticRegression)
-
-
-if __name__ == "__main__":
-    from pyspark.ml.tests.tuning.test_tvs_io_pipeline import *  # noqa: F401
-
-    try:
-        import xmlrunner
-
-        testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
-    except ImportError:
-        testRunner = None
-    unittest.main(testRunner=testRunner, verbosity=2)

snowflake/snowpark_connect/includes/python/pyspark/ml/torch/tests/__init__.py

@@ -1,16 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#

snowflake/snowpark_connect/includes/python/pyspark/ml/torch/tests/test_data_loader.py

@@ -1,137 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import os
-import numpy as np
-import unittest
-
-have_torch = "SPARK_SKIP_CONNECT_COMPAT_TESTS" not in os.environ
-try:
-    import torch  # noqa: F401
-except ImportError:
-    have_torch = False
-
-from pyspark.ml.torch.distributor import (
-    TorchDistributor,
-    _get_spark_partition_data_loader,
-)
-from pyspark.sql import SparkSession
-from pyspark.ml.linalg import Vectors
-
-
-@unittest.skipIf(not have_torch, "torch is required")
-class TorchDistributorDataLoaderUnitTests(unittest.TestCase):
-    def setUp(self) -> None:
-        self.spark = (
-            SparkSession.builder.master("local[1]")
-            .config("spark.default.parallelism", "1")
-            .getOrCreate()
-        )
-
-    def tearDown(self) -> None:
-        self.spark.stop()
-
-    def _check_data_loader_result_correctness(self, result, expected):
-        assert len(result) == len(expected)
-
-        for res_row, exp_row in zip(result, expected):
-            assert len(res_row) == len(exp_row)
-            for res_field, exp_field in zip(res_row, exp_row):
-                np.testing.assert_almost_equal(res_field.numpy(), exp_field)
-
-    def test_data_loader(self):
-        spark_df = self.spark.createDataFrame(
-            [
-                (Vectors.dense([1.0, 2.0, 3.5]), 0, 10.5),
-                (Vectors.sparse(3, [1, 2], [4.5, 5.5]), 3, 12.5),
-                (Vectors.dense([6.0, 7.0, 8.5]), 1, 1.5),
-                (Vectors.sparse(3, [0, 2], [-2.5, -6.5]), 2, 9.5),
-            ],
-            schema=["features", "label", "weight"],
-        )
-
-        torch_distributor = TorchDistributor(local_mode=False, use_gpu=False)
-
-        def train_function(num_samples, batch_size):
-            data_loader = _get_spark_partition_data_loader(num_samples, batch_size)
-            return list(data_loader)
-
-        result = torch_distributor._train_on_dataframe(
-            train_function,
-            spark_df,
-            num_samples=4,
-            batch_size=2,
-        )
-        self._check_data_loader_result_correctness(
-            result,
-            [
-                [[[1.0, 2.0, 3.5], [0.0, 4.5, 5.5]], [0, 3], [10.5, 12.5]],
-                [[[6.0, 7.0, 8.5], [-2.5, 0.0, -6.5]], [1, 2], [1.5, 9.5]],
-            ],
-        )
-
-        result = torch_distributor._train_on_dataframe(
-            train_function,
-            spark_df,
-            num_samples=4,
-            batch_size=3,
-        )
-        self._check_data_loader_result_correctness(
-            result,
-            [
-                [
-                    [[1.0, 2.0, 3.5], [0.0, 4.5, 5.5], [6.0, 7.0, 8.5]],
-                    [0, 3, 1],
-                    [10.5, 12.5, 1.5],
-                ],
-                [[[-2.5, 0.0, -6.5]], [2], [9.5]],
-            ],
-        )
-
-        result = torch_distributor._train_on_dataframe(
-            train_function,
-            spark_df,
-            num_samples=6,
-            batch_size=3,
-        )
-        self._check_data_loader_result_correctness(
-            result,
-            [
-                [
-                    [[1.0, 2.0, 3.5], [0.0, 4.5, 5.5], [6.0, 7.0, 8.5]],
-                    [0, 3, 1],
-                    [10.5, 12.5, 1.5],
-                ],
-                [
-                    [[-2.5, 0.0, -6.5], [1.0, 2.0, 3.5], [0.0, 4.5, 5.5]],
-                    [2, 0, 3],
-                    [9.5, 10.5, 12.5],
-                ],
-            ],
-        )
-
-
-if __name__ == "__main__":
-    from pyspark.ml.torch.tests.test_data_loader import *  # noqa: F401,F403
-
-    try:
-        import xmlrunner
-
-        testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
-    except ImportError:
-        testRunner = None
-    unittest.main(testRunner=testRunner, verbosity=2)