snowpark-connect 0.23.0__py3-none-any.whl → 0.25.0__py3-none-any.whl

snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_wrapper.py

@@ -1,138 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import unittest
-
-import py4j
-
-from pyspark.ml.linalg import DenseVector, Vectors
-from pyspark.ml.regression import LinearRegression
-from pyspark.ml.wrapper import (
-    _java2py,
-    _py2java,
-    JavaParams,
-    JavaWrapper,
-)
-from pyspark.testing.mllibutils import MLlibTestCase
-from pyspark.testing.mlutils import SparkSessionTestCase
-from pyspark.testing.utils import eventually
-
-
-class JavaWrapperMemoryTests(SparkSessionTestCase):
-    def test_java_object_gets_detached(self):
-        df = self.spark.createDataFrame(
-            [(1.0, 2.0, Vectors.dense(1.0)), (0.0, 2.0, Vectors.sparse(1, [], []))],
-            ["label", "weight", "features"],
-        )
-        lr = LinearRegression(
-            maxIter=1, regParam=0.0, solver="normal", weightCol="weight", fitIntercept=False
-        )
-
-        model = lr.fit(df)
-        summary = model.summary
-
-        self.assertIsInstance(model, JavaWrapper)
-        self.assertIsInstance(summary, JavaWrapper)
-        self.assertIsInstance(model, JavaParams)
-        self.assertNotIsInstance(summary, JavaParams)
-
-        error_no_object = "Target Object ID does not exist for this gateway"
-
-        self.assertIn("LinearRegression_", model._java_obj.toString())
-        self.assertIn("LinearRegressionTrainingSummary", summary._java_obj.toString())
-
-        model.__del__()
-
-        def condition():
-            with self.assertRaisesRegex(py4j.protocol.Py4JError, error_no_object):
-                model._java_obj.toString()
-            self.assertIn("LinearRegressionTrainingSummary", summary._java_obj.toString())
-            return True
-
-        eventually(condition, timeout=10, catch_assertions=True)
-
-        try:
-            summary.__del__()
-        except BaseException:
-            pass
-
-        def condition():
-            with self.assertRaisesRegex(py4j.protocol.Py4JError, error_no_object):
-                model._java_obj.toString()
-            with self.assertRaisesRegex(py4j.protocol.Py4JError, error_no_object):
-                summary._java_obj.toString()
-            return True
-
-        eventually(condition, timeout=10, catch_assertions=True)
-
-
-class WrapperTests(MLlibTestCase):
-    def test_new_java_array(self):
-        # test array of strings
-        str_list = ["a", "b", "c"]
-        java_class = self.sc._gateway.jvm.java.lang.String
-        java_array = JavaWrapper._new_java_array(str_list, java_class)
-        self.assertEqual(_java2py(self.sc, java_array), str_list)
-        # test array of integers
-        int_list = [1, 2, 3]
-        java_class = self.sc._gateway.jvm.java.lang.Integer
-        java_array = JavaWrapper._new_java_array(int_list, java_class)
-        self.assertEqual(_java2py(self.sc, java_array), int_list)
-        # test array of floats
-        float_list = [0.1, 0.2, 0.3]
-        java_class = self.sc._gateway.jvm.java.lang.Double
-        java_array = JavaWrapper._new_java_array(float_list, java_class)
-        self.assertEqual(_java2py(self.sc, java_array), float_list)
-        # test array of bools
-        bool_list = [False, True, True]
-        java_class = self.sc._gateway.jvm.java.lang.Boolean
-        java_array = JavaWrapper._new_java_array(bool_list, java_class)
-        self.assertEqual(_java2py(self.sc, java_array), bool_list)
-        # test array of Java DenseVectors
-        v1 = DenseVector([0.0, 1.0])
-        v2 = DenseVector([1.0, 0.0])
-        vec_java_list = [_py2java(self.sc, v1), _py2java(self.sc, v2)]
-        java_class = self.sc._gateway.jvm.org.apache.spark.ml.linalg.DenseVector
-        java_array = JavaWrapper._new_java_array(vec_java_list, java_class)
-        self.assertEqual(_java2py(self.sc, java_array), [v1, v2])
-        # test empty array
-        java_class = self.sc._gateway.jvm.java.lang.Integer
-        java_array = JavaWrapper._new_java_array([], java_class)
-        self.assertEqual(_java2py(self.sc, java_array), [])
-        # test array of array of strings
-        str_list = [["a", "b", "c"], ["d", "e"], ["f", "g", "h", "i"], []]
-        expected_str_list = [
-            ("a", "b", "c", None),
-            ("d", "e", None, None),
-            ("f", "g", "h", "i"),
-            (None, None, None, None),
-        ]
-        java_class = self.sc._gateway.jvm.java.lang.String
-        java_array = JavaWrapper._new_java_array(str_list, java_class)
-        self.assertEqual(_java2py(self.sc, java_array), expected_str_list)
-
-
-if __name__ == "__main__":
-    from pyspark.ml.tests.test_wrapper import *  # noqa: F401
-
-    try:
-        import xmlrunner
-
-        testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
-    except ImportError:
-        testRunner = None
-    unittest.main(testRunner=testRunner, verbosity=2)

snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/__init__.py

@@ -1,16 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#

snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/test_cv_io_basic.py

@@ -1,151 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import tempfile
-import unittest
-
-from pyspark.ml.classification import LogisticRegression, LogisticRegressionModel
-from pyspark.ml.evaluation import BinaryClassificationEvaluator
-from pyspark.ml.linalg import Vectors
-from pyspark.ml.tuning import (
-    CrossValidator,
-    CrossValidatorModel,
-    ParamGridBuilder,
-)
-from pyspark.testing.mlutils import (
-    DummyEvaluator,
-    DummyLogisticRegression,
-    DummyLogisticRegressionModel,
-    SparkSessionTestCase,
-)
-from pyspark.ml.tests.tuning.test_tuning import ValidatorTestUtilsMixin
-
-
-class CrossValidatorIOBasicTests(SparkSessionTestCase, ValidatorTestUtilsMixin):
-    def _run_test_save_load_trained_model(self, LogisticRegressionCls, LogisticRegressionModelCls):
-        # This tests saving and loading the trained model only.
-        # Save/load for CrossValidator will be added later: SPARK-13786
-        temp_path = tempfile.mkdtemp()
-        dataset = self.spark.createDataFrame(
-            [
-                (Vectors.dense([0.0]), 0.0),
-                (Vectors.dense([0.4]), 1.0),
-                (Vectors.dense([0.5]), 0.0),
-                (Vectors.dense([0.6]), 1.0),
-                (Vectors.dense([1.0]), 1.0),
-            ]
-            * 10,
-            ["features", "label"],
-        )
-        lr = LogisticRegressionCls()
-        grid = ParamGridBuilder().addGrid(lr.maxIter, [0, 1]).build()
-        evaluator = BinaryClassificationEvaluator()
-        cv = CrossValidator(
-            estimator=lr,
-            estimatorParamMaps=grid,
-            evaluator=evaluator,
-            collectSubModels=True,
-            numFolds=4,
-            seed=42,
-        )
-        cvModel = cv.fit(dataset)
-        lrModel = cvModel.bestModel
-
-        lrModelPath = temp_path + "/lrModel"
-        lrModel.save(lrModelPath)
-        loadedLrModel = LogisticRegressionModelCls.load(lrModelPath)
-        self.assertEqual(loadedLrModel.uid, lrModel.uid)
-        self.assertEqual(loadedLrModel.intercept, lrModel.intercept)
-
-        # SPARK-32092: Saving and then loading CrossValidatorModel should not change the params
-        cvModelPath = temp_path + "/cvModel"
-        cvModel.save(cvModelPath)
-        loadedCvModel = CrossValidatorModel.load(cvModelPath)
-        for param in [
-            lambda x: x.getNumFolds(),
-            lambda x: x.getFoldCol(),
-            lambda x: x.getSeed(),
-            lambda x: len(x.subModels),
-        ]:
-            self.assertEqual(param(cvModel), param(loadedCvModel))
-
-        self.assertTrue(all(loadedCvModel.isSet(param) for param in loadedCvModel.params))
-
-        # mimic old version CrossValidatorModel (without stdMetrics attribute)
-        # test loading model backwards compatibility
-        cvModel2 = cvModel.copy()
-        cvModel2.stdMetrics = []
-        cvModelPath2 = temp_path + "/cvModel2"
-        cvModel2.save(cvModelPath2)
-        loadedCvModel2 = CrossValidatorModel.load(cvModelPath2)
-        assert loadedCvModel2.stdMetrics == []
-
-    def test_save_load_trained_model(self):
-        self._run_test_save_load_trained_model(LogisticRegression, LogisticRegressionModel)
-        self._run_test_save_load_trained_model(
-            DummyLogisticRegression, DummyLogisticRegressionModel
-        )
-
-    def _run_test_save_load_simple_estimator(self, LogisticRegressionCls, evaluatorCls):
-        temp_path = tempfile.mkdtemp()
-        dataset = self.spark.createDataFrame(
-            [
-                (Vectors.dense([0.0]), 0.0),
-                (Vectors.dense([0.4]), 1.0),
-                (Vectors.dense([0.5]), 0.0),
-                (Vectors.dense([0.6]), 1.0),
-                (Vectors.dense([1.0]), 1.0),
-            ]
-            * 10,
-            ["features", "label"],
-        )
-
-        lr = LogisticRegressionCls()
-        grid = ParamGridBuilder().addGrid(lr.maxIter, [0, 1]).build()
-        evaluator = evaluatorCls()
-
-        # test save/load of CrossValidator
-        cv = CrossValidator(estimator=lr, estimatorParamMaps=grid, evaluator=evaluator)
-        cvModel = cv.fit(dataset)
-        cvPath = temp_path + "/cv"
-        cv.save(cvPath)
-        loadedCV = CrossValidator.load(cvPath)
-        self.assertEqual(loadedCV.getEstimator().uid, cv.getEstimator().uid)
-        self.assertEqual(loadedCV.getEvaluator().uid, cv.getEvaluator().uid)
-        self.assert_param_maps_equal(loadedCV.getEstimatorParamMaps(), cv.getEstimatorParamMaps())
-
-        # test save/load of CrossValidatorModel
-        cvModelPath = temp_path + "/cvModel"
-        cvModel.save(cvModelPath)
-        loadedModel = CrossValidatorModel.load(cvModelPath)
-        self.assertEqual(loadedModel.bestModel.uid, cvModel.bestModel.uid)
-
-    def test_save_load_simple_estimator(self):
-        self._run_test_save_load_simple_estimator(LogisticRegression, BinaryClassificationEvaluator)
-        self._run_test_save_load_simple_estimator(DummyLogisticRegression, DummyEvaluator)
-
-
-if __name__ == "__main__":
-    from pyspark.ml.tests.tuning.test_cv_io_basic import *  # noqa: F401
-
-    try:
-        import xmlrunner
-
-        testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
-    except ImportError:
-        testRunner = None
-    unittest.main(testRunner=testRunner, verbosity=2)

snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/test_cv_io_nested.py

@@ -1,97 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import tempfile
-import unittest
-
-from pyspark.ml.classification import LogisticRegression, OneVsRest
-from pyspark.ml.evaluation import MulticlassClassificationEvaluator
-from pyspark.ml.linalg import Vectors
-from pyspark.ml.tuning import (
-    CrossValidator,
-    CrossValidatorModel,
-    ParamGridBuilder,
-)
-from pyspark.testing.mlutils import (
-    DummyLogisticRegression,
-    SparkSessionTestCase,
-)
-from pyspark.ml.tests.tuning.test_tuning import ValidatorTestUtilsMixin
-
-
-class CrossValidatorIONestedTests(SparkSessionTestCase, ValidatorTestUtilsMixin):
-    def _run_test_save_load_nested_estimator(self, LogisticRegressionCls):
-        temp_path = tempfile.mkdtemp()
-        dataset = self.spark.createDataFrame(
-            [
-                (Vectors.dense([0.0]), 0.0),
-                (Vectors.dense([0.4]), 1.0),
-                (Vectors.dense([0.5]), 0.0),
-                (Vectors.dense([0.6]), 1.0),
-                (Vectors.dense([1.0]), 1.0),
-            ]
-            * 10,
-            ["features", "label"],
-        )
-
-        ova = OneVsRest(classifier=LogisticRegressionCls())
-        lr1 = LogisticRegressionCls().setMaxIter(100)
-        lr2 = LogisticRegressionCls().setMaxIter(150)
-        grid = ParamGridBuilder().addGrid(ova.classifier, [lr1, lr2]).build()
-        evaluator = MulticlassClassificationEvaluator()
-
-        # test save/load of CrossValidator
-        cv = CrossValidator(estimator=ova, estimatorParamMaps=grid, evaluator=evaluator)
-        cvModel = cv.fit(dataset)
-        cvPath = temp_path + "/cv"
-        cv.save(cvPath)
-        loadedCV = CrossValidator.load(cvPath)
-        self.assert_param_maps_equal(loadedCV.getEstimatorParamMaps(), grid)
-        self.assertEqual(loadedCV.getEstimator().uid, cv.getEstimator().uid)
-        self.assertEqual(loadedCV.getEvaluator().uid, cv.getEvaluator().uid)
-
-        originalParamMap = cv.getEstimatorParamMaps()
-        loadedParamMap = loadedCV.getEstimatorParamMaps()
-        for i, param in enumerate(loadedParamMap):
-            for p in param:
-                if p.name == "classifier":
-                    self.assertEqual(param[p].uid, originalParamMap[i][p].uid)
-                else:
-                    self.assertEqual(param[p], originalParamMap[i][p])
-
-        # test save/load of CrossValidatorModel
-        cvModelPath = temp_path + "/cvModel"
-        cvModel.save(cvModelPath)
-        loadedModel = CrossValidatorModel.load(cvModelPath)
-        self.assert_param_maps_equal(loadedModel.getEstimatorParamMaps(), grid)
-        self.assertEqual(loadedModel.bestModel.uid, cvModel.bestModel.uid)
-
-    def test_save_load_nested_estimator(self):
-        self._run_test_save_load_nested_estimator(LogisticRegression)
-        self._run_test_save_load_nested_estimator(DummyLogisticRegression)
-
-
-if __name__ == "__main__":
-    from pyspark.ml.tests.tuning.test_cv_io_nested import *  # noqa: F401
-
-    try:
-        import xmlrunner
-
-        testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
-    except ImportError:
-        testRunner = None
-    unittest.main(testRunner=testRunner, verbosity=2)

snowflake/snowpark_connect/includes/python/pyspark/ml/tests/tuning/test_cv_io_pipeline.py

@@ -1,143 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import tempfile
-import unittest
-
-from pyspark.ml.feature import HashingTF, Tokenizer
-from pyspark.ml import Pipeline
-from pyspark.ml.classification import LogisticRegression, OneVsRest
-from pyspark.ml.evaluation import MulticlassClassificationEvaluator
-from pyspark.ml.tuning import (
-    CrossValidator,
-    CrossValidatorModel,
-    ParamGridBuilder,
-)
-from pyspark.testing.mlutils import (
-    DummyLogisticRegression,
-    SparkSessionTestCase,
-)
-from pyspark.ml.tests.tuning.test_tuning import ValidatorTestUtilsMixin
-
-
-class CrossValidatorIOPipelineTests(SparkSessionTestCase, ValidatorTestUtilsMixin):
-    def _run_test_save_load_pipeline_estimator(self, LogisticRegressionCls):
-        temp_path = tempfile.mkdtemp()
-        training = self.spark.createDataFrame(
-            [
-                (0, "a b c d e spark", 1.0),
-                (1, "b d", 0.0),
-                (2, "spark f g h", 1.0),
-                (3, "hadoop mapreduce", 0.0),
-                (4, "b spark who", 1.0),
-                (5, "g d a y", 0.0),
-                (6, "spark fly", 1.0),
-                (7, "was mapreduce", 0.0),
-            ],
-            ["id", "text", "label"],
-        )
-
-        # Configure an ML pipeline, which consists of tree stages: tokenizer, hashingTF, and lr.
-        tokenizer = Tokenizer(inputCol="text", outputCol="words")
-        hashingTF = HashingTF(inputCol=tokenizer.getOutputCol(), outputCol="features")
-
-        ova = OneVsRest(classifier=LogisticRegressionCls())
-        lr1 = LogisticRegressionCls().setMaxIter(5)
-        lr2 = LogisticRegressionCls().setMaxIter(10)
-
-        pipeline = Pipeline(stages=[tokenizer, hashingTF, ova])
-
-        paramGrid = (
-            ParamGridBuilder()
-            .addGrid(hashingTF.numFeatures, [10, 100])
-            .addGrid(ova.classifier, [lr1, lr2])
-            .build()
-        )
-
-        crossval = CrossValidator(
-            estimator=pipeline,
-            estimatorParamMaps=paramGrid,
-            evaluator=MulticlassClassificationEvaluator(),
-            numFolds=2,
-        )  # use 3+ folds in practice
-        cvPath = temp_path + "/cv"
-        crossval.save(cvPath)
-        loadedCV = CrossValidator.load(cvPath)
-        self.assert_param_maps_equal(loadedCV.getEstimatorParamMaps(), paramGrid)
-        self.assertEqual(loadedCV.getEstimator().uid, crossval.getEstimator().uid)
-
-        # Run cross-validation, and choose the best set of parameters.
-        cvModel = crossval.fit(training)
-
-        # test save/load of CrossValidatorModel
-        cvModelPath = temp_path + "/cvModel"
-        cvModel.save(cvModelPath)
-        loadedModel = CrossValidatorModel.load(cvModelPath)
-        self.assertEqual(loadedModel.bestModel.uid, cvModel.bestModel.uid)
-        self.assertEqual(len(loadedModel.bestModel.stages), len(cvModel.bestModel.stages))
-        for loadedStage, originalStage in zip(
-            loadedModel.bestModel.stages, cvModel.bestModel.stages
-        ):
-            self.assertEqual(loadedStage.uid, originalStage.uid)
-
-        # Test nested pipeline
-        nested_pipeline = Pipeline(stages=[tokenizer, Pipeline(stages=[hashingTF, ova])])
-        crossval2 = CrossValidator(
-            estimator=nested_pipeline,
-            estimatorParamMaps=paramGrid,
-            evaluator=MulticlassClassificationEvaluator(),
-            numFolds=2,
-        )  # use 3+ folds in practice
-        cv2Path = temp_path + "/cv2"
-        crossval2.save(cv2Path)
-        loadedCV2 = CrossValidator.load(cv2Path)
-        self.assert_param_maps_equal(loadedCV2.getEstimatorParamMaps(), paramGrid)
-        self.assertEqual(loadedCV2.getEstimator().uid, crossval2.getEstimator().uid)
-
-        # Run cross-validation, and choose the best set of parameters.
-        cvModel2 = crossval2.fit(training)
-        # test save/load of CrossValidatorModel
-        cvModelPath2 = temp_path + "/cvModel2"
-        cvModel2.save(cvModelPath2)
-        loadedModel2 = CrossValidatorModel.load(cvModelPath2)
-        self.assertEqual(loadedModel2.bestModel.uid, cvModel2.bestModel.uid)
-        loaded_nested_pipeline_model = loadedModel2.bestModel.stages[1]
-        original_nested_pipeline_model = cvModel2.bestModel.stages[1]
-        self.assertEqual(loaded_nested_pipeline_model.uid, original_nested_pipeline_model.uid)
-        self.assertEqual(
-            len(loaded_nested_pipeline_model.stages), len(original_nested_pipeline_model.stages)
-        )
-        for loadedStage, originalStage in zip(
-            loaded_nested_pipeline_model.stages, original_nested_pipeline_model.stages
-        ):
-            self.assertEqual(loadedStage.uid, originalStage.uid)
-
-    def test_save_load_pipeline_estimator(self):
-        self._run_test_save_load_pipeline_estimator(LogisticRegression)
-        self._run_test_save_load_pipeline_estimator(DummyLogisticRegression)
-
-
-if __name__ == "__main__":
-    from pyspark.ml.tests.tuning.test_cv_io_pipeline import *  # noqa: F401
-
-    try:
-        import xmlrunner
-
-        testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
-    except ImportError:
-        testRunner = None
-    unittest.main(testRunner=testRunner, verbosity=2)