snowpark-connect 0.23.0__py3-none-any.whl → 0.25.0__py3-none-any.whl

snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_dl_util.py

@@ -1,186 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-from contextlib import contextmanager
-import os
-import textwrap
-from typing import Any, BinaryIO, Callable, Iterator
-
-import unittest
-from parameterized import parameterized
-from pyspark import cloudpickle
-from pyspark.ml.dl_util import FunctionPickler
-
-
-class TestFunctionPickler(unittest.TestCase):
-
-    # Function that will be used to test pickling.
-    @staticmethod
-    def _test_function(x: float, y: float) -> float:
-        return x**2 + y**2
-
-    def _check_if_test_function_pickled(
-        self,
-        file: BinaryIO,
-        desired_function: Callable,
-        output_value: Any,
-        *arguments,
-        **key_word_args,
-    ):
-        fn, args, kwargs = cloudpickle.load(file)
-        self.assertEqual(fn, desired_function)
-        self.assertEqual(args, arguments)
-        self.assertEqual(kwargs, key_word_args)
-        fn_output = fn(*args, **kwargs)
-        self.assertEqual(fn_output, output_value)
-
-    @parameterized.expand(
-        [
-            ("See if it pickles correctly with no path specified", "", ""),
-            ("See if it pickles correctly with path specified", "silly_bear", ""),
-            (
-                "See if it pickles correctly with both path and save_dir specified",
-                "silly_bear",
-                "tmp_dir",
-            ),
-        ]
-    )
-    def test_pickle_fn_and_save(self, _: str, file_path_to_save: str, save_dir: str):
-        x, y = 1, 3  # args of test_function
-        if save_dir != "":
-            os.makedirs(save_dir, exist_ok=True)
-        pickled_fn_path = FunctionPickler.pickle_fn_and_save(
-            TestFunctionPickler._test_function, file_path_to_save, save_dir, x, y
-        )
-        if file_path_to_save != "":
-            self.assertEqual(file_path_to_save, pickled_fn_path)
-
-        with open(pickled_fn_path, "rb") as f:
-            self._check_if_test_function_pickled(f, TestFunctionPickler._test_function, 10, x, y)
-        os.remove(pickled_fn_path)
-
-        if save_dir != "":
-            os.rmdir(save_dir)
-
-    def test_getting_output_from_pickle_file(self):
-        a, b = 2, 0  # arguments for _test_function
-        pickle_fn_file = FunctionPickler.pickle_fn_and_save(
-            TestFunctionPickler._test_function, "", "", a, b
-        )
-        fn, args, kwargs = FunctionPickler.get_fn_output(pickle_fn_file)
-        self.assertEqual(fn, TestFunctionPickler._test_function)
-        self.assertEqual(len(args), 2)
-        self.assertEqual(len(kwargs), 0)
-        self.assertEqual(args[0], a)
-        self.assertEqual(args[1], b)
-        self.assertEqual(fn(*args, **kwargs), 4)
-        os.remove(pickle_fn_file)
-
-    @contextmanager
-    def create_reference_file(
-        self, body: str, prefix: str = "", suffix: str = "", fname: str = "reference.py"
-    ) -> Iterator[None]:
-        try:
-            with open(fname, "w") as f:
-                if prefix != "":
-                    f.write(prefix)
-                f.write(body)
-                if suffix != "":
-                    f.write(suffix)
-            yield
-        finally:
-            os.remove(fname)
-
-    def _create_code_snippet_body(self, pickled_fn_path: str, fn_output_save_path: str) -> str:
-        code_snippet = textwrap.dedent(
-            f"""
-                    from pyspark import cloudpickle
-                    import os
-
-                    if __name__ == "__main__":
-                        with open("{pickled_fn_path}", "rb") as f:
-                            fn, args, kwargs = cloudpickle.load(f)
-                        output = fn(*args, **kwargs)
-                        with open("{fn_output_save_path}", "wb") as f:
-                            cloudpickle.dump(output, f)
-                    """
-        )
-        return code_snippet
-
-    def _are_two_files_identical(self, fpath1: str, fpath2: str) -> bool:
-        with open(fpath1, "rb") as f:
-            contents_one = f.read()
-        with open(fpath2, "rb") as f:
-            contents_two = f.read()
-        self.assertEqual(contents_one, contents_two)
-        return contents_one == contents_two
-
-    @parameterized.expand(
-        [
-            ("Check if it creates the correct file with no prefix nor suffix", "", ""),
-            (
-                "Check if it creates the correct file with only prefix + body",
-                "print('hello before')\n",
-                "",
-            ),
-            (
-                "Check if it creates the correct file with only suffix + boddy",
-                "",
-                "print('goodbye')",
-            ),
-            (
-                "Check if it creates the correct file prefix, body, and suffix",
-                "print('hello')\n",
-                "print('goodbye')\n",
-            ),
-        ]
-    )
-    def test_create_fn_run_script(self, mesg: str, prefix_test: str, suffix_test: str):
-        arg1, arg2 = 3, 4
-        pickled_fn_path = FunctionPickler.pickle_fn_and_save(
-            TestFunctionPickler._test_function, "", "", arg1, arg2
-        )
-        fn_out_path = "output.pickled"
-        reference_path = "ref_result_file.py"
-        test_path = "test_result.py"
-        body_for_reference = self._create_code_snippet_body(pickled_fn_path, fn_out_path)
-
-        with self.create_reference_file(
-            body_for_reference, prefix=prefix_test, suffix=suffix_test, fname=reference_path
-        ) as _:
-            executable_file_path = FunctionPickler.create_fn_run_script(
-                pickled_fn_path,
-                fn_out_path,
-                test_path,
-                prefix_code=prefix_test,
-                suffix_code=suffix_test,
-            )
-            self.assertTrue(self._are_two_files_identical(reference_path, executable_file_path))
-            os.remove(executable_file_path)
-
-        os.remove(pickled_fn_path)
-
-
-if __name__ == "__main__":
-    from pyspark.ml.tests.test_dl_util import *  # noqa: F401
-
-    try:
-        import xmlrunner
-
-        testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
-    except ImportError:
-        testRunner = None
-    unittest.main(testRunner=testRunner, verbosity=2)

snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_evaluation.py

@@ -1,77 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import unittest
-
-import numpy as np
-
-from pyspark.ml.evaluation import ClusteringEvaluator, RegressionEvaluator
-from pyspark.ml.linalg import Vectors
-from pyspark.sql import Row
-from pyspark.testing.mlutils import SparkSessionTestCase
-
-
-class EvaluatorTests(SparkSessionTestCase):
-    def test_evaluate_invalid_type(self):
-        evaluator = RegressionEvaluator(metricName="r2")
-        df = self.spark.createDataFrame([Row(label=1.0, prediction=1.1)])
-        invalid_type = ""
-        self.assertRaises(TypeError, evaluator.evaluate, df, invalid_type)
-
-    def test_java_params(self):
-        """
-        This tests a bug fixed by SPARK-18274 which causes multiple copies
-        of a Params instance in Python to be linked to the same Java instance.
-        """
-        evaluator = RegressionEvaluator(metricName="r2")
-        df = self.spark.createDataFrame([Row(label=1.0, prediction=1.1)])
-        evaluator.evaluate(df)
-        self.assertEqual(evaluator._java_obj.getMetricName(), "r2")
-        evaluatorCopy = evaluator.copy({evaluator.metricName: "mae"})
-        evaluator.evaluate(df)
-        evaluatorCopy.evaluate(df)
-        self.assertEqual(evaluator._java_obj.getMetricName(), "r2")
-        self.assertEqual(evaluatorCopy._java_obj.getMetricName(), "mae")
-
-    def test_clustering_evaluator_with_cosine_distance(self):
-        featureAndPredictions = map(
-            lambda x: (Vectors.dense(x[0]), x[1]),
-            [
-                ([1.0, 1.0], 1.0),
-                ([10.0, 10.0], 1.0),
-                ([1.0, 0.5], 2.0),
-                ([10.0, 4.4], 2.0),
-                ([-1.0, 1.0], 3.0),
-                ([-100.0, 90.0], 3.0),
-            ],
-        )
-        dataset = self.spark.createDataFrame(featureAndPredictions, ["features", "prediction"])
-        evaluator = ClusteringEvaluator(predictionCol="prediction", distanceMeasure="cosine")
-        self.assertEqual(evaluator.getDistanceMeasure(), "cosine")
-        self.assertTrue(np.isclose(evaluator.evaluate(dataset), 0.992671213, atol=1e-5))
-
-
-if __name__ == "__main__":
-    from pyspark.ml.tests.test_evaluation import *  # noqa: F401
-
-    try:
-        import xmlrunner
-
-        testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
-    except ImportError:
-        testRunner = None
-    unittest.main(testRunner=testRunner, verbosity=2)

snowflake/snowpark_connect/includes/python/pyspark/ml/tests/test_feature.py

@@ -1,401 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import unittest
-
-from pyspark.ml.feature import (
-    Binarizer,
-    CountVectorizer,
-    CountVectorizerModel,
-    HashingTF,
-    IDF,
-    NGram,
-    RFormula,
-    StopWordsRemover,
-    StringIndexer,
-    StringIndexerModel,
-    VectorSizeHint,
-)
-from pyspark.ml.linalg import DenseVector, SparseVector, Vectors
-from pyspark.sql import Row
-from pyspark.testing.utils import QuietTest
-from pyspark.testing.mlutils import check_params, SparkSessionTestCase
-
-
-class FeatureTests(SparkSessionTestCase):
-    def test_binarizer(self):
-        b0 = Binarizer()
-        self.assertListEqual(
-            b0.params,
-            [b0.inputCol, b0.inputCols, b0.outputCol, b0.outputCols, b0.threshold, b0.thresholds],
-        )
-        self.assertTrue(all([~b0.isSet(p) for p in b0.params]))
-        self.assertTrue(b0.hasDefault(b0.threshold))
-        self.assertEqual(b0.getThreshold(), 0.0)
-        b0.setParams(inputCol="input", outputCol="output").setThreshold(1.0)
-        self.assertTrue(not all([b0.isSet(p) for p in b0.params]))
-        self.assertEqual(b0.getThreshold(), 1.0)
-        self.assertEqual(b0.getInputCol(), "input")
-        self.assertEqual(b0.getOutputCol(), "output")
-
-        b0c = b0.copy({b0.threshold: 2.0})
-        self.assertEqual(b0c.uid, b0.uid)
-        self.assertListEqual(b0c.params, b0.params)
-        self.assertEqual(b0c.getThreshold(), 2.0)
-
-        b1 = Binarizer(threshold=2.0, inputCol="input", outputCol="output")
-        self.assertNotEqual(b1.uid, b0.uid)
-        self.assertEqual(b1.getThreshold(), 2.0)
-        self.assertEqual(b1.getInputCol(), "input")
-        self.assertEqual(b1.getOutputCol(), "output")
-
-    def test_idf(self):
-        dataset = self.spark.createDataFrame(
-            [(DenseVector([1.0, 2.0]),), (DenseVector([0.0, 1.0]),), (DenseVector([3.0, 0.2]),)],
-            ["tf"],
-        )
-        idf0 = IDF(inputCol="tf")
-        self.assertListEqual(idf0.params, [idf0.inputCol, idf0.minDocFreq, idf0.outputCol])
-        idf0m = idf0.fit(dataset, {idf0.outputCol: "idf"})
-        self.assertEqual(
-            idf0m.uid, idf0.uid, "Model should inherit the UID from its parent estimator."
-        )
-        output = idf0m.transform(dataset)
-        self.assertIsNotNone(output.head().idf)
-        self.assertIsNotNone(idf0m.docFreq)
-        self.assertEqual(idf0m.numDocs, 3)
-        # Test that parameters transferred to Python Model
-        check_params(self, idf0m)
-
-    def test_ngram(self):
-        dataset = self.spark.createDataFrame([Row(input=["a", "b", "c", "d", "e"])])
-        ngram0 = NGram(n=4, inputCol="input", outputCol="output")
-        self.assertEqual(ngram0.getN(), 4)
-        self.assertEqual(ngram0.getInputCol(), "input")
-        self.assertEqual(ngram0.getOutputCol(), "output")
-        transformedDF = ngram0.transform(dataset)
-        self.assertEqual(transformedDF.head().output, ["a b c d", "b c d e"])
-
-    def test_stopwordsremover(self):
-        dataset = self.spark.createDataFrame([Row(input=["a", "panda"])])
-        stopWordRemover = StopWordsRemover(inputCol="input", outputCol="output")
-        # Default
-        self.assertEqual(stopWordRemover.getInputCol(), "input")
-        transformedDF = stopWordRemover.transform(dataset)
-        self.assertEqual(transformedDF.head().output, ["panda"])
-        self.assertEqual(type(stopWordRemover.getStopWords()), list)
-        self.assertTrue(isinstance(stopWordRemover.getStopWords()[0], str))
-        # Custom
-        stopwords = ["panda"]
-        stopWordRemover.setStopWords(stopwords)
-        self.assertEqual(stopWordRemover.getInputCol(), "input")
-        self.assertEqual(stopWordRemover.getStopWords(), stopwords)
-        transformedDF = stopWordRemover.transform(dataset)
-        self.assertEqual(transformedDF.head().output, ["a"])
-        # with language selection
-        stopwords = StopWordsRemover.loadDefaultStopWords("turkish")
-        dataset = self.spark.createDataFrame([Row(input=["acaba", "ama", "biri"])])
-        stopWordRemover.setStopWords(stopwords)
-        self.assertEqual(stopWordRemover.getStopWords(), stopwords)
-        transformedDF = stopWordRemover.transform(dataset)
-        self.assertEqual(transformedDF.head().output, [])
-        # with locale
-        stopwords = ["BELKİ"]
-        dataset = self.spark.createDataFrame([Row(input=["belki"])])
-        stopWordRemover.setStopWords(stopwords).setLocale("tr")
-        self.assertEqual(stopWordRemover.getStopWords(), stopwords)
-        transformedDF = stopWordRemover.transform(dataset)
-        self.assertEqual(transformedDF.head().output, [])
-
-    def test_count_vectorizer_with_binary(self):
-        dataset = self.spark.createDataFrame(
-            [
-                (
-                    0,
-                    "a a a b b c".split(" "),
-                    SparseVector(3, {0: 1.0, 1: 1.0, 2: 1.0}),
-                ),
-                (
-                    1,
-                    "a a".split(" "),
-                    SparseVector(3, {0: 1.0}),
-                ),
-                (
-                    2,
-                    "a b".split(" "),
-                    SparseVector(3, {0: 1.0, 1: 1.0}),
-                ),
-                (
-                    3,
-                    "c".split(" "),
-                    SparseVector(3, {2: 1.0}),
-                ),
-            ],
-            ["id", "words", "expected"],
-        )
-        cv = CountVectorizer(binary=True, inputCol="words", outputCol="features")
-        model = cv.fit(dataset)
-
-        transformedList = model.transform(dataset).select("features", "expected").collect()
-
-        for r in transformedList:
-            feature, expected = r
-            self.assertEqual(feature, expected)
-
-    def test_count_vectorizer_with_maxDF(self):
-        dataset = self.spark.createDataFrame(
-            [
-                (
-                    0,
-                    "a b c d".split(" "),
-                    SparseVector(3, {0: 1.0, 1: 1.0, 2: 1.0}),
-                ),
-                (
-                    1,
-                    "a b c".split(" "),
-                    SparseVector(3, {0: 1.0, 1: 1.0}),
-                ),
-                (
-                    2,
-                    "a b".split(" "),
-                    SparseVector(3, {0: 1.0}),
-                ),
-                (
-                    3,
-                    "a".split(" "),
-                    SparseVector(3, {}),
-                ),
-            ],
-            ["id", "words", "expected"],
-        )
-        cv = CountVectorizer(inputCol="words", outputCol="features")
-        model1 = cv.setMaxDF(3).fit(dataset)
-        self.assertEqual(model1.vocabulary, ["b", "c", "d"])
-
-        transformedList1 = model1.transform(dataset).select("features", "expected").collect()
-
-        for r in transformedList1:
-            feature, expected = r
-            self.assertEqual(feature, expected)
-
-        model2 = cv.setMaxDF(0.75).fit(dataset)
-        self.assertEqual(model2.vocabulary, ["b", "c", "d"])
-
-        transformedList2 = model2.transform(dataset).select("features", "expected").collect()
-
-        for r in transformedList2:
-            feature, expected = r
-            self.assertEqual(feature, expected)
-
-    def test_count_vectorizer_from_vocab(self):
-        model = CountVectorizerModel.from_vocabulary(
-            ["a", "b", "c"], inputCol="words", outputCol="features", minTF=2
-        )
-        self.assertEqual(model.vocabulary, ["a", "b", "c"])
-        self.assertEqual(model.getMinTF(), 2)
-
-        dataset = self.spark.createDataFrame(
-            [
-                (
-                    0,
-                    "a a a b b c".split(" "),
-                    SparseVector(3, {0: 3.0, 1: 2.0}),
-                ),
-                (
-                    1,
-                    "a a".split(" "),
-                    SparseVector(3, {0: 2.0}),
-                ),
-                (
-                    2,
-                    "a b".split(" "),
-                    SparseVector(3, {}),
-                ),
-            ],
-            ["id", "words", "expected"],
-        )
-
-        transformed_list = model.transform(dataset).select("features", "expected").collect()
-
-        for r in transformed_list:
-            feature, expected = r
-            self.assertEqual(feature, expected)
-
-        # Test an empty vocabulary
-        with QuietTest(self.sc):
-            with self.assertRaisesRegex(Exception, "vocabSize.*invalid.*0"):
-                CountVectorizerModel.from_vocabulary([], inputCol="words")
-
-        # Test model with default settings can transform
-        model_default = CountVectorizerModel.from_vocabulary(["a", "b", "c"], inputCol="words")
-        transformed_list = (
-            model_default.transform(dataset)
-            .select(model_default.getOrDefault(model_default.outputCol))
-            .collect()
-        )
-        self.assertEqual(len(transformed_list), 3)
-
-    def test_rformula_force_index_label(self):
-        df = self.spark.createDataFrame(
-            [(1.0, 1.0, "a"), (0.0, 2.0, "b"), (1.0, 0.0, "a")], ["y", "x", "s"]
-        )
-        # Does not index label by default since it's numeric type.
-        rf = RFormula(formula="y ~ x + s")
-        model = rf.fit(df)
-        transformedDF = model.transform(df)
-        self.assertEqual(transformedDF.head().label, 1.0)
-        # Force to index label.
-        rf2 = RFormula(formula="y ~ x + s").setForceIndexLabel(True)
-        model2 = rf2.fit(df)
-        transformedDF2 = model2.transform(df)
-        self.assertEqual(transformedDF2.head().label, 0.0)
-
-    def test_rformula_string_indexer_order_type(self):
-        df = self.spark.createDataFrame(
-            [(1.0, 1.0, "a"), (0.0, 2.0, "b"), (1.0, 0.0, "a")], ["y", "x", "s"]
-        )
-        rf = RFormula(formula="y ~ x + s", stringIndexerOrderType="alphabetDesc")
-        self.assertEqual(rf.getStringIndexerOrderType(), "alphabetDesc")
-        transformedDF = rf.fit(df).transform(df)
-        observed = transformedDF.select("features").collect()
-        expected = [[1.0, 0.0], [2.0, 1.0], [0.0, 0.0]]
-        for i in range(0, len(expected)):
-            self.assertTrue(all(observed[i]["features"].toArray() == expected[i]))
-
-    def test_string_indexer_handle_invalid(self):
-        df = self.spark.createDataFrame([(0, "a"), (1, "d"), (2, None)], ["id", "label"])
-
-        si1 = StringIndexer(
-            inputCol="label",
-            outputCol="indexed",
-            handleInvalid="keep",
-            stringOrderType="alphabetAsc",
-        )
-        model1 = si1.fit(df)
-        td1 = model1.transform(df)
-        actual1 = td1.select("id", "indexed").collect()
-        expected1 = [Row(id=0, indexed=0.0), Row(id=1, indexed=1.0), Row(id=2, indexed=2.0)]
-        self.assertEqual(actual1, expected1)
-
-        si2 = si1.setHandleInvalid("skip")
-        model2 = si2.fit(df)
-        td2 = model2.transform(df)
-        actual2 = td2.select("id", "indexed").collect()
-        expected2 = [Row(id=0, indexed=0.0), Row(id=1, indexed=1.0)]
-        self.assertEqual(actual2, expected2)
-
-    def test_string_indexer_from_labels(self):
-        model = StringIndexerModel.from_labels(
-            ["a", "b", "c"], inputCol="label", outputCol="indexed", handleInvalid="keep"
-        )
-        self.assertEqual(model.labels, ["a", "b", "c"])
-        self.assertEqual(model.labelsArray, [("a", "b", "c")])
-
-        df1 = self.spark.createDataFrame(
-            [(0, "a"), (1, "c"), (2, None), (3, "b"), (4, "b")], ["id", "label"]
-        )
-
-        result1 = model.transform(df1)
-        actual1 = result1.select("id", "indexed").collect()
-        expected1 = [
-            Row(id=0, indexed=0.0),
-            Row(id=1, indexed=2.0),
-            Row(id=2, indexed=3.0),
-            Row(id=3, indexed=1.0),
-            Row(id=4, indexed=1.0),
-        ]
-        self.assertEqual(actual1, expected1)
-
-        model_empty_labels = StringIndexerModel.from_labels(
-            [], inputCol="label", outputCol="indexed", handleInvalid="keep"
-        )
-        actual2 = model_empty_labels.transform(df1).select("id", "indexed").collect()
-        expected2 = [
-            Row(id=0, indexed=0.0),
-            Row(id=1, indexed=0.0),
-            Row(id=2, indexed=0.0),
-            Row(id=3, indexed=0.0),
-            Row(id=4, indexed=0.0),
-        ]
-        self.assertEqual(actual2, expected2)
-
-        # Test model with default settings can transform
-        model_default = StringIndexerModel.from_labels(["a", "b", "c"], inputCol="label")
-        df2 = self.spark.createDataFrame(
-            [(0, "a"), (1, "c"), (2, "b"), (3, "b"), (4, "b")], ["id", "label"]
-        )
-        transformed_list = (
-            model_default.transform(df2)
-            .select(model_default.getOrDefault(model_default.outputCol))
-            .collect()
-        )
-        self.assertEqual(len(transformed_list), 5)
-
-    def test_vector_size_hint(self):
-        df = self.spark.createDataFrame(
-            [
-                (0, Vectors.dense([0.0, 10.0, 0.5])),
-                (1, Vectors.dense([1.0, 11.0, 0.5, 0.6])),
-                (2, Vectors.dense([2.0, 12.0])),
-            ],
-            ["id", "vector"],
-        )
-
-        sizeHint = VectorSizeHint(inputCol="vector", handleInvalid="skip")
-        sizeHint.setSize(3)
-        self.assertEqual(sizeHint.getSize(), 3)
-
-        output = sizeHint.transform(df).head().vector
-        expected = DenseVector([0.0, 10.0, 0.5])
-        self.assertEqual(output, expected)
-
-
-class HashingTFTest(SparkSessionTestCase):
-    def test_apply_binary_term_freqs(self):
-
-        df = self.spark.createDataFrame([(0, ["a", "a", "b", "c", "c", "c"])], ["id", "words"])
-        n = 10
-        hashingTF = HashingTF()
-        hashingTF.setInputCol("words").setOutputCol("features").setNumFeatures(n).setBinary(True)
-        output = hashingTF.transform(df)
-        features = output.select("features").first().features.toArray()
-        expected = Vectors.dense([0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 1.0, 0.0]).toArray()
-        for i in range(0, n):
-            self.assertAlmostEqual(
-                features[i],
-                expected[i],
-                14,
-                "Error at "
-                + str(i)
-                + ": expected "
-                + str(expected[i])
-                + ", got "
-                + str(features[i]),
-            )
-
-
-if __name__ == "__main__":
-    from pyspark.ml.tests.test_feature import *  # noqa: F401
-
-    try:
-        import xmlrunner
-
-        testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
-    except ImportError:
-        testRunner = None
-    unittest.main(testRunner=testRunner, verbosity=2)