snowpark-connect 0.24.0__py3-none-any.whl → 0.26.0__py3-none-any.whl

snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_group.py

@@ -1,53 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-from pyspark.sql import Row
-from pyspark.testing.sqlutils import ReusedSQLTestCase
-
-
-class GroupTestsMixin:
-    def test_aggregator(self):
-        df = self.df
-        g = df.groupBy()
-        self.assertEqual([99, 100], sorted(g.agg({"key": "max", "value": "count"}).collect()[0]))
-        self.assertEqual([Row(**{"AVG(key#0)": 49.5})], g.mean().collect())
-
-        from pyspark.sql import functions
-
-        self.assertEqual(
-            (0, "99"), tuple(g.agg(functions.first(df.key), functions.last(df.value)).first())
-        )
-        self.assertTrue(95 < g.agg(functions.approx_count_distinct(df.key)).first()[0])
-        # test deprecated countDistinct
-        self.assertEqual(100, g.agg(functions.countDistinct(df.value)).first()[0])
-
-
-class GroupTests(GroupTestsMixin, ReusedSQLTestCase):
-    pass
-
-
-if __name__ == "__main__":
-    import unittest
-    from pyspark.sql.tests.test_group import *  # noqa: F401
-
-    try:
-        import xmlrunner
-
-        testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
-    except ImportError:
-        testRunner = None
-    unittest.main(testRunner=testRunner, verbosity=2)

snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_pandas_sqlmetrics.py

@@ -1,68 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import unittest
-from typing import cast
-
-from pyspark.sql.functions import pandas_udf
-from pyspark.testing.sqlutils import (
-    ReusedSQLTestCase,
-    have_pandas,
-    have_pyarrow,
-    pandas_requirement_message,
-    pyarrow_requirement_message,
-)
-
-
-@unittest.skipIf(
-    not have_pandas or not have_pyarrow,
-    cast(str, pandas_requirement_message or pyarrow_requirement_message),
-)
-class PandasSQLMetrics(ReusedSQLTestCase):
-    def test_pandas_sql_metrics_basic(self):
-        # SPARK-34265: Instrument Python UDFs using SQL metrics
-
-        python_sql_metrics = [
-            "data sent to Python workers",
-            "data returned from Python workers",
-            "number of output rows",
-        ]
-
-        @pandas_udf("long")
-        def test_pandas(col1):
-            return col1 * col1
-
-        self.spark.range(10).select(test_pandas("id")).collect()
-
-        statusStore = self.spark._jsparkSession.sharedState().statusStore()
-        lastExecId = statusStore.executionsList().last().executionId()
-        executionMetrics = statusStore.execution(lastExecId).get().metrics().mkString()
-
-        for metric in python_sql_metrics:
-            self.assertIn(metric, executionMetrics)
-
-
-if __name__ == "__main__":
-    from pyspark.sql.tests.test_pandas_sqlmetrics import *  # noqa: F401
-
-    try:
-        import xmlrunner
-
-        testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
-    except ImportError:
-        testRunner = None
-    unittest.main(testRunner=testRunner, verbosity=2)

snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_readwriter.py

@@ -1,283 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import os
-import unittest
-import shutil
-import tempfile
-
-from pyspark.errors import AnalysisException
-from pyspark.sql.functions import col, lit
-from pyspark.sql.readwriter import DataFrameWriterV2
-from pyspark.sql.types import StructType, StructField, StringType
-from pyspark.testing.sqlutils import ReusedSQLTestCase
-
-
-class ReadwriterTestsMixin:
-    def test_save_and_load(self):
-        df = self.df
-        tmpPath = tempfile.mkdtemp()
-        shutil.rmtree(tmpPath)
-        try:
-            df.write.json(tmpPath)
-            actual = self.spark.read.json(tmpPath)
-            self.assertEqual(sorted(df.collect()), sorted(actual.collect()))
-
-            schema = StructType([StructField("value", StringType(), True)])
-            actual = self.spark.read.json(tmpPath, schema)
-            self.assertEqual(sorted(df.select("value").collect()), sorted(actual.collect()))
-
-            df.write.json(tmpPath, "overwrite")
-            actual = self.spark.read.json(tmpPath)
-            self.assertEqual(sorted(df.collect()), sorted(actual.collect()))
-
-            df.write.save(
-                format="json",
-                mode="overwrite",
-                path=tmpPath,
-                noUse="this options will not be used in save.",
-            )
-            actual = self.spark.read.load(
-                format="json", path=tmpPath, noUse="this options will not be used in load."
-            )
-            self.assertEqual(sorted(df.collect()), sorted(actual.collect()))
-
-            try:
-                self.spark.sql("SET spark.sql.sources.default=org.apache.spark.sql.json")
-                actual = self.spark.read.load(path=tmpPath)
-                self.assertEqual(sorted(df.collect()), sorted(actual.collect()))
-            finally:
-                self.spark.sql("RESET spark.sql.sources.default")
-
-            csvpath = os.path.join(tempfile.mkdtemp(), "data")
-            df.write.option("quote", None).format("csv").save(csvpath)
-        finally:
-            shutil.rmtree(tmpPath)
-
-    def test_save_and_load_builder(self):
-        df = self.df
-        tmpPath = tempfile.mkdtemp()
-        shutil.rmtree(tmpPath)
-        try:
-            df.write.json(tmpPath)
-            actual = self.spark.read.json(tmpPath)
-            self.assertEqual(sorted(df.collect()), sorted(actual.collect()))
-
-            schema = StructType([StructField("value", StringType(), True)])
-            actual = self.spark.read.json(tmpPath, schema)
-            self.assertEqual(sorted(df.select("value").collect()), sorted(actual.collect()))
-
-            df.write.mode("overwrite").json(tmpPath)
-            actual = self.spark.read.json(tmpPath)
-            self.assertEqual(sorted(df.collect()), sorted(actual.collect()))
-
-            df.write.mode("overwrite").options(
-                noUse="this options will not be used in save."
-            ).option("noUse", "this option will not be used in save.").format("json").save(
-                path=tmpPath
-            )
-            actual = self.spark.read.format("json").load(
-                path=tmpPath, noUse="this options will not be used in load."
-            )
-            self.assertEqual(sorted(df.collect()), sorted(actual.collect()))
-
-            try:
-                self.spark.sql("SET spark.sql.sources.default=org.apache.spark.sql.json")
-                actual = self.spark.read.load(path=tmpPath)
-                self.assertEqual(sorted(df.collect()), sorted(actual.collect()))
-            finally:
-                self.spark.sql("RESET spark.sql.sources.default")
-        finally:
-            shutil.rmtree(tmpPath)
-
-    def test_bucketed_write(self):
-        data = [
-            (1, "foo", 3.0),
-            (2, "foo", 5.0),
-            (3, "bar", -1.0),
-            (4, "bar", 6.0),
-        ]
-        df = self.spark.createDataFrame(data, ["x", "y", "z"])
-
-        def count_bucketed_cols(names, table="pyspark_bucket"):
-            """Given a sequence of column names and a table name
-            query the catalog and return number o columns which are
-            used for bucketing
-            """
-            cols = self.spark.catalog.listColumns(table)
-            num = len([c for c in cols if c.name in names and c.isBucket])
-            return num
-
-        with self.table("pyspark_bucket"):
-            # Test write with one bucketing column
-            df.write.bucketBy(3, "x").mode("overwrite").saveAsTable("pyspark_bucket")
-            self.assertEqual(count_bucketed_cols(["x"]), 1)
-            self.assertSetEqual(set(data), set(self.spark.table("pyspark_bucket").collect()))
-
-            # Test write two bucketing columns
-            df.write.bucketBy(3, "x", "y").mode("overwrite").saveAsTable("pyspark_bucket")
-            self.assertEqual(count_bucketed_cols(["x", "y"]), 2)
-            self.assertSetEqual(set(data), set(self.spark.table("pyspark_bucket").collect()))
-
-            # Test write with bucket and sort
-            df.write.bucketBy(2, "x").sortBy("z").mode("overwrite").saveAsTable("pyspark_bucket")
-            self.assertEqual(count_bucketed_cols(["x"]), 1)
-            self.assertSetEqual(set(data), set(self.spark.table("pyspark_bucket").collect()))
-
-            # Test write with a list of columns
-            df.write.bucketBy(3, ["x", "y"]).mode("overwrite").saveAsTable("pyspark_bucket")
-            self.assertEqual(count_bucketed_cols(["x", "y"]), 2)
-            self.assertSetEqual(set(data), set(self.spark.table("pyspark_bucket").collect()))
-
-            # Test write with bucket and sort with a list of columns
-            (
-                df.write.bucketBy(2, "x")
-                .sortBy(["y", "z"])
-                .mode("overwrite")
-                .saveAsTable("pyspark_bucket")
-            )
-            self.assertSetEqual(set(data), set(self.spark.table("pyspark_bucket").collect()))
-
-            # Test write with bucket and sort with multiple columns
-            (
-                df.write.bucketBy(2, "x")
-                .sortBy("y", "z")
-                .mode("overwrite")
-                .saveAsTable("pyspark_bucket")
-            )
-            self.assertSetEqual(set(data), set(self.spark.table("pyspark_bucket").collect()))
-
-    def test_insert_into(self):
-        df = self.spark.createDataFrame([("a", 1), ("b", 2)], ["C1", "C2"])
-        with self.table("test_table"):
-            df.write.saveAsTable("test_table")
-            self.assertEqual(2, self.spark.sql("select * from test_table").count())
-
-            df.write.insertInto("test_table")
-            self.assertEqual(4, self.spark.sql("select * from test_table").count())
-
-            df.write.mode("overwrite").insertInto("test_table")
-            self.assertEqual(2, self.spark.sql("select * from test_table").count())
-
-            df.write.insertInto("test_table", True)
-            self.assertEqual(2, self.spark.sql("select * from test_table").count())
-
-            df.write.insertInto("test_table", False)
-            self.assertEqual(4, self.spark.sql("select * from test_table").count())
-
-            df.write.mode("overwrite").insertInto("test_table", False)
-            self.assertEqual(6, self.spark.sql("select * from test_table").count())
-
-    def test_cached_table(self):
-        with self.table("test_cached_table_1"):
-            self.spark.range(10).withColumn(
-                "value_1",
-                lit(1),
-            ).write.saveAsTable("test_cached_table_1")
-
-            with self.table("test_cached_table_2"):
-                self.spark.range(10).withColumnRenamed("id", "index").withColumn(
-                    "value_2", lit(2)
-                ).write.saveAsTable("test_cached_table_2")
-
-                df1 = self.spark.read.table("test_cached_table_1")
-                df2 = self.spark.read.table("test_cached_table_2")
-                df3 = self.spark.read.table("test_cached_table_1")
-
-                join1 = df1.join(df2, on=df1.id == df2.index).select(df2.index, df2.value_2)
-                join2 = df3.join(join1, how="left", on=join1.index == df3.id)
-
-                self.assertEqual(join2.columns, ["id", "value_1", "index", "value_2"])
-
-
-class ReadwriterV2TestsMixin:
-    def test_api(self):
-        self.check_api(DataFrameWriterV2)
-
-    def check_api(self, tpe):
-        df = self.df
-        writer = df.writeTo("testcat.t")
-        self.assertIsInstance(writer, tpe)
-        self.assertIsInstance(writer.option("property", "value"), tpe)
-        self.assertIsInstance(writer.options(property="value"), tpe)
-        self.assertIsInstance(writer.using("source"), tpe)
-        self.assertIsInstance(writer.partitionedBy("id"), tpe)
-        self.assertIsInstance(writer.partitionedBy(col("id")), tpe)
-        self.assertIsInstance(writer.tableProperty("foo", "bar"), tpe)
-
-    def test_partitioning_functions(self):
-        self.check_partitioning_functions(DataFrameWriterV2)
-
-    def check_partitioning_functions(self, tpe):
-        import datetime
-        from pyspark.sql.functions import years, months, days, hours, bucket
-
-        df = self.spark.createDataFrame(
-            [(1, datetime.datetime(2000, 1, 1), "foo")], ("id", "ts", "value")
-        )
-
-        writer = df.writeTo("testcat.t")
-
-        self.assertIsInstance(writer.partitionedBy(years("ts")), tpe)
-        self.assertIsInstance(writer.partitionedBy(months("ts")), tpe)
-        self.assertIsInstance(writer.partitionedBy(days("ts")), tpe)
-        self.assertIsInstance(writer.partitionedBy(hours("ts")), tpe)
-        self.assertIsInstance(writer.partitionedBy(bucket(11, "id")), tpe)
-        self.assertIsInstance(writer.partitionedBy(bucket(11, col("id"))), tpe)
-        self.assertIsInstance(writer.partitionedBy(bucket(3, "id"), hours(col("ts"))), tpe)
-
-    def test_create(self):
-        df = self.df
-        with self.table("test_table"):
-            df.writeTo("test_table").using("parquet").create()
-            self.assertEqual(100, self.spark.sql("select * from test_table").count())
-
-    @unittest.skipIf(
-        "SPARK_SKIP_CONNECT_COMPAT_TESTS" in os.environ, "Known behavior change in 4.0"
-    )
-    def test_create_without_provider(self):
-        df = self.df
-        with self.assertRaisesRegex(
-            AnalysisException, "NOT_SUPPORTED_COMMAND_WITHOUT_HIVE_SUPPORT"
-        ):
-            df.writeTo("test_table").create()
-
-    def test_table_overwrite(self):
-        df = self.df
-        with self.assertRaisesRegex(AnalysisException, "TABLE_OR_VIEW_NOT_FOUND"):
-            df.writeTo("test_table").overwrite(lit(True))
-
-
-class ReadwriterTests(ReadwriterTestsMixin, ReusedSQLTestCase):
-    pass
-
-
-class ReadwriterV2Tests(ReadwriterV2TestsMixin, ReusedSQLTestCase):
-    pass
-
-
-if __name__ == "__main__":
-    import unittest
-    from pyspark.sql.tests.test_readwriter import *  # noqa: F401
-
-    try:
-        import xmlrunner
-
-        testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
-    except ImportError:
-        testRunner = None
-    unittest.main(testRunner=testRunner, verbosity=2)

snowflake/snowpark_connect/includes/python/pyspark/sql/tests/test_serde.py

@@ -1,155 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import datetime
-import shutil
-import tempfile
-import time
-
-from pyspark.sql import Row
-from pyspark.sql.functions import lit
-from pyspark.sql.types import StructType, StructField, DecimalType, BinaryType
-from pyspark.testing.objects import UTCOffsetTimezone
-from pyspark.testing.sqlutils import ReusedSQLTestCase
-
-
-class SerdeTestsMixin:
-    def test_serialize_nested_array_and_map(self):
-        d = [Row(lst=[Row(a=1, b="s")], d={"key": Row(c=1.0, d="2")})]
-        rdd = self.sc.parallelize(d)
-        df = self.spark.createDataFrame(rdd)
-        row = df.head()
-        self.assertEqual(1, len(row.lst))
-        self.assertEqual(1, row.lst[0].a)
-        self.assertEqual("2", row.d["key"].d)
-
-        lst = df.rdd.map(lambda x: x.lst).first()
-        self.assertEqual(1, len(lst))
-        self.assertEqual("s", lst[0].b)
-
-        d = df.rdd.map(lambda x: x.d).first()
-        self.assertEqual(1, len(d))
-        self.assertEqual(1.0, d["key"].c)
-
-        row = df.rdd.map(lambda x: x.d["key"]).first()
-        self.assertEqual(1.0, row.c)
-        self.assertEqual("2", row.d)
-
-    def test_select_null_literal(self):
-        df = self.spark.sql("select null as col")
-        self.assertEqual(Row(col=None), df.first())
-
-    def test_struct_in_map(self):
-        d = [Row(m={Row(i=1): Row(s="")})]
-        df = self.spark.createDataFrame(d)
-        k, v = list(df.head().m.items())[0]
-        self.assertEqual(1, k.i)
-        self.assertEqual("", v.s)
-
-    def test_filter_with_datetime(self):
-        time = datetime.datetime(2015, 4, 17, 23, 1, 2, 3000)
-        date = time.date()
-        row = Row(date=date, time=time)
-        df = self.spark.createDataFrame([row])
-        self.assertEqual(1, df.filter(df.date == date).count())
-        self.assertEqual(1, df.filter(df.time == time).count())
-        self.assertEqual(0, df.filter(df.date > date).count())
-        self.assertEqual(0, df.filter(df.time > time).count())
-
-    def test_filter_with_datetime_timezone(self):
-        dt1 = datetime.datetime(2015, 4, 17, 23, 1, 2, 3000, tzinfo=UTCOffsetTimezone(0))
-        dt2 = datetime.datetime(2015, 4, 17, 23, 1, 2, 3000, tzinfo=UTCOffsetTimezone(1))
-        row = Row(date=dt1)
-        df = self.spark.createDataFrame([row])
-        self.assertEqual(0, df.filter(df.date == dt2).count())
-        self.assertEqual(1, df.filter(df.date > dt2).count())
-        self.assertEqual(0, df.filter(df.date < dt2).count())
-
-    def test_time_with_timezone(self):
-        day = datetime.date.today()
-        now = datetime.datetime.now()
-        ts = time.mktime(now.timetuple())
-        utc = UTCOffsetTimezone()
-        utcnow = datetime.datetime.utcfromtimestamp(ts)  # without microseconds
-        # add microseconds to utcnow (keeping year,month,day,hour,minute,second)
-        utcnow = datetime.datetime(*(utcnow.timetuple()[:6] + (now.microsecond, utc)))
-        df = self.spark.createDataFrame([(day, now, utcnow)])
-        day1, now1, utcnow1 = df.first()
-        self.assertEqual(day1, day)
-        self.assertEqual(now, now1)
-        self.assertEqual(now, utcnow1)
-
-    # regression test for SPARK-19561
-    def test_datetime_at_epoch(self):
-        epoch = datetime.datetime.fromtimestamp(0)
-        df = self.spark.createDataFrame([Row(date=epoch)])
-        first = df.select("date", lit(epoch).alias("lit_date")).first()
-        self.assertEqual(first["date"], epoch)
-        self.assertEqual(first["lit_date"], epoch)
-
-    def test_decimal(self):
-        from decimal import Decimal
-
-        schema = StructType([StructField("decimal", DecimalType(10, 5))])
-        df = self.spark.createDataFrame([(Decimal("3.14159"),)], schema)
-        row = df.select(df.decimal + 1).first()
-        self.assertEqual(row[0], Decimal("4.14159"))
-        tmpPath = tempfile.mkdtemp()
-        shutil.rmtree(tmpPath)
-        df.write.parquet(tmpPath)
-        df2 = self.spark.read.parquet(tmpPath)
-        row = df2.first()
-        self.assertEqual(row[0], Decimal("3.14159"))
-
-    def test_BinaryType_serialization(self):
-        # Pyrolite version <= 4.9 could not serialize BinaryType with Python3 SPARK-17808
-        # The empty bytearray is test for SPARK-21534.
-        schema = StructType([StructField("mybytes", BinaryType())])
-        data = [
-            [bytearray(b"here is my data")],
-            [bytearray(b"and here is some more")],
-            [bytearray(b"")],
-        ]
-        df = self.spark.createDataFrame(data, schema=schema)
-        df.collect()
-
-    def test_int_array_serialization(self):
-        # Note that this test seems dependent on parallelism.
-        data = self.spark.sparkContext.parallelize([[1, 2, 3, 4]] * 100, numSlices=12)
-        df = self.spark.createDataFrame(data, "array<integer>")
-        self.assertEqual(len(list(filter(lambda r: None in r.value, df.collect()))), 0)
-
-    def test_bytes_as_binary_type(self):
-        df = self.spark.createDataFrame([[b"abcd"]], "col binary")
-        self.assertEqual(df.first().col, bytearray(b"abcd"))
-
-
-class SerdeTests(SerdeTestsMixin, ReusedSQLTestCase):
-    pass
-
-
-if __name__ == "__main__":
-    import unittest
-    from pyspark.sql.tests.test_serde import *  # noqa: F401
-
-    try:
-        import xmlrunner
-
-        testRunner = xmlrunner.XMLTestRunner(output="target/test-reports", verbosity=2)
-    except ImportError:
-        testRunner = None
-    unittest.main(testRunner=testRunner, verbosity=2)