sqlframe 3.28.1__tar.gz → 3.29.0__tar.gz

{sqlframe-3.28.1 → sqlframe-3.29.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.28.1
+Version: 3.29.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman
@@ -34,18 +34,19 @@ License-File: LICENSE
 
 SQLFrame implements the PySpark DataFrame API in order to enable running transformation pipelines directly on database engines - no Spark clusters or dependencies required.
 
-SQLFrame currently supports the following engines (many more in development):
+SQLFrame currently supports the following engines:
 
 * [BigQuery](https://sqlframe.readthedocs.io/en/stable/bigquery/)
+* [Databricks](https://sqlframe.readthedocs.io/en/stable/databricks)
 * [DuckDB](https://sqlframe.readthedocs.io/en/stable/duckdb)
 * [Postgres](https://sqlframe.readthedocs.io/en/stable/postgres)
 * [Snowflake](https://sqlframe.readthedocs.io/en/stable/snowflake)
 * [Spark](https://sqlframe.readthedocs.io/en/stable/spark)
 
-There are also two engines in development. These engines lack test coverage and robust documentation, but are available for early testing:
+There is also one engine in development. This engine lacks test coverage and robust documentation, but is available for testing:
 
 * [Redshift](https://sqlframe.readthedocs.io/en/stable/redshift)
-* [Databricks](https://sqlframe.readthedocs.io/en/stable/databricks)
+
 
 SQLFrame also has a "Standalone" session that be used to generate SQL without any connection to a database engine.
 
@@ -63,6 +64,8 @@ SQLFrame is great for:
 ```bash
 # BigQuery
 pip install "sqlframe[bigquery]"
+# Databricks
+pip install "sqlframe[databricks]"
 # DuckDB
 pip install "sqlframe[duckdb]"
 # Postgres
@@ -73,8 +76,6 @@ pip install "sqlframe[snowflake]"
 pip install "sqlframe[spark]"
 # Redshift (in development)
 pip install "sqlframe[redshift]"
-# Databricks (in development)
-pip install "sqlframe[databricks]"
 # Standalone
 pip install sqlframe
 ```

{sqlframe-3.28.1 → sqlframe-3.29.0}/README.md

@@ -4,18 +4,19 @@
 
 SQLFrame implements the PySpark DataFrame API in order to enable running transformation pipelines directly on database engines - no Spark clusters or dependencies required.
 
-SQLFrame currently supports the following engines (many more in development):
+SQLFrame currently supports the following engines:
 
 * [BigQuery](https://sqlframe.readthedocs.io/en/stable/bigquery/)
+* [Databricks](https://sqlframe.readthedocs.io/en/stable/databricks)
 * [DuckDB](https://sqlframe.readthedocs.io/en/stable/duckdb)
 * [Postgres](https://sqlframe.readthedocs.io/en/stable/postgres)
 * [Snowflake](https://sqlframe.readthedocs.io/en/stable/snowflake)
 * [Spark](https://sqlframe.readthedocs.io/en/stable/spark)
 
-There are also two engines in development. These engines lack test coverage and robust documentation, but are available for early testing:
+There is also one engine in development. This engine lacks test coverage and robust documentation, but is available for testing:
 
 * [Redshift](https://sqlframe.readthedocs.io/en/stable/redshift)
-* [Databricks](https://sqlframe.readthedocs.io/en/stable/databricks)
+
 
 SQLFrame also has a "Standalone" session that be used to generate SQL without any connection to a database engine.
 
@@ -33,6 +34,8 @@ SQLFrame is great for:
 ```bash
 # BigQuery
 pip install "sqlframe[bigquery]"
+# Databricks
+pip install "sqlframe[databricks]"
 # DuckDB
 pip install "sqlframe[duckdb]"
 # Postgres
@@ -43,8 +46,6 @@ pip install "sqlframe[snowflake]"
 pip install "sqlframe[spark]"
 # Redshift (in development)
 pip install "sqlframe[redshift]"
-# Databricks (in development)
-pip install "sqlframe[databricks]"
 # Standalone
 pip install sqlframe
 ```

{sqlframe-3.28.1 → sqlframe-3.29.0}/docs/databricks.md

@@ -1,6 +1,4 @@
-from sqlframe.base.table import WhenNotMatchedBySourcefrom sqlframe.base.table import WhenMatched
-
-# Databricks (In Development)
+# Databricks
 
 ## Installation
 

{sqlframe-3.28.1 → sqlframe-3.29.0}/sqlframe/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.28.1'
-__version_tuple__ = version_tuple = (3, 28, 1)
+__version__ = version = '3.29.0'
+__version_tuple__ = version_tuple = (3, 29, 0)

{sqlframe-3.28.1 → sqlframe-3.29.0}/sqlframe/base/dataframe.py

@@ -80,8 +80,6 @@ JOIN_HINTS = {
 }
 
 JOIN_TYPE_MAPPING = {
-    "inner": "inner",
-    "cross": "cross",
     "outer": "full_outer",
     "full": "full_outer",
     "fullouter": "full_outer",
@@ -91,10 +89,8 @@ JOIN_TYPE_MAPPING = {
     "rightouter": "right_outer",
     "semi": "left_semi",
     "leftsemi": "left_semi",
-    "left_semi": "left_semi",
     "anti": "left_anti",
     "leftanti": "left_anti",
-    "left_anti": "left_anti",
 }
 
 DF = t.TypeVar("DF", bound="BaseDataFrame")
@@ -1028,7 +1024,7 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
     @operation(Operation.FROM)
     def join(
         self,
-        other_df: Self,
+        other: Self,
         on: t.Optional[t.Union[str, t.List[str], Column, t.List[Column]]] = None,
         how: str = "inner",
         **kwargs,
@@ -1044,7 +1040,7 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
             logger.warning("Got cross join with an 'on' value. This will result in an inner join.")
             how = "inner"
 
-        other_df = other_df._convert_leaf_to_cte()
+        other_df = other._convert_leaf_to_cte()
         join_expression = self._add_ctes_to_expression(self.expression, other_df.expression.ctes)
         # We will determine actual "join on" expression later so we don't provide it at first
         join_type = JOIN_TYPE_MAPPING.get(how, how).replace("_", " ")
@@ -1599,6 +1595,8 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
 
     @operation(Operation.LIMIT)
     def limit(self, num: int) -> Self:
+        if limit_exp := self.expression.args.get("limit"):
+            num = min(num, int(limit_exp.expression.this))
         return self.copy(expression=self.expression.limit(num))
 
     def toDF(self, *cols: str) -> Self:
@@ -1854,11 +1852,11 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
     def head(self, n: int) -> t.List[Row]: ...
 
     def head(self, n: t.Optional[int] = None) -> t.Union[t.Optional[Row], t.List[Row]]:
-        n = n or 1
-        df = self.limit(n)
-        if n == 1:
-            return df.collect()[0]
-        return df.collect()
+        df = self.limit(n or 1)
+        collected = df.collect()
+        if n is None:
+            return seq_get(collected, 0)
+        return collected
 
     def first(self) -> t.Optional[Row]:
         return self.head()
@@ -1939,6 +1937,11 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
     def createGlobalTempView(self, name: str) -> None:
         raise NotImplementedError("Global temp views are not yet supported")
 
+    def isEmpty(self) -> bool:
+        from sqlframe.base import functions as F
+
+        return not bool(self.select(F.lit(True)).head())
+
     """
     Stat Functions
     """

{sqlframe-3.28.1 → sqlframe-3.29.0}/sqlframe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.28.1
+Version: 3.29.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman
@@ -34,18 +34,19 @@ License-File: LICENSE
 
 SQLFrame implements the PySpark DataFrame API in order to enable running transformation pipelines directly on database engines - no Spark clusters or dependencies required.
 
-SQLFrame currently supports the following engines (many more in development):
+SQLFrame currently supports the following engines:
 
 * [BigQuery](https://sqlframe.readthedocs.io/en/stable/bigquery/)
+* [Databricks](https://sqlframe.readthedocs.io/en/stable/databricks)
 * [DuckDB](https://sqlframe.readthedocs.io/en/stable/duckdb)
 * [Postgres](https://sqlframe.readthedocs.io/en/stable/postgres)
 * [Snowflake](https://sqlframe.readthedocs.io/en/stable/snowflake)
 * [Spark](https://sqlframe.readthedocs.io/en/stable/spark)
 
-There are also two engines in development. These engines lack test coverage and robust documentation, but are available for early testing:
+There is also one engine in development. This engine lacks test coverage and robust documentation, but is available for testing:
 
 * [Redshift](https://sqlframe.readthedocs.io/en/stable/redshift)
-* [Databricks](https://sqlframe.readthedocs.io/en/stable/databricks)
+
 
 SQLFrame also has a "Standalone" session that be used to generate SQL without any connection to a database engine.
 
@@ -63,6 +64,8 @@ SQLFrame is great for:
 ```bash
 # BigQuery
 pip install "sqlframe[bigquery]"
+# Databricks
+pip install "sqlframe[databricks]"
 # DuckDB
 pip install "sqlframe[duckdb]"
 # Postgres
@@ -73,8 +76,6 @@ pip install "sqlframe[snowflake]"
 pip install "sqlframe[spark]"
 # Redshift (in development)
 pip install "sqlframe[redshift]"
-# Databricks (in development)
-pip install "sqlframe[databricks]"
 # Standalone
 pip install sqlframe
 ```

{sqlframe-3.28.1 → sqlframe-3.29.0}/tests/integration/engines/test_engine_dataframe.py

@@ -189,3 +189,47 @@ def test_show_from_create_with_space_with_schema(get_session: t.Callable[[], _Ba
     df.printSchema()
     captured = capsys.readouterr()
     assert "|-- an tan:" in captured.out.strip()
+
+
+def test_multiple_limit(get_session: t.Callable[[], _BaseSession], capsys, get_func) -> None:
+    session = get_session()
+    lit = get_func("lit", session)
+    df = session.range(20)
+    assert df.select("id").limit(10).limit(0).count() == 0
+    assert df.select("id").limit(0).limit(10).count() == 0
+    assert df.limit(0).limit(10).limit(5).count() == 0
+    assert df.limit(0).filter(lit(True)).limit(10).limit(5).count() == 0
+
+
+def test_is_empty(get_session):
+    session = get_session()
+    df_non_empty = session.createDataFrame([(1, 4), (2, 5), (3, 6)], schema=["foo", "BAR"])
+    df_empty = session.createDataFrame([], schema=["foo", "BAR"])
+
+    assert not df_non_empty.isEmpty()
+    assert df_empty.isEmpty()
+
+
+def test_head(get_session, get_func):
+    session = get_session()
+    lit = get_func("lit", session)
+    df_non_empty = session.createDataFrame(
+        data=[(2, "Alice"), (5, "Bob"), (8, "Charly")],
+        schema=["age", "name"],
+    )
+    df_empty = df_non_empty.filter(lit(False))
+
+    # head() on DataFrame containing data: single row
+    assert df_non_empty.head() == Row(**{"age": 2, "name": "Alice"})
+
+    # head() on empty DataFrame: None
+    assert df_empty.head() is None
+
+    # head(n) on DataFrame containing data: list of rows
+    assert df_non_empty.head(2) == [
+        Row(**{"age": 2, "name": "Alice"}),
+        Row(**{"age": 5, "name": "Bob"}),
+    ]
+
+    # head(n) on empty DataFrame : empty list
+    assert df_empty.head(2) == []

{sqlframe-3.28.1 → sqlframe-3.29.0}/tests/integration/test_int_dataframe.py

@@ -431,7 +431,7 @@ def test_join_inner(
 ):
     employee = get_df("employee")
     store = get_df("store")
-    df_joined = pyspark_employee.join(pyspark_store, on=["store_id"], how="inner").select(
+    df_joined = pyspark_employee.join(other=pyspark_store, on=["store_id"], how="inner").select(
         pyspark_employee.employee_id,
         pyspark_employee["fname"],
         F.col("lname"),