sqlframe 1.1.0__tar.gz → 1.1.2__tar.gz

{sqlframe-1.1.0 → sqlframe-1.1.2}/Makefile

@@ -1,5 +1,5 @@
 install-dev:
-	pip install -e ".[dev,duckdb,postgres,redshift,bigquery,snowflake,spark]"
+	pip install -e ".[dev,docs,duckdb,postgres,redshift,bigquery,snowflake,spark]"
 
 install-pre-commit:
 	pre-commit install

{sqlframe-1.1.0 → sqlframe-1.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 1.1.0
+Version: 1.1.2
 Summary: Taking the Spark out of PySpark by converting to SQL
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-1.1.0 → sqlframe-1.1.2}/docs/bigquery.md

@@ -43,6 +43,25 @@ Regardless of approach, it is recommended to configure `default_dataset` in the
     session = BigQuerySession(conn=conn, default_dataset="sqlframe.db1")
     ```
 
+## Imports
+
+If converting a PySpark pipeline, all `pyspark.sql` should be replaced with `sqlframe.bigquery`.
+In addition, many classes will have a `BigQuery` prefix. 
+For example, `BigQueryDataFrame` instead of `DataFrame`.
+
+
+```python
+# PySpark import
+# from pyspark.sql import SparkSession
+# from pyspark.sql import functions as F
+# from pyspark.sql.dataframe import DataFrame
+# SQLFrame import
+from sqlframe.bigquery import BigQuerySession
+from sqlframe.bigquery import functions as F
+from sqlframe.bigquery import BigQueryDataFrame
+```
+
+
 ## Example Usage
 
 ```python

{sqlframe-1.1.0 → sqlframe-1.1.2}/docs/duckdb.md

@@ -30,7 +30,23 @@ By default, SQLFrame will create a connection to an in-memory database.
     conn = duckdb.connect(database=":memory:")
     session = DuckDBSession(conn=conn)
     ```
+## Imports
 
+If converting a PySpark pipeline, all `pyspark.sql` should be replaced with `sqlframe.duckdb`.
+In addition, many classes will have a `DuckDB` prefix. 
+For example, `DuckDBDataFrame` instead of `DataFrame`.
+
+
+```python
+# PySpark import
+# from pyspark.sql import SparkSession
+# from pyspark.sql import functions as F
+# from pyspark.sql.dataframe import DataFrame
+# SQLFrame import
+from sqlframe.duckdb import DuckDBSession
+from sqlframe.duckdb import functions as F
+from sqlframe.duckdb import DuckDBDataFrame
+```
 
 ## Example Usage
 

{sqlframe-1.1.0 → sqlframe-1.1.2}/docs/postgres.md

@@ -25,6 +25,24 @@ conn = connect(
 session = PostgresSession(conn=conn)
 ```
 
+## Imports
+
+If converting a PySpark pipeline, all `pyspark.sql` should be replaced with `sqlframe.postgres`.
+In addition, many classes will have a `Postgres` prefix. 
+For example, `PostgresDataFrame` instead of `DataFrame`.
+
+
+```python
+# PySpark import
+# from pyspark.sql import SparkSession
+# from pyspark.sql import functions as F
+# from pyspark.sql.dataframe import DataFrame
+# SQLFrame import
+from sqlframe.postgres import PostgresSession
+from sqlframe.postgres import functions as F
+from sqlframe.postgres import PostgresDataFrame
+```
+
 ## Example Usage
 
 ```python

{sqlframe-1.1.0 → sqlframe-1.1.2}/docs/standalone.md

@@ -24,6 +24,24 @@ from sqlframe.standalone import StandaloneSession
 session = StandaloneSession.builder.config(map={"sqlframe.input.dialect": 'duckdb', "sqlframe.output.dialect": 'bigquery'}).getOrCreate()
 ```
 
+## Imports
+
+If converting a PySpark pipeline, all `pyspark.sql` should be replaced with `sqlframe.standalone`.
+In addition, many classes will have a `Standalone` prefix. 
+For example, `StandaloneDataFrame` instead of `DataFrame`.
+
+
+```python
+# PySpark import
+# from pyspark.sql import SparkSession
+# from pyspark.sql import functions as F
+# from pyspark.sql.dataframe import DataFrame
+# SQLFrame import
+from sqlframe.standalone import StandaloneSession
+from sqlframe.standalone import functions as F
+from sqlframe.standalone import StandaloneDataFrame
+```
+
 ## Accessing Tables
 
 PySpark DataFrame API, and currently SQLFrame, requires that a table can be access to get it's schema information.

{sqlframe-1.1.0 → sqlframe-1.1.2}/setup.py

@@ -47,7 +47,7 @@ setup(
         ],
         "docs": [
             "mkdocs==1.4.2",
-            "mkdocs-include-markdown-plugin==4.0.3",
+            "mkdocs-include-markdown-plugin==6.0.6",
             "mkdocs-material==9.0.5",
             "mkdocs-material-extensions==1.1.1",
             "pymdown-extensions",

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.1.0'
-__version_tuple__ = version_tuple = (1, 1, 0)
+__version__ = version = '1.1.2'
+__version_tuple__ = version_tuple = (1, 1, 2)

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe/base/catalog.py

@@ -1,4 +1,4 @@
-# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'dataframe' folder.
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
 
 from __future__ import annotations
 

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe/base/column.py

@@ -1,4 +1,4 @@
-# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'dataframe' folder.
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
 
 from __future__ import annotations
 

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe/base/dataframe.py

@@ -1,4 +1,4 @@
-# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'dataframe' folder.
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
 
 from __future__ import annotations
 
@@ -608,7 +608,9 @@ class _BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
     @operation(Operation.WHERE)
     def where(self, column: t.Union[Column, str, bool], **kwargs) -> Self:
         if isinstance(column, str):
-            col = sqlglot.parse_one(column, dialect=self.session.input_dialect)
+            col = self._ensure_and_normalize_col(
+                sqlglot.parse_one(column, dialect=self.session.input_dialect)
+            )
         else:
             col = self._ensure_and_normalize_col(column)
         return self.copy(expression=self.expression.where(col.expression))

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe/base/functions.py

@@ -1,4 +1,4 @@
-# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'dataframe' folder.
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
 
 from __future__ import annotations
 

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe/base/group.py

@@ -1,4 +1,4 @@
-# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'dataframe' folder.
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
 
 from __future__ import annotations
 

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe/base/normalize.py

@@ -1,4 +1,4 @@
-# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'dataframe' folder.
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
 
 from __future__ import annotations
 

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe/base/operations.py

@@ -1,4 +1,4 @@
-# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'dataframe' folder.
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
 
 from __future__ import annotations
 

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe/base/readerwriter.py

@@ -1,4 +1,4 @@
-# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'dataframe' folder.
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
 
 from __future__ import annotations
 

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe/base/session.py

@@ -1,4 +1,4 @@
-# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'dataframe' folder.
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
 
 from __future__ import annotations
 

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe/base/transforms.py

@@ -1,4 +1,4 @@
-# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'dataframe' folder.
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
 
 import typing as t
 

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe/base/types.py

@@ -1,4 +1,4 @@
-# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'dataframe' folder.
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
 
 from __future__ import annotations
 

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe/base/window.py

@@ -1,4 +1,4 @@
-# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'dataframe' folder.
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
 
 from __future__ import annotations
 

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe/bigquery/group.py

@@ -1,4 +1,4 @@
-# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'dataframe' folder.
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
 
 from __future__ import annotations
 

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe/bigquery/readwriter.py

@@ -1,4 +1,4 @@
-# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'dataframe' folder.
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
 
 from __future__ import annotations
 

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe/duckdb/catalog.py

@@ -1,4 +1,4 @@
-# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'dataframe' folder.
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
 
 from __future__ import annotations
 

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe/duckdb/group.py

@@ -1,4 +1,4 @@
-# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'dataframe' folder.
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
 
 from __future__ import annotations
 

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe/duckdb/readwriter.py

@@ -1,4 +1,4 @@
-# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'dataframe' folder.
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
 
 from __future__ import annotations
 
@@ -74,7 +74,10 @@ class DuckDBDataFrameReader(_BaseDataFrameReader["DuckDBSession", "DuckDBDataFra
         """
         if schema:
             column_mapping = ensure_column_mapping(schema)
-            select_columns = [x.expression for x in self._to_casted_columns(column_mapping)]
+            select_column_mapping = column_mapping.copy()
+            if options.get("filename"):
+                select_column_mapping["filename"] = "VARCHAR"
+            select_columns = [x.expression for x in self._to_casted_columns(select_column_mapping)]
             if format == "csv":
                 duckdb_columns = ", ".join(
                     [f"'{column}': '{dtype}'" for column, dtype in column_mapping.items()]

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe/postgres/catalog.py

@@ -1,4 +1,4 @@
-# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'dataframe' folder.
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
 
 from __future__ import annotations
 

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe/postgres/group.py

@@ -1,4 +1,4 @@
-# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'dataframe' folder.
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
 
 from __future__ import annotations
 

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe/postgres/readwriter.py

@@ -1,4 +1,4 @@
-# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'dataframe' folder.
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
 
 from __future__ import annotations
 

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe/redshift/catalog.py

@@ -1,4 +1,4 @@
-# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'dataframe' folder.
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
 
 from __future__ import annotations
 

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe/redshift/group.py

@@ -1,4 +1,4 @@
-# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'dataframe' folder.
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
 
 from __future__ import annotations
 

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe/redshift/readwriter.py

@@ -1,4 +1,4 @@
-# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'dataframe' folder.
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
 
 from __future__ import annotations
 

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe/snowflake/catalog.py

@@ -1,4 +1,4 @@
-# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'dataframe' folder.
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
 
 from __future__ import annotations
 

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe/snowflake/group.py

@@ -1,4 +1,4 @@
-# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'dataframe' folder.
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
 
 from __future__ import annotations
 

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe/snowflake/readwriter.py

@@ -1,4 +1,4 @@
-# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'dataframe' folder.
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
 
 from __future__ import annotations
 

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe/spark/catalog.py

@@ -1,4 +1,4 @@
-# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'dataframe' folder.
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
 
 from __future__ import annotations
 

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe/spark/group.py

@@ -1,4 +1,4 @@
-# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'dataframe' folder.
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
 
 from __future__ import annotations
 

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe/spark/readwriter.py

@@ -1,4 +1,4 @@
-# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'dataframe' folder.
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
 
 from __future__ import annotations
 

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe/standalone/group.py

@@ -1,4 +1,4 @@
-# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'dataframe' folder.
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
 
 from __future__ import annotations
 

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe/standalone/readwriter.py

@@ -1,4 +1,4 @@
-# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'dataframe' folder.
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
 
 from __future__ import annotations
 

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe/standalone/session.py

@@ -1,4 +1,4 @@
-# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'dataframe' folder.
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
 
 from __future__ import annotations
 

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 1.1.0
+Version: 1.1.2
 Summary: Taking the Spark out of PySpark by converting to SQL
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-1.1.0 → sqlframe-1.1.2}/sqlframe.egg-info/requires.txt

@@ -28,7 +28,7 @@ pre-commit>=3.5
 pre-commit<3.8,>=3.7
 
 [docs]
-mkdocs-include-markdown-plugin==4.0.3
+mkdocs-include-markdown-plugin==6.0.6
 mkdocs-material-extensions==1.1.1
 mkdocs-material==9.0.5
 mkdocs==1.4.2

{sqlframe-1.1.0 → sqlframe-1.1.2}/tests/integration/engines/duck/test_duckdb_reader.py

@@ -17,13 +17,56 @@ def test_employee_extra_line_csv(duckdb_session: DuckDBSession):
         auto_detect=False,
     )
     assert df.collect() == [
-        Row(**{"employee_id": 1, "fname": "Jack", "lname": "Shephard", "age": 37, "store_id": 1}),
-        Row(**{"employee_id": 2, "fname": "John", "lname": "Locke", "age": 65, "store_id": 1}),
-        Row(**{"employee_id": 3, "fname": "Kate", "lname": "Austen", "age": 37, "store_id": 2}),
         Row(
-            **{"employee_id": 4, "fname": "Claire", "lname": "Littleton", "age": 27, "store_id": 2}
+            **{
+                "employee_id": 1,
+                "fname": "Jack",
+                "lname": "Shephard",
+                "age": 37,
+                "store_id": 1,
+                "filename": "tests/fixtures/employee_extra_line.csv",
+            }
+        ),
+        Row(
+            **{
+                "employee_id": 2,
+                "fname": "John",
+                "lname": "Locke",
+                "age": 65,
+                "store_id": 1,
+                "filename": "tests/fixtures/employee_extra_line.csv",
+            }
+        ),
+        Row(
+            **{
+                "employee_id": 3,
+                "fname": "Kate",
+                "lname": "Austen",
+                "age": 37,
+                "store_id": 2,
+                "filename": "tests/fixtures/employee_extra_line.csv",
+            }
+        ),
+        Row(
+            **{
+                "employee_id": 4,
+                "fname": "Claire",
+                "lname": "Littleton",
+                "age": 27,
+                "store_id": 2,
+                "filename": "tests/fixtures/employee_extra_line.csv",
+            }
+        ),
+        Row(
+            **{
+                "employee_id": 5,
+                "fname": "Hugo",
+                "lname": "Reyes",
+                "age": 29,
+                "store_id": 100,
+                "filename": "tests/fixtures/employee_extra_line.csv",
+            }
         ),
-        Row(**{"employee_id": 5, "fname": "Hugo", "lname": "Reyes", "age": 29, "store_id": 100}),
     ]
 
 
@@ -34,7 +77,7 @@ def test_employee_extra_line_csv_multiple(duckdb_session: DuckDBSession):
         schema="employee_id INT, fname STRING, lname STRING, age INT, store_id INT",
         skip=1,
         header=1,
-        filename=1,
+        filename=0,
         null_padding=True,
         ignore_errors=1,
         auto_detect=False,

{sqlframe-1.1.0 → sqlframe-1.1.2}/tests/unit/standalone/test_dataframe.py

@@ -55,3 +55,12 @@ def test_with_column_duplicate_alias(standalone_employee: StandaloneDataFrame):
         df.sql(pretty=False)
         == "SELECT `a1`.`employee_id` AS `employee_id`, CAST(`a1`.`age` AS STRING) AS `fname`, CAST(`a1`.`lname` AS STRING) AS `lname`, `a1`.`age` AS `age`, `a1`.`store_id` AS `store_id` FROM VALUES (1, 'Jack', 'Shephard', 37, 1), (2, 'John', 'Locke', 65, 1), (3, 'Kate', 'Austen', 37, 2), (4, 'Claire', 'Littleton', 27, 2), (5, 'Hugo', 'Reyes', 29, 100) AS `a1`(`employee_id`, `fname`, `lname`, `age`, `store_id`)"
     )
+
+
+def test_where_expr(standalone_employee: StandaloneDataFrame):
+    df = standalone_employee.where("fname = 'Jack' AND age = 37")
+    assert df.columns == ["employee_id", "fname", "lname", "age", "store_id"]
+    assert (
+        df.sql(pretty=False)
+        == "SELECT `a1`.`employee_id` AS `employee_id`, CAST(`a1`.`fname` AS STRING) AS `fname`, CAST(`a1`.`lname` AS STRING) AS `lname`, `a1`.`age` AS `age`, `a1`.`store_id` AS `store_id` FROM VALUES (1, 'Jack', 'Shephard', 37, 1), (2, 'John', 'Locke', 65, 1), (3, 'Kate', 'Austen', 37, 2), (4, 'Claire', 'Littleton', 27, 2), (5, 'Hugo', 'Reyes', 29, 100) AS `a1`(`employee_id`, `fname`, `lname`, `age`, `store_id`) WHERE `a1`.`age` = 37 AND CAST(`a1`.`fname` AS STRING) = 'Jack'"
+    )