sqlframe 3.17.0__tar.gz → 3.18.0__tar.gz

{sqlframe-3.17.0 → sqlframe-3.18.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.17.0
+Version: 3.18.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.17.0 → sqlframe-3.18.0}/setup.py

@@ -20,7 +20,7 @@ setup(
     python_requires=">=3.9",
     install_requires=[
         "prettytable<4",
-        "sqlglot>=24.0.0,<26.4",
+        "sqlglot>=24.0.0,<26.5",
         "typing_extensions",
     ],
     extras_require={
@@ -31,7 +31,7 @@ setup(
         "dev": [
             "duckdb>=0.9,<1.2",
             "findspark>=2,<3",
-            "mypy>=1.10.0,<1.15",
+            "mypy>=1.10.0,<1.16",
             "openai>=1.30,<2",
             "pandas>=2,<3",
             "pandas-stubs>=2,<3",

{sqlframe-3.17.0 → sqlframe-3.18.0}/sqlframe/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '3.17.0'
-__version_tuple__ = version_tuple = (3, 17, 0)
+__version__ = version = '3.18.0'
+__version_tuple__ = version_tuple = (3, 18, 0)

{sqlframe-3.17.0 → sqlframe-3.18.0}/sqlframe/base/dataframe.py

@@ -23,7 +23,6 @@ from sqlglot import lineage as sqlglot_lineage
 from sqlglot.helper import ensure_list, flatten, object_to_dict, seq_get
 from sqlglot.optimizer.pushdown_projections import pushdown_projections
 from sqlglot.optimizer.qualify import qualify
-from sqlglot.optimizer.qualify_columns import quote_identifiers
 
 from sqlframe.base.catalog import Column as CatalogColumn
 from sqlframe.base.operations import Operation, operation
@@ -616,6 +615,7 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         self,
         optimize: bool = True,
         openai_config: t.Optional[t.Union[t.Dict[str, t.Any], OpenAIConfig]] = None,
+        quote_identifiers: bool = True,
     ) -> t.List[exp.Expression]:
         df = self._resolve_pending_hints()
         select_expressions = df._get_select_expressions()
@@ -644,7 +644,7 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
             if optimize:
                 select_expression = t.cast(
                     exp.Select,
-                    self.session._optimize(select_expression),
+                    self.session._optimize(select_expression, quote_identifiers=quote_identifiers),
                 )
             elif openai_config:
                 qualify(
@@ -713,8 +713,9 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         dialect: DialectType = ...,
         optimize: bool = ...,
         pretty: bool = ...,
+        quote_identifiers: bool = ...,
         *,
-        as_list: t.Literal[False],
+        as_list: t.Literal[False] = False,
         **kwargs: t.Any,
     ) -> str: ...
 
@@ -724,6 +725,7 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         dialect: DialectType = ...,
         optimize: bool = ...,
         pretty: bool = ...,
+        quote_identifiers: bool = ...,
         *,
         as_list: t.Literal[True],
         **kwargs: t.Any,
@@ -734,14 +736,23 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         dialect: DialectType = None,
         optimize: bool = True,
         pretty: bool = True,
+        quote_identifiers: bool = True,
         openai_config: t.Optional[t.Union[t.Dict[str, t.Any], OpenAIConfig]] = None,
         as_list: bool = False,
         **kwargs,
     ) -> t.Union[str, t.List[str]]:
         dialect = Dialect.get_or_raise(dialect) if dialect else self.session.output_dialect
         results = []
-        for expression in self._get_expressions(optimize=optimize, openai_config=openai_config):
-            sql = self.session._to_sql(expression, dialect=dialect, pretty=pretty, **kwargs)
+        for expression in self._get_expressions(
+            optimize=optimize, openai_config=openai_config, quote_identifiers=quote_identifiers
+        ):
+            sql = self.session._to_sql(
+                expression,
+                dialect=dialect,
+                pretty=pretty,
+                quote_identifiers=quote_identifiers,
+                **kwargs,
+            )
             if openai_config:
                 assert isinstance(openai_config, OpenAIConfig)
                 verify_openai_installed()
@@ -822,8 +833,9 @@ class BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
                 if cte:
                     resolved_column_position[ambiguous_col] += 1
                 else:
-                    cte = ctes_with_column[resolved_column_position[ambiguous_col]]
-                ambiguous_col.set("table", exp.to_identifier(cte.alias_or_name))
+                    cte = seq_get(ctes_with_column, resolved_column_position[ambiguous_col])
+                if cte:
+                    ambiguous_col.set("table", exp.to_identifier(cte.alias_or_name))
 
     @operation(Operation.SELECT)
     def select(self, *cols, **kwargs) -> Self:

{sqlframe-3.17.0 → sqlframe-3.18.0}/sqlframe/base/session.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import contextlib
 import datetime
 import logging
 import sys
@@ -15,6 +16,7 @@ from sqlglot import Dialect, exp
 from sqlglot.dialects.dialect import DialectType, NormalizationStrategy
 from sqlglot.expressions import parse_identifier
 from sqlglot.helper import ensure_list, seq_get
+from sqlglot.optimizer import RULES as OPTIMIZER_RULES
 from sqlglot.optimizer import optimize
 from sqlglot.optimizer.normalize_identifiers import normalize_identifiers
 from sqlglot.optimizer.qualify import qualify as qualify_func
@@ -213,13 +215,16 @@ class _BaseSession(t.Generic[CATALOG, READER, WRITER, DF, TABLE, CONN, UDF_REGIS
 
     def createDataFrame(
         self,
-        data: t.Sequence[
-            t.Union[
-                t.Dict[str, ColumnLiterals],
-                t.List[ColumnLiterals],
-                t.Tuple[ColumnLiterals, ...],
-                ColumnLiterals,
-            ]
+        data: t.Union[
+            t.Sequence[
+                t.Union[
+                    t.Dict[str, ColumnLiterals],
+                    t.List[ColumnLiterals],
+                    t.Tuple[ColumnLiterals, ...],
+                    ColumnLiterals,
+                ],
+            ],
+            pd.DataFrame,
         ],
         schema: t.Optional[SchemaInput] = None,
         samplingRatio: t.Optional[float] = None,
@@ -240,11 +245,18 @@ class _BaseSession(t.Generic[CATALOG, READER, WRITER, DF, TABLE, CONN, UDF_REGIS
         ):
             raise NotImplementedError("Only schema of either list or string of list supported")
 
+        with contextlib.suppress(ImportError):
+            from pandas import DataFrame as pd_DataFrame
+
+            if isinstance(data, pd_DataFrame):
+                data = data.to_dict("records")  # type: ignore
+
         column_mapping: t.Mapping[str, t.Optional[exp.DataType]]
         if schema is not None:
             column_mapping = get_column_mapping_from_schema_input(
                 schema, dialect=self.input_dialect
             )
+
         elif data:
             if isinstance(data[0], Row):
                 column_mapping = {col_name.strip(): None for col_name in data[0].__fields__}
@@ -386,7 +398,8 @@ class _BaseSession(t.Generic[CATALOG, READER, WRITER, DF, TABLE, CONN, UDF_REGIS
         dialect = Dialect.get_or_raise(dialect or self.input_dialect)
         expression = (
             sqlglot.parse_one(
-                normalize_string(sqlQuery, from_dialect=dialect, is_query=True), read=dialect
+                normalize_string(sqlQuery, from_dialect=dialect, is_query=True),
+                read=dialect,
             )
             if isinstance(sqlQuery, str)
             else sqlQuery
@@ -544,6 +557,7 @@ class _BaseSession(t.Generic[CATALOG, READER, WRITER, DF, TABLE, CONN, UDF_REGIS
         dialect: DialectType = None,
         quote_identifiers: bool = True,
         pretty: bool = False,
+        **kwargs,
     ) -> str:
         return normalize_string(
             sql,
@@ -562,9 +576,19 @@ class _BaseSession(t.Generic[CATALOG, READER, WRITER, DF, TABLE, CONN, UDF_REGIS
     ) -> exp.Expression:
         dialect = dialect or self.input_dialect
         normalize_identifiers(expression, dialect=dialect)
+        rules = list(OPTIMIZER_RULES)
         if quote_identifiers:
             quote_identifiers_func(expression, dialect=dialect)
-        return optimize(expression, dialect=dialect, schema=self.catalog._schema, infer_schema=True)
+        else:
+            rules.remove(quote_identifiers_func)
+        return optimize(
+            expression,
+            dialect=dialect,
+            schema=self.catalog._schema,
+            infer_schema=True,
+            quote_identifiers=quote_identifiers,
+            rules=rules,  # type: ignore
+        )
 
     def _execute(self, sql: str) -> None:
         self._cur.execute(sql)

{sqlframe-3.17.0 → sqlframe-3.18.0}/sqlframe/bigquery/session.py

@@ -27,7 +27,7 @@ class BigQuerySession(
         BigQueryDataFrameWriter,
         BigQueryDataFrame,
         BigQueryTable,
-        BigQueryConnection,
+        BigQueryConnection,  # type: ignore
         BigQueryUDFRegistration,
     ],
 ):

{sqlframe-3.17.0 → sqlframe-3.18.0}/sqlframe/databricks/session.py

@@ -26,7 +26,7 @@ class DatabricksSession(
         DatabricksDataFrameWriter,
         DatabricksDataFrame,
         DatabricksTable,
-        DatabricksConnection,
+        DatabricksConnection,  # type: ignore
         DatabricksUDFRegistration,
     ],
 ):

{sqlframe-3.17.0 → sqlframe-3.18.0}/sqlframe/duckdb/session.py

@@ -28,7 +28,7 @@ class DuckDBSession(
         DuckDBDataFrameWriter,
         DuckDBDataFrame,
         DuckDBTable,
-        DuckDBPyConnection,
+        DuckDBPyConnection,  # type: ignore
         DuckDBUDFRegistration,
     ]
 ):

{sqlframe-3.17.0 → sqlframe-3.18.0}/sqlframe/postgres/session.py

@@ -29,7 +29,7 @@ class PostgresSession(
         PostgresDataFrameWriter,
         PostgresDataFrame,
         PostgresTable,
-        psycopg2_connection,
+        psycopg2_connection,  # type: ignore
         PostgresUDFRegistration,
     ],
 ):

{sqlframe-3.17.0 → sqlframe-3.18.0}/sqlframe/snowflake/session.py

@@ -53,7 +53,7 @@ class SnowflakeSession(
         SnowflakeDataFrameWriter,
         SnowflakeDataFrame,
         SnowflakeTable,
-        SnowflakeConnection,
+        SnowflakeConnection,  # type: ignore
         SnowflakeUDFRegistration,
     ],
 ):

{sqlframe-3.17.0 → sqlframe-3.18.0}/sqlframe/spark/session.py

@@ -34,7 +34,7 @@ class SparkSession(
         SparkDataFrameWriter,
         SparkDataFrame,
         SparkTable,
-        PySparkSession,
+        PySparkSession,  # type: ignore
         SparkUDFRegistration,
     ],
 ):

{sqlframe-3.17.0 → sqlframe-3.18.0}/sqlframe/standalone/session.py

@@ -20,7 +20,7 @@ class StandaloneSession(
         StandaloneDataFrameWriter,
         StandaloneDataFrame,
         StandaloneTable,
-        object,
+        object,  # type: ignore
         StandaloneUDFRegistration,
     ]
 ):  # type: ignore

{sqlframe-3.17.0 → sqlframe-3.18.0}/sqlframe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 3.17.0
+Version: 3.18.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-3.17.0 → sqlframe-3.18.0}/sqlframe.egg-info/requires.txt

@@ -1,5 +1,5 @@
 prettytable<4
-sqlglot<26.4,>=24.0.0
+sqlglot<26.5,>=24.0.0
 typing_extensions
 
 [bigquery]
@@ -12,7 +12,7 @@ databricks-sql-connector<5,>=3.6
 [dev]
 duckdb<1.2,>=0.9
 findspark<3,>=2
-mypy<1.15,>=1.10.0
+mypy<1.16,>=1.10.0
 openai<2,>=1.30
 pandas-stubs<3,>=2
 pandas<3,>=2

{sqlframe-3.17.0 → sqlframe-3.18.0}/tests/integration/test_int_dataframe.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import typing as t
 
+import pandas as pd
 import pytest
 from _pytest.fixtures import FixtureRequest
 from pyspark.sql import DataFrame as PySparkDataFrame
@@ -29,6 +30,20 @@ def test_empty_df(
     compare_frames(df_empty, dfs_empty, no_empty=False)
 
 
+def test_dataframe_from_pandas(
+    pyspark_employee: PySparkDataFrame,
+    get_df: t.Callable[[str], BaseDataFrame],
+    compare_frames: t.Callable,
+):
+    employee = get_df("employee")
+    compare_frames(
+        pyspark_employee,
+        employee.session.createDataFrame(
+            pyspark_employee.toPandas(), schema=pyspark_employee.schema.simpleString()
+        ),
+    )
+
+
 def test_simple_select(
     pyspark_employee: PySparkDataFrame,
     get_df: t.Callable[[str], BaseDataFrame],
@@ -2414,3 +2429,25 @@ def test_filtering_join_key(
     ).filter(SF.col("store_id") > 1)
 
     compare_frames(df, dfs, compare_schema=False, sort=True)
+
+
+# https://github.com/eakmanrq/sqlframe/issues/281
+def test_create_column_after_join(
+    pyspark_employee: PySparkDataFrame,
+    pyspark_store: PySparkDataFrame,
+    get_df: t.Callable[[str], BaseDataFrame],
+    compare_frames: t.Callable,
+):
+    df = pyspark_employee.join(
+        pyspark_store,
+        on="store_id",
+    ).withColumn("new_col", F.lit(1))
+
+    employee = get_df("employee")
+    store = get_df("store")
+    dfs = employee.join(
+        store,
+        on="store_id",
+    ).withColumn("new_col", SF.lit(1))
+
+    compare_frames(df, dfs, compare_schema=False, sort=True)

{sqlframe-3.17.0 → sqlframe-3.18.0}/tests/unit/standalone/test_dataframe.py

@@ -148,3 +148,11 @@ def test_lineage(standalone_employee: StandaloneDataFrame):
         standalone_employee.session.sql("SELECT * FROM employee").lineage("age").source.sql()
         == "SELECT employee.age AS age FROM employee AS employee"
     )
+
+
+# Issue: https://github.com/eakmanrq/sqlframe/issues/286
+def test_unquoted_identifiers(standalone_employee: StandaloneDataFrame):
+    assert (
+        standalone_employee.sql(dialect="snowflake", pretty=False, quote_identifiers=False)
+        == "SELECT A1.EMPLOYEE_ID AS EMPLOYEE_ID, CAST(A1.FNAME AS TEXT) AS FNAME, CAST(A1.LNAME AS TEXT) AS LNAME, A1.AGE AS AGE, A1.STORE_ID AS STORE_ID FROM (VALUES (1, 'Jack', 'Shephard', 37, 1), (2, 'John', 'Locke', 65, 1), (3, 'Kate', 'Austen', 37, 2), (4, 'Claire', 'Littleton', 27, 2), (5, 'Hugo', 'Reyes', 29, 100)) AS A1(EMPLOYEE_ID, FNAME, LNAME, AGE, STORE_ID)"
+    )

{sqlframe-3.17.0 → sqlframe-3.18.0}/tests/unit/standalone/test_functions.py

@@ -1197,10 +1197,10 @@ def test_current_timestamp():
 @pytest.mark.parametrize(
     "expression, expected",
     [
-        (SF.date_format("cola", "MM/dd/yyy"), "DATE_FORMAT(CAST(cola AS TIMESTAMP), 'MM/dd/yyy')"),
+        (SF.date_format("cola", "MM/dd/yyy"), "DATE_FORMAT(cola, 'MM/dd/yyy')"),
         (
             SF.date_format(SF.col("cola"), "MM/dd/yyy"),
-            "DATE_FORMAT(CAST(cola AS TIMESTAMP), 'MM/dd/yyy')",
+            "DATE_FORMAT(cola, 'MM/dd/yyy')",
         ),
     ],
 )