sqlframe 1.1.3__py3-none-any.whl

sqlframe/spark/column.py

@@ -0,0 +1 @@
+from sqlframe.base.column import Column

sqlframe/spark/dataframe.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+import logging
+import sys
+import typing as t
+
+from sqlframe.base.dataframe import (
+    _BaseDataFrame,
+    _BaseDataFrameNaFunctions,
+    _BaseDataFrameStatFunctions,
+)
+from sqlframe.spark.group import SparkGroupedData
+
+if sys.version_info >= (3, 11):
+    from typing import Self
+else:
+    from typing_extensions import Self
+
+if t.TYPE_CHECKING:
+    from sqlframe.spark.readwriter import SparkDataFrameWriter
+    from sqlframe.spark.session import SparkSession
+
+
+logger = logging.getLogger(__name__)
+
+
+class SparkDataFrameNaFunctions(_BaseDataFrameNaFunctions["SparkDataFrame"]):
+    pass
+
+
+class SparkDataFrameStatFunctions(_BaseDataFrameStatFunctions["SparkDataFrame"]):
+    pass
+
+
+class SparkDataFrame(
+    _BaseDataFrame[
+        "SparkSession",
+        "SparkDataFrameWriter",
+        "SparkDataFrameNaFunctions",
+        "SparkDataFrameStatFunctions",
+        "SparkGroupedData",
+    ]
+):
+    _na = SparkDataFrameNaFunctions
+    _stat = SparkDataFrameStatFunctions
+    _group_data = SparkGroupedData
+
+    def cache(self) -> Self:
+        logger.warning("Spark does not support caching. Ignoring cache() call.")
+        return self
+
+    def persist(self) -> Self:
+        logger.warning("Spark does not support persist. Ignoring persist() call.")
+        return self

sqlframe/spark/functions.py

@@ -0,0 +1,22 @@
+import inspect
+import sys
+
+import sqlframe.base.functions  # noqa
+
+module = sys.modules["sqlframe.base.functions"]
+globals().update(
+    {
+        name: func
+        for name, func in inspect.getmembers(module, inspect.isfunction)
+        if hasattr(func, "unsupported_engines")
+        and "spark" not in func.unsupported_engines
+        and "*" not in func.unsupported_engines
+    }
+)
+
+
+from sqlframe.base.function_alternatives import (  # noqa
+    percentile_without_disc as percentile,
+    add_months_by_multiplication as add_months,
+    arrays_overlap_renamed as arrays_overlap,
+)

sqlframe/spark/group.py

@@ -0,0 +1,14 @@
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
+
+from __future__ import annotations
+
+import typing as t
+
+from sqlframe.base.group import _BaseGroupedData
+
+if t.TYPE_CHECKING:
+    from sqlframe.spark.dataframe import SparkDataFrame
+
+
+class SparkGroupedData(_BaseGroupedData["SparkDataFrame"]):
+    pass

sqlframe/spark/readwriter.py

@@ -0,0 +1,29 @@
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
+
+from __future__ import annotations
+
+import typing as t
+
+from sqlframe.base.mixins.readwriter_mixins import PandasLoaderMixin, PandasWriterMixin
+from sqlframe.base.readerwriter import (
+    _BaseDataFrameReader,
+    _BaseDataFrameWriter,
+)
+
+if t.TYPE_CHECKING:
+    from sqlframe.spark.dataframe import SparkDataFrame
+    from sqlframe.spark.session import SparkSession
+
+
+class SparkDataFrameReader(
+    PandasLoaderMixin["SparkSession", "SparkDataFrame"],
+    _BaseDataFrameReader["SparkSession", "SparkDataFrame"],
+):
+    pass
+
+
+class SparkDataFrameWriter(
+    PandasWriterMixin["SparkSession", "SparkDataFrame"],
+    _BaseDataFrameWriter["SparkSession", "SparkDataFrame"],
+):
+    pass

sqlframe/spark/session.py

@@ -0,0 +1,90 @@
+from __future__ import annotations
+
+import typing as t
+import warnings
+
+import pandas as pd
+from sqlglot import exp
+
+from sqlframe.base.session import _BaseSession
+from sqlframe.spark.catalog import SparkCatalog
+from sqlframe.spark.dataframe import SparkDataFrame
+from sqlframe.spark.readwriter import (
+    SparkDataFrameReader,
+    SparkDataFrameWriter,
+)
+from sqlframe.spark.types import Row
+
+
+class SparkSession(
+    _BaseSession[  # type: ignore
+        SparkCatalog,
+        SparkDataFrameReader,
+        SparkDataFrameWriter,
+        SparkDataFrame,
+        object,
+    ],
+):
+    _catalog = SparkCatalog
+    _reader = SparkDataFrameReader
+    _writer = SparkDataFrameWriter
+    _df = SparkDataFrame
+
+    def __init__(self, conn: t.Optional[t.Any] = None):
+        warnings.warn(
+            "SparkSession is still in active development. Functions may not work as expected."
+        )
+
+        from pyspark.sql.session import DataFrame, SparkSession
+
+        if not hasattr(self, "spark_session"):
+            super().__init__(conn)
+            self.spark_session = SparkSession.builder.getOrCreate()
+            self._last_df: t.Optional[DataFrame] = None
+
+    @property
+    def _conn(self) -> t.Any:
+        raise NotImplementedError()
+
+    @property
+    def _cur(self) -> t.Any:
+        raise NotImplementedError()
+
+    def _fetch_rows(
+        self, sql: t.Union[str, exp.Expression], *, quote_identifiers: bool = True
+    ) -> t.List[Row]:
+        self._execute(sql, quote_identifiers=quote_identifiers)
+        assert self._last_df is not None
+        return [Row(**row.asDict()) for row in self._last_df.collect()]
+
+    def _execute(
+        self, sql: t.Union[str, exp.Expression], *, quote_identifiers: bool = True
+    ) -> None:
+        self._last_df = self.spark_session.sql(
+            self._to_sql(sql, quote_identifiers=quote_identifiers)
+        )
+
+    def _fetchdf(
+        self, sql: t.Union[str, exp.Expression], *, quote_identifiers: bool = True
+    ) -> pd.DataFrame:
+        self._execute(sql, quote_identifiers=quote_identifiers)
+        assert self._last_df is not None
+        return self._last_df.toPandas()
+
+    @property
+    def _has_connection(self) -> bool:
+        return True
+
+    class Builder(_BaseSession.Builder):
+        DEFAULT_INPUT_DIALECT = "spark"
+        DEFAULT_OUTPUT_DIALECT = "spark"
+
+        @property
+        def session(self) -> SparkSession:
+            return SparkSession(**self._session_kwargs)
+
+        def getOrCreate(self) -> SparkSession:
+            self._set_session_properties()
+            return self.session
+
+    builder = Builder()

sqlframe/spark/types.py

@@ -0,0 +1 @@
+from sqlframe.base.types import *

sqlframe/spark/window.py

@@ -0,0 +1 @@
+from sqlframe.base.window import *

sqlframe/standalone/__init__.py

@@ -0,0 +1,26 @@
+from sqlframe.standalone.catalog import StandaloneCatalog
+from sqlframe.standalone.column import Column
+from sqlframe.standalone.dataframe import (
+    StandaloneDataFrame,
+    StandaloneDataFrameNaFunctions,
+)
+from sqlframe.standalone.group import StandaloneGroupedData
+from sqlframe.standalone.readwriter import (
+    StandaloneDataFrameReader,
+    StandaloneDataFrameWriter,
+)
+from sqlframe.standalone.session import StandaloneSession
+from sqlframe.standalone.window import Window, WindowSpec
+
+__all__ = [
+    "StandaloneCatalog",
+    "Column",
+    "StandaloneDataFrame",
+    "StandaloneDataFrameNaFunctions",
+    "StandaloneGroupedData",
+    "StandaloneDataFrameReader",
+    "StandaloneDataFrameWriter",
+    "StandaloneSession",
+    "Window",
+    "WindowSpec",
+]

sqlframe/standalone/catalog.py

@@ -0,0 +1,13 @@
+import typing as t
+
+from sqlframe.base.catalog import _BaseCatalog
+
+if t.TYPE_CHECKING:
+    from sqlframe.standalone.dataframe import StandaloneDataFrame
+    from sqlframe.standalone.session import StandaloneSession
+
+
+class StandaloneCatalog(_BaseCatalog["StandaloneSession", "StandaloneDataFrame"]):
+    """User-facing catalog API, accessible through `SparkSession.catalog`."""
+
+    pass

sqlframe/standalone/column.py

@@ -0,0 +1 @@
+from sqlframe.base.column import Column

sqlframe/standalone/dataframe.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+import typing as t
+
+from sqlframe.base.dataframe import (
+    _BaseDataFrame,
+    _BaseDataFrameNaFunctions,
+    _BaseDataFrameStatFunctions,
+)
+from sqlframe.standalone.group import StandaloneGroupedData
+
+if t.TYPE_CHECKING:
+    from sqlframe.standalone.readwriter import StandaloneDataFrameWriter
+    from sqlframe.standalone.session import StandaloneSession
+
+
+class StandaloneDataFrameNaFunctions(_BaseDataFrameNaFunctions["StandaloneDataFrame"]):
+    pass
+
+
+class StandaloneDataFrameStatFunctions(_BaseDataFrameStatFunctions["StandaloneDataFrame"]):
+    pass
+
+
+class StandaloneDataFrame(
+    _BaseDataFrame[
+        "StandaloneSession",
+        "StandaloneDataFrameWriter",
+        "StandaloneDataFrameNaFunctions",
+        "StandaloneDataFrameStatFunctions",
+        "StandaloneGroupedData",
+    ]
+):
+    _na = StandaloneDataFrameNaFunctions
+    _stat = StandaloneDataFrameStatFunctions
+    _group_data = StandaloneGroupedData

sqlframe/standalone/functions.py

@@ -0,0 +1 @@
+from sqlframe.base.functions import *

sqlframe/standalone/group.py

@@ -0,0 +1,14 @@
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
+
+from __future__ import annotations
+
+import typing as t
+
+from sqlframe.base.group import _BaseGroupedData
+
+if t.TYPE_CHECKING:
+    from sqlframe.standalone.dataframe import StandaloneDataFrame
+
+
+class StandaloneGroupedData(_BaseGroupedData["StandaloneDataFrame"]):
+    pass

sqlframe/standalone/readwriter.py

@@ -0,0 +1,19 @@
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
+
+from __future__ import annotations
+
+import typing as t
+
+from sqlframe.base.readerwriter import _BaseDataFrameReader, _BaseDataFrameWriter
+
+if t.TYPE_CHECKING:
+    from sqlframe.standalone.dataframe import StandaloneDataFrame
+    from sqlframe.standalone.session import StandaloneSession
+
+
+class StandaloneDataFrameReader(_BaseDataFrameReader["StandaloneSession", "StandaloneDataFrame"]):
+    pass
+
+
+class StandaloneDataFrameWriter(_BaseDataFrameWriter["StandaloneSession", "StandaloneDataFrame"]):
+    pass

sqlframe/standalone/session.py

@@ -0,0 +1,40 @@
+# This code is based on code from Apache Spark under the license found in the LICENSE file located in the 'sqlframe' folder.
+
+from __future__ import annotations
+
+from sqlframe.base.session import _BaseSession
+from sqlframe.standalone.catalog import StandaloneCatalog
+from sqlframe.standalone.dataframe import StandaloneDataFrame
+from sqlframe.standalone.readwriter import (
+    StandaloneDataFrameReader,
+    StandaloneDataFrameWriter,
+)
+
+
+class StandaloneSession(
+    _BaseSession[  # type: ignore
+        StandaloneCatalog,
+        StandaloneDataFrameReader,
+        StandaloneDataFrameWriter,
+        StandaloneDataFrame,
+        object,
+    ]
+):  # type: ignore
+    _catalog = StandaloneCatalog
+    _reader = StandaloneDataFrameReader
+    _writer = StandaloneDataFrameWriter
+    _df = StandaloneDataFrame
+
+    class Builder(_BaseSession.Builder):
+        DEFAULT_INPUT_DIALECT = "spark"
+        DEFAULT_OUTPUT_DIALECT = "spark"
+
+        @property
+        def session(self) -> StandaloneSession:
+            return StandaloneSession()
+
+        def getOrCreate(self) -> StandaloneSession:
+            self._set_session_properties()
+            return self.session
+
+    builder = Builder()

sqlframe/standalone/types.py

@@ -0,0 +1 @@
+from sqlframe.base.types import *

sqlframe/standalone/window.py

@@ -0,0 +1 @@
+from sqlframe.base.window import *

sqlframe-1.1.3.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Ryan Eakman
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

sqlframe-1.1.3.dist-info/METADATA

@@ -0,0 +1,172 @@
+Metadata-Version: 2.1
+Name: sqlframe
+Version: 1.1.3
+Summary: Taking the Spark out of PySpark by converting to SQL
+Home-page: https://github.com/eakmanrq/sqlframe
+Author: Ryan Eakman
+Author-email: eakmanrq@gmail.com
+License: MIT
+Platform: UNKNOWN
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: SQL
+Classifier: Programming Language :: Python :: 3 :: Only
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: prettytable (<3.11.0)
+Requires-Dist: sqlglot (<24.1,>=24.0.0)
+Provides-Extra: bigquery
+Requires-Dist: google-cloud-bigquery-storage (<3,>=2) ; extra == 'bigquery'
+Requires-Dist: google-cloud-bigquery[pandas] (<4,>=3) ; extra == 'bigquery'
+Requires-Dist: pandas (<3,>=2) ; extra == 'bigquery'
+Provides-Extra: dev
+Requires-Dist: duckdb (<0.11,>=0.9) ; extra == 'dev'
+Requires-Dist: mypy (<1.11,>=1.10.0) ; extra == 'dev'
+Requires-Dist: pandas-stubs (<3,>=2) ; extra == 'dev'
+Requires-Dist: pandas (<3,>=2) ; extra == 'dev'
+Requires-Dist: psycopg (<4,>=3.1) ; extra == 'dev'
+Requires-Dist: pyarrow (<17,>=10) ; extra == 'dev'
+Requires-Dist: pyspark (<3.6,>=2) ; extra == 'dev'
+Requires-Dist: pytest-postgresql (<7,>=6) ; extra == 'dev'
+Requires-Dist: pytest-xdist (<3.7,>=3.6) ; extra == 'dev'
+Requires-Dist: pytest (<8.3,>=8.2.0) ; extra == 'dev'
+Requires-Dist: ruff (<0.5,>=0.4.4) ; extra == 'dev'
+Requires-Dist: types-psycopg2 (<3,>=2.9) ; extra == 'dev'
+Requires-Dist: typing-extensions (<5,>=4.11) ; extra == 'dev'
+Requires-Dist: pre-commit (>=3.5) ; (python_version == "3.8") and extra == 'dev'
+Requires-Dist: pre-commit (<3.8,>=3.7) ; (python_version >= "3.9") and extra == 'dev'
+Provides-Extra: docs
+Requires-Dist: mkdocs-include-markdown-plugin (==6.0.6) ; extra == 'docs'
+Requires-Dist: mkdocs-material-extensions (==1.1.1) ; extra == 'docs'
+Requires-Dist: mkdocs-material (==9.0.5) ; extra == 'docs'
+Requires-Dist: mkdocs (==1.4.2) ; extra == 'docs'
+Requires-Dist: pymdown-extensions ; extra == 'docs'
+Provides-Extra: duckdb
+Requires-Dist: duckdb (<0.11,>=0.9) ; extra == 'duckdb'
+Requires-Dist: pandas (<3,>=2) ; extra == 'duckdb'
+Provides-Extra: postgres
+Requires-Dist: pandas (<3,>=2) ; extra == 'postgres'
+Requires-Dist: psycopg2 (<3,>=2.8) ; extra == 'postgres'
+Provides-Extra: redshift
+Requires-Dist: pandas (<3,>=2) ; extra == 'redshift'
+Requires-Dist: redshift-connector (<2.2.0,>=2.1.1) ; extra == 'redshift'
+Provides-Extra: snowflake
+Requires-Dist: pandas (<3,>=2) ; extra == 'snowflake'
+Requires-Dist: snowflake-connector-python[pandas,secure-local-storage] (<3.11,>=3.10.0) ; extra == 'snowflake'
+Provides-Extra: spark
+Requires-Dist: pyspark (<3.6,>=2) ; extra == 'spark'
+
+<div align="center">
+  <img src="https://sqlframe.readthedocs.io/en/latest/docs/images/sqlframe_logo.png" alt="SQLFrame Logo" width="400"/>
+</div>
+
+SQLFrame implements the PySpark DataFrame API in order to enable running transformation pipelines directly on database engines - no Spark clusters or dependencies required.
+
+SQLFrame currently supports the following engines (many more in development):
+
+* [BigQuery](https://sqlframe.readthedocs.io/en/latest/bigquery/)
+* [DuckDB](https://sqlframe.readthedocs.io/en/latest/duckdb)
+* [Postgres](https://sqlframe.readthedocs.io/en/latest/postgres)
+
+SQLFrame also has a "Standalone" session that be used to generate SQL without any connection to a database engine.
+* [Standalone](https://sqlframe.readthedocs.io/en/latest/standalone)
+
+SQLFrame is great for:
+
+* Users who want to run PySpark DataFrame code without having to use a Spark cluster
+* Users who want a SQL representation of their DataFrame code for debugging or sharing with others
+* Users who want a DataFrame API that leverages the full power of their engine to do the processing 
+
+## Installation
+
+```bash
+# BigQuery
+pip install "sqlframe[bigquery]"
+# DuckDB
+pip install "sqlframe[duckdb]"
+# Postgres
+pip install "sqlframe[postgres]"
+# Standalone
+pip install sqlframe
+```
+
+See specific engine documentation for additional setup instructions.
+
+## Example Usage
+
+```python
+from sqlframe.bigquery import BigQuerySession
+from sqlframe.bigquery import functions as F
+from sqlframe.bigquery import Window
+
+session = BigQuerySession()
+table_path = "bigquery-public-data.samples.natality"
+# Top 5 years with the greatest year-over-year % change in new families with single child
+df = (
+    session.table(table_path)
+    .where(F.col("ever_born") == 1)
+    .groupBy("year")
+    .agg(F.count("*").alias("num_single_child_families"))
+    .withColumn(
+        "last_year_num_single_child_families", 
+        F.lag(F.col("num_single_child_families"), 1).over(Window.orderBy("year"))
+    )
+    .withColumn(
+        "percent_change", 
+        (F.col("num_single_child_families") - F.col("last_year_num_single_child_families")) 
+        / F.col("last_year_num_single_child_families")
+    )
+    .orderBy(F.abs(F.col("percent_change")).desc())
+    .select(
+        F.col("year").alias("year"),
+        F.format_number("num_single_child_families", 0).alias("new families single child"),
+        F.format_number(F.col("percent_change") * 100, 2).alias("percent change"),
+    )
+    .limit(5)
+)
+```
+```python
+>>> df.sql()
+WITH `t94228` AS (
+  SELECT
+    `natality`.`year` AS `year`,
+    COUNT(*) AS `num_single_child_families`
+  FROM `bigquery-public-data`.`samples`.`natality` AS `natality`
+  WHERE
+    `natality`.`ever_born` = 1
+  GROUP BY
+    `natality`.`year`
+), `t39093` AS (
+  SELECT
+    `t94228`.`year` AS `year`,
+    `t94228`.`num_single_child_families` AS `num_single_child_families`,
+    LAG(`t94228`.`num_single_child_families`, 1) OVER (ORDER BY `t94228`.`year`) AS `last_year_num_single_child_families`
+  FROM `t94228` AS `t94228`
+)
+SELECT
+  `t39093`.`year` AS `year`,
+  FORMAT('%\'.0f', ROUND(CAST(`t39093`.`num_single_child_families` AS FLOAT64), 0)) AS `new families single child`,
+  FORMAT('%\'.2f', ROUND(CAST((((`t39093`.`num_single_child_families` - `t39093`.`last_year_num_single_child_families`) / `t39093`.`last_year_num_single_child_families`) * 100) AS FLOAT64), 2)) AS `percent change`
+FROM `t39093` AS `t39093`
+ORDER BY
+  ABS(`percent_change`) DESC
+LIMIT 5
+```
+```python
+>>> df.show()
++------+---------------------------+----------------+
+| year | new families single child | percent change |
++------+---------------------------+----------------+
+| 1989 |         1,650,246         |     25.02      |
+| 1974 |          783,448          |     14.49      |
+| 1977 |         1,057,379         |     11.38      |
+| 1985 |         1,308,476         |     11.15      |
+| 1975 |          868,985          |     10.92      |
++------+---------------------------+----------------+
+```
+
+