sqlframe 2.1.0__tar.gz → 2.2.0__tar.gz

{sqlframe-2.1.0 → sqlframe-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 2.1.0
+Version: 2.2.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman
@@ -90,7 +90,7 @@ from sqlframe.bigquery import functions as F
 from sqlframe.bigquery import Window
 
 session = BigQuerySession()
-table_path = "bigquery-public-data.samples.natality"
+table_path = '"bigquery-public-data".samples.natality'
 # Top 5 years with the greatest year-over-year % change in new families with single child
 df = (
     session.table(table_path)

{sqlframe-2.1.0 → sqlframe-2.2.0}/README.md

@@ -60,7 +60,7 @@ from sqlframe.bigquery import functions as F
 from sqlframe.bigquery import Window
 
 session = BigQuerySession()
-table_path = "bigquery-public-data.samples.natality"
+table_path = '"bigquery-public-data".samples.natality'
 # Top 5 years with the greatest year-over-year % change in new families with single child
 df = (
     session.table(table_path)

{sqlframe-2.1.0 → sqlframe-2.2.0}/blogs/add_chatgpt_support.md

@@ -47,7 +47,7 @@ from sqlframe.bigquery import functions as F
 from sqlframe.bigquery import Window
 
 session = BigQuerySession()
-table_path = "bigquery-public-data.samples.natality"
+table_path = '"bigquery-public-data".samples.natality'
 # Top 5 years with the greatest year-over-year % change in new families with single child
 df = (
     session.table(table_path)

{sqlframe-2.1.0 → sqlframe-2.2.0}/blogs/sqlframe_universal_dataframe_api.md

@@ -39,7 +39,7 @@ from sqlframe.bigquery import Window
 
 # Unique to SQLFrame: Ability to connect directly to BigQuery
 session = BigQuerySession()
-table_path = "bigquery-public-data.samples.natality"
+table_path = '"bigquery-public-data".samples.natality'
 # Get the top 5 years with the greatest year-over-year % change in new families with a single child
 df = (
     session.table(table_path)

{sqlframe-2.1.0 → sqlframe-2.2.0}/docs/bigquery.md

@@ -72,7 +72,7 @@ from sqlframe.bigquery import functions as F
 
 session = BigQuerySession(default_dataset="sqlframe.db1")
 (
-    session.table("bigquery-public-data.samples.natality")
+    session.table('"bigquery-public-data".samples.natality')
     .select(F.call_function("FARM_FINGERPRINT", F.col("source")).alias("source_hash"))
     .show()
 )
@@ -86,7 +86,7 @@ from sqlframe.bigquery import functions as F
 from sqlframe.bigquery import Window
 
 session = BigQuerySession(default_dataset="sqlframe.db1")
-table_path = "bigquery-public-data.samples.natality"
+table_path = '"bigquery-public-data".samples.natality'
 # Get columns in the table
 print(session.catalog.listColumns(table_path))
 # Get the top 5 years with the greatest year-over-year % change in new families with a single child
@@ -214,6 +214,8 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [intersectAll](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.intersectAll.html)
 * [join](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.join.html)
 * [limit](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.limit.html)
+* lineage
+   * Get lineage for a specific column. [Returns a SQLGlot Node](https://sqlglot.com/sqlglot/lineage.html#Node). Can be used to get lineage SQL or HTML representation.
 * [na](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.na.html)
 * [orderBy](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.orderBy.html)
 * [persist](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.persist.html)

{sqlframe-2.1.0 → sqlframe-2.2.0}/docs/configuration.md

@@ -62,7 +62,7 @@ from sqlframe.bigquery import functions as F
 from sqlframe.bigquery import Window
 
 session = BigQuerySession()
-table_path = "bigquery-public-data.samples.natality"
+table_path = '"bigquery-public-data".samples.natality'
 # Top 5 years with the greatest year-over-year % change in new families with single child
 df = (
     session.table(table_path)

{sqlframe-2.1.0 → sqlframe-2.2.0}/docs/duckdb.md

@@ -187,6 +187,8 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [intersectAll](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.intersectAll.html)
 * [join](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.join.html)
 * [limit](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.limit.html)
+* lineage
+   * Get lineage for a specific column. [Returns a SQLGlot Node](https://sqlglot.com/sqlglot/lineage.html#Node). Can be used to get lineage SQL or HTML representation.
 * [na](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.na.html)
 * [orderBy](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.orderBy.html)
 * [persist](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.persist.html)

{sqlframe-2.1.0 → sqlframe-2.2.0}/docs/postgres.md

@@ -198,6 +198,8 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [intersectAll](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.intersectAll.html)
 * [join](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.join.html)
 * [limit](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.limit.html)
+* lineage
+   * Get lineage for a specific column. [Returns a SQLGlot Node](https://sqlglot.com/sqlglot/lineage.html#Node). Can be used to get lineage SQL or HTML representation.
 * [na](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.na.html)
 * [orderBy](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.orderBy.html)
 * [persist](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.persist.html)

{sqlframe-2.1.0 → sqlframe-2.2.0}/docs/snowflake.md

@@ -209,6 +209,8 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [intersectAll](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.intersectAll.html)
 * [join](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.join.html)
 * [limit](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.limit.html)
+* lineage
+   * Get lineage for a specific column. [Returns a SQLGlot Node](https://sqlglot.com/sqlglot/lineage.html#Node). Can be used to get lineage SQL or HTML representation.
 * [na](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.na.html)
 * [orderBy](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.orderBy.html)
 * [persist](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.persist.html)

{sqlframe-2.1.0 → sqlframe-2.2.0}/docs/spark.md

@@ -156,6 +156,8 @@ df.show(5)
 * [intersectAll](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.intersectAll.html)
 * [join](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.join.html)
 * [limit](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.limit.html)
+* lineage
+   * Get lineage for a specific column. [Returns a SQLGlot Node](https://sqlglot.com/sqlglot/lineage.html#Node). Can be used to get lineage SQL or HTML representation.
 * [na](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.na.html)
 * [orderBy](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.orderBy.html)
 * [persist](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.persist.html)

{sqlframe-2.1.0 → sqlframe-2.2.0}/docs/standalone.md

@@ -133,6 +133,8 @@ See something that you would like to see supported? [Open an issue](https://gith
 * [intersectAll](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.intersectAll.html)
 * [join](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.join.html)
 * [limit](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.limit.html)
+* lineage
+   * Get lineage for a specific column. [Returns a SQLGlot Node](https://sqlglot.com/sqlglot/lineage.html#Node). Can be used to get lineage SQL or HTML representation.
 * [na](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.na.html)
 * [orderBy](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.orderBy.html)
 * [persist](https://spark.apache.org/docs/latest/api/python/reference/pyspark.sql/api/pyspark.sql.DataFrame.persist.html)

{sqlframe-2.1.0 → sqlframe-2.2.0}/sqlframe/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2.1.0'
-__version_tuple__ = version_tuple = (2, 1, 0)
+__version__ = version = '2.2.0'
+__version_tuple__ = version_tuple = (2, 2, 0)

{sqlframe-2.1.0 → sqlframe-2.2.0}/sqlframe/base/dataframe.py

@@ -17,6 +17,7 @@ import sqlglot
 from prettytable import PrettyTable
 from sqlglot import Dialect
 from sqlglot import expressions as exp
+from sqlglot import lineage as sqlglot_lineage
 from sqlglot.helper import ensure_list, flatten, object_to_dict, seq_get
 from sqlglot.optimizer.pushdown_projections import pushdown_projections
 from sqlglot.optimizer.qualify import qualify
@@ -1613,6 +1614,13 @@ class _BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
                 0,
             )
 
+    def lineage(self, col: ColumnOrName, optimize: bool = True) -> sqlglot_lineage.Node:
+        return sqlglot_lineage.lineage(
+            column=self._ensure_and_normalize_col(col).alias_or_name,
+            sql=self._get_expressions(optimize=optimize)[0],
+            schema=self.session.catalog._schema,
+        )
+
     def toPandas(self) -> pd.DataFrame:
         return self.session._fetchdf(self._get_expressions(optimize=False))
 

{sqlframe-2.1.0 → sqlframe-2.2.0}/sqlframe.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 2.1.0
+Version: 2.2.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman
@@ -90,7 +90,7 @@ from sqlframe.bigquery import functions as F
 from sqlframe.bigquery import Window
 
 session = BigQuerySession()
-table_path = "bigquery-public-data.samples.natality"
+table_path = '"bigquery-public-data".samples.natality'
 # Top 5 years with the greatest year-over-year % change in new families with single child
 df = (
     session.table(table_path)

{sqlframe-2.1.0 → sqlframe-2.2.0}/tests/unit/standalone/test_dataframe.py

@@ -137,3 +137,14 @@ def test_expand_star_table_alias(standalone_employee: StandaloneDataFrame):
         standalone_employee.alias("blah").select("blah.*").sql(pretty=False, optimize=False)
         == "WITH `t51718876` AS (SELECT CAST(`employee_id` AS INT) AS `employee_id`, CAST(`fname` AS STRING) AS `fname`, CAST(`lname` AS STRING) AS `lname`, CAST(`age` AS INT) AS `age`, CAST(`store_id` AS INT) AS `store_id` FROM VALUES (1, 'Jack', 'Shephard', 37, 1), (2, 'John', 'Locke', 65, 1), (3, 'Kate', 'Austen', 37, 2), (4, 'Claire', 'Littleton', 27, 2), (5, 'Hugo', 'Reyes', 29, 100) AS `a1`(`employee_id`, `fname`, `lname`, `age`, `store_id`)), `t37842204` AS (SELECT `employee_id`, `fname`, `lname`, `age`, `store_id` FROM `t51718876`) SELECT `t37842204`.`employee_id`, `t37842204`.`fname`, `t37842204`.`lname`, `t37842204`.`age`, `t37842204`.`store_id` FROM `t37842204`"
     )
+
+
+def test_lineage(standalone_employee: StandaloneDataFrame):
+    assert (
+        standalone_employee.lineage("age").source.sql()
+        == "SELECT a1.age AS age FROM (VALUES (1, 'Jack', 'Shephard', 37, 1), (2, 'John', 'Locke', 65, 1), (3, 'Kate', 'Austen', 37, 2), (4, 'Claire', 'Littleton', 27, 2), (5, 'Hugo', 'Reyes', 29, 100)) AS a1(employee_id, fname, lname, age, store_id)"
+    )
+    assert (
+        standalone_employee.session.sql("SELECT * FROM employee").lineage("age").source.sql()
+        == "SELECT employee.age AS age FROM employee AS employee"
+    )