dagster-duckdb-pyspark 0.18.1__py3-none-any.whl → 0.18.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of dagster-duckdb-pyspark might be problematic. Click here for more details.

Files changed (8) hide show
  1. dagster_duckdb_pyspark/duckdb_pyspark_type_handler.py +5 -1
  2. dagster_duckdb_pyspark/version.py +1 -1
  3. {dagster_duckdb_pyspark-0.18.1.dist-info → dagster_duckdb_pyspark-0.18.3.dist-info}/METADATA +3 -6
  4. dagster_duckdb_pyspark-0.18.3.dist-info/RECORD +9 -0
  5. dagster_duckdb_pyspark-0.18.1.dist-info/RECORD +0 -9
  6. {dagster_duckdb_pyspark-0.18.1.dist-info → dagster_duckdb_pyspark-0.18.3.dist-info}/LICENSE +0 -0
  7. {dagster_duckdb_pyspark-0.18.1.dist-info → dagster_duckdb_pyspark-0.18.3.dist-info}/WHEEL +0 -0
  8. {dagster_duckdb_pyspark-0.18.1.dist-info → dagster_duckdb_pyspark-0.18.3.dist-info}/top_level.txt +0 -0
dagster_duckdb_pyspark/duckdb_pyspark_type_handler.py CHANGED
@@ -4,6 +4,7 @@ from dagster import InputContext, MetadataValue, OutputContext, TableColumn, Tab
4
4
  from dagster._core.storage.db_io_manager import DbTypeHandler, TableSlice
5
5
  from dagster_duckdb.io_manager import DuckDbClient, build_duckdb_io_manager
6
6
  from pyspark.sql import SparkSession
7
+ from pyspark.sql.types import StructType
7
8
 
8
9
 
9
10
  class DuckDBPySparkTypeHandler(DbTypeHandler[pyspark.sql.DataFrame]):
@@ -69,8 +70,11 @@ class DuckDBPySparkTypeHandler(DbTypeHandler[pyspark.sql.DataFrame]):
69
70
  self, context: InputContext, table_slice: TableSlice, connection
70
71
  ) -> pyspark.sql.DataFrame:
71
72
  """Loads the return of the query as the correct type."""
72
- pd_df = connection.execute(DuckDbClient.get_select_statement(table_slice)).fetchdf()
73
73
  spark = SparkSession.builder.getOrCreate()
74
+ if table_slice.partition_dimensions and len(context.asset_partition_keys) == 0:
75
+ return spark.createDataFrame([], StructType([]))
76
+
77
+ pd_df = connection.execute(DuckDbClient.get_select_statement(table_slice)).fetchdf()
74
78
  return spark.createDataFrame(pd_df)
75
79
 
76
80
  @property
dagster_duckdb_pyspark/version.py CHANGED
@@ -1 +1 @@
1
- __version__ = "0.18.1"
1
+ __version__ = "0.18.3"
{dagster_duckdb_pyspark-0.18.1.dist-info → dagster_duckdb_pyspark-0.18.3.dist-info}/METADATA RENAMED
@@ -1,12 +1,11 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dagster-duckdb-pyspark
3
- Version: 0.18.1
3
+ Version: 0.18.3
4
4
  Summary: Package for storing PySpark DataFrames in DuckDB.
5
5
  Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-duckb-pyspark
6
6
  Author: Elementl
7
7
  Author-email: hello@elementl.com
8
8
  License: Apache-2.0
9
- Platform: UNKNOWN
10
9
  Classifier: Programming Language :: Python :: 3.7
11
10
  Classifier: Programming Language :: Python :: 3.8
12
11
  Classifier: Programming Language :: Python :: 3.9
@@ -14,11 +13,9 @@ Classifier: Programming Language :: Python :: 3.10
14
13
  Classifier: License :: OSI Approved :: Apache Software License
15
14
  Classifier: Operating System :: OS Independent
16
15
  License-File: LICENSE
17
- Requires-Dist: dagster (==1.2.1)
18
- Requires-Dist: dagster-duckdb (==0.18.1)
16
+ Requires-Dist: dagster (==1.2.3)
17
+ Requires-Dist: dagster-duckdb (==0.18.3)
19
18
  Requires-Dist: pandas
20
19
  Requires-Dist: pyspark (>=2.0.2) ; python_version < "3.8"
21
20
  Requires-Dist: pyspark (>=3.0.0) ; python_version >= "3.8"
22
21
 
23
- UNKNOWN
24
-
dagster_duckdb_pyspark-0.18.3.dist-info/RECORD ADDED
@@ -0,0 +1,9 @@
1
+ dagster_duckdb_pyspark/__init__.py,sha256=EsnAP_X64pR76FL4a6RicGeG6H_vl4l9k-tb441KA4M,328
2
+ dagster_duckdb_pyspark/duckdb_pyspark_type_handler.py,sha256=LwM8_6CDRw8n6XZI4VFliPxKfChwlQRawfPTtPuWasY,4970
3
+ dagster_duckdb_pyspark/py.typed,sha256=mDShSrm8qg9qjacQc2F-rI8ATllqP6EdgHuEYxuCXZ0,7
4
+ dagster_duckdb_pyspark/version.py,sha256=gFZ2jBuhpo2tEeGzLqhbiYFDJrpaFW_VBCOIhFFdh2I,23
5
+ dagster_duckdb_pyspark-0.18.3.dist-info/LICENSE,sha256=-gtoVIAZYUHYmNHISZg982FI4Oh19mV1nxgTVW8eCB8,11344
6
+ dagster_duckdb_pyspark-0.18.3.dist-info/METADATA,sha256=I2hwpNnuY4k0-ewEyt9epoAP--nH0DYlKSvNYHyAQ6w,851
7
+ dagster_duckdb_pyspark-0.18.3.dist-info/WHEEL,sha256=p46_5Uhzqz6AzeSosiOnxK-zmFja1i22CrQCjmYe8ec,92
8
+ dagster_duckdb_pyspark-0.18.3.dist-info/top_level.txt,sha256=UYh0E2YiAlK01-DAkx0eikRaH-TIk0n9jijQK2joJBs,23
9
+ dagster_duckdb_pyspark-0.18.3.dist-info/RECORD,,
dagster_duckdb_pyspark-0.18.1.dist-info/RECORD DELETED
@@ -1,9 +0,0 @@
1
- dagster_duckdb_pyspark/__init__.py,sha256=EsnAP_X64pR76FL4a6RicGeG6H_vl4l9k-tb441KA4M,328
2
- dagster_duckdb_pyspark/duckdb_pyspark_type_handler.py,sha256=0RnS_b3JkpvIknBKefDybs7tEWscRG--XARAvsqYm0g,4779
3
- dagster_duckdb_pyspark/py.typed,sha256=mDShSrm8qg9qjacQc2F-rI8ATllqP6EdgHuEYxuCXZ0,7
4
- dagster_duckdb_pyspark/version.py,sha256=i8LHPc0qiYueFP9CAFrtufkGZfMfDAhFNqWW_Pa8HsI,23
5
- dagster_duckdb_pyspark-0.18.1.dist-info/LICENSE,sha256=-gtoVIAZYUHYmNHISZg982FI4Oh19mV1nxgTVW8eCB8,11344
6
- dagster_duckdb_pyspark-0.18.1.dist-info/METADATA,sha256=PQQlGey8Q3AzuXEffpvQq4BeBJAkD6Jbtg3n8v5bkRg,878
7
- dagster_duckdb_pyspark-0.18.1.dist-info/WHEEL,sha256=p46_5Uhzqz6AzeSosiOnxK-zmFja1i22CrQCjmYe8ec,92
8
- dagster_duckdb_pyspark-0.18.1.dist-info/top_level.txt,sha256=UYh0E2YiAlK01-DAkx0eikRaH-TIk0n9jijQK2joJBs,23
9
- dagster_duckdb_pyspark-0.18.1.dist-info/RECORD,,