dagster-duckdb-pyspark 0.18.1__tar.gz → 0.18.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of dagster-duckdb-pyspark might be problematic. Click here for more details.

Files changed (17) hide show
  1. {dagster-duckdb-pyspark-0.18.1/dagster_duckdb_pyspark.egg-info → dagster-duckdb-pyspark-0.18.3}/PKG-INFO +1 -5
  2. {dagster-duckdb-pyspark-0.18.1 → dagster-duckdb-pyspark-0.18.3}/dagster_duckdb_pyspark/duckdb_pyspark_type_handler.py +5 -1
  3. dagster-duckdb-pyspark-0.18.3/dagster_duckdb_pyspark/version.py +1 -0
  4. {dagster-duckdb-pyspark-0.18.1 → dagster-duckdb-pyspark-0.18.3/dagster_duckdb_pyspark.egg-info}/PKG-INFO +1 -5
  5. {dagster-duckdb-pyspark-0.18.1 → dagster-duckdb-pyspark-0.18.3}/dagster_duckdb_pyspark.egg-info/requires.txt +2 -2
  6. {dagster-duckdb-pyspark-0.18.1 → dagster-duckdb-pyspark-0.18.3}/setup.py +3 -3
  7. dagster-duckdb-pyspark-0.18.1/dagster_duckdb_pyspark/version.py +0 -1
  8. {dagster-duckdb-pyspark-0.18.1 → dagster-duckdb-pyspark-0.18.3}/LICENSE +0 -0
  9. {dagster-duckdb-pyspark-0.18.1 → dagster-duckdb-pyspark-0.18.3}/MANIFEST.in +0 -0
  10. {dagster-duckdb-pyspark-0.18.1 → dagster-duckdb-pyspark-0.18.3}/README.md +0 -0
  11. {dagster-duckdb-pyspark-0.18.1 → dagster-duckdb-pyspark-0.18.3}/dagster_duckdb_pyspark/__init__.py +0 -0
  12. {dagster-duckdb-pyspark-0.18.1 → dagster-duckdb-pyspark-0.18.3}/dagster_duckdb_pyspark/py.typed +0 -0
  13. {dagster-duckdb-pyspark-0.18.1 → dagster-duckdb-pyspark-0.18.3}/dagster_duckdb_pyspark.egg-info/SOURCES.txt +0 -0
  14. {dagster-duckdb-pyspark-0.18.1 → dagster-duckdb-pyspark-0.18.3}/dagster_duckdb_pyspark.egg-info/dependency_links.txt +0 -0
  15. {dagster-duckdb-pyspark-0.18.1 → dagster-duckdb-pyspark-0.18.3}/dagster_duckdb_pyspark.egg-info/not-zip-safe +0 -0
  16. {dagster-duckdb-pyspark-0.18.1 → dagster-duckdb-pyspark-0.18.3}/dagster_duckdb_pyspark.egg-info/top_level.txt +0 -0
  17. {dagster-duckdb-pyspark-0.18.1 → dagster-duckdb-pyspark-0.18.3}/setup.cfg +0 -0
{dagster-duckdb-pyspark-0.18.1/dagster_duckdb_pyspark.egg-info → dagster-duckdb-pyspark-0.18.3}/PKG-INFO RENAMED
@@ -1,12 +1,11 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dagster-duckdb-pyspark
3
- Version: 0.18.1
3
+ Version: 0.18.3
4
4
  Summary: Package for storing PySpark DataFrames in DuckDB.
5
5
  Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-duckb-pyspark
6
6
  Author: Elementl
7
7
  Author-email: hello@elementl.com
8
8
  License: Apache-2.0
9
- Platform: UNKNOWN
10
9
  Classifier: Programming Language :: Python :: 3.7
11
10
  Classifier: Programming Language :: Python :: 3.8
12
11
  Classifier: Programming Language :: Python :: 3.9
@@ -14,6 +13,3 @@ Classifier: Programming Language :: Python :: 3.10
14
13
  Classifier: License :: OSI Approved :: Apache Software License
15
14
  Classifier: Operating System :: OS Independent
16
15
  License-File: LICENSE
17
-
18
- UNKNOWN
19
-
{dagster-duckdb-pyspark-0.18.1 → dagster-duckdb-pyspark-0.18.3}/dagster_duckdb_pyspark/duckdb_pyspark_type_handler.py RENAMED
@@ -4,6 +4,7 @@ from dagster import InputContext, MetadataValue, OutputContext, TableColumn, Tab
4
4
  from dagster._core.storage.db_io_manager import DbTypeHandler, TableSlice
5
5
  from dagster_duckdb.io_manager import DuckDbClient, build_duckdb_io_manager
6
6
  from pyspark.sql import SparkSession
7
+ from pyspark.sql.types import StructType
7
8
 
8
9
 
9
10
  class DuckDBPySparkTypeHandler(DbTypeHandler[pyspark.sql.DataFrame]):
@@ -69,8 +70,11 @@ class DuckDBPySparkTypeHandler(DbTypeHandler[pyspark.sql.DataFrame]):
69
70
  self, context: InputContext, table_slice: TableSlice, connection
70
71
  ) -> pyspark.sql.DataFrame:
71
72
  """Loads the return of the query as the correct type."""
72
- pd_df = connection.execute(DuckDbClient.get_select_statement(table_slice)).fetchdf()
73
73
  spark = SparkSession.builder.getOrCreate()
74
+ if table_slice.partition_dimensions and len(context.asset_partition_keys) == 0:
75
+ return spark.createDataFrame([], StructType([]))
76
+
77
+ pd_df = connection.execute(DuckDbClient.get_select_statement(table_slice)).fetchdf()
74
78
  return spark.createDataFrame(pd_df)
75
79
 
76
80
  @property
dagster-duckdb-pyspark-0.18.3/dagster_duckdb_pyspark/version.py ADDED
@@ -0,0 +1 @@
1
+ __version__ = "0.18.3"
{dagster-duckdb-pyspark-0.18.1 → dagster-duckdb-pyspark-0.18.3/dagster_duckdb_pyspark.egg-info}/PKG-INFO RENAMED
@@ -1,12 +1,11 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dagster-duckdb-pyspark
3
- Version: 0.18.1
3
+ Version: 0.18.3
4
4
  Summary: Package for storing PySpark DataFrames in DuckDB.
5
5
  Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-duckb-pyspark
6
6
  Author: Elementl
7
7
  Author-email: hello@elementl.com
8
8
  License: Apache-2.0
9
- Platform: UNKNOWN
10
9
  Classifier: Programming Language :: Python :: 3.7
11
10
  Classifier: Programming Language :: Python :: 3.8
12
11
  Classifier: Programming Language :: Python :: 3.9
@@ -14,6 +13,3 @@ Classifier: Programming Language :: Python :: 3.10
14
13
  Classifier: License :: OSI Approved :: Apache Software License
15
14
  Classifier: Operating System :: OS Independent
16
15
  License-File: LICENSE
17
-
18
- UNKNOWN
19
-
{dagster-duckdb-pyspark-0.18.1 → dagster-duckdb-pyspark-0.18.3}/dagster_duckdb_pyspark.egg-info/requires.txt RENAMED
@@ -1,5 +1,5 @@
1
- dagster==1.2.1
2
- dagster-duckdb==0.18.1
1
+ dagster==1.2.3
2
+ dagster-duckdb==0.18.3
3
3
  pandas
4
4
 
5
5
  [:python_version < "3.8"]
{dagster-duckdb-pyspark-0.18.1 → dagster-duckdb-pyspark-0.18.3}/setup.py RENAMED
@@ -7,7 +7,7 @@ from setuptools import find_packages, setup
7
7
  def get_version() -> str:
8
8
  version: Dict[str, str] = {}
9
9
  with open(Path(__file__).parent / "dagster_duckdb_pyspark/version.py", encoding="utf8") as fp:
10
- exec(fp.read(), version) # pylint: disable=W0122
10
+ exec(fp.read(), version)
11
11
 
12
12
  return version["__version__"]
13
13
 
@@ -34,8 +34,8 @@ setup(
34
34
  packages=find_packages(exclude=["dagster_duckdb_pyspark_tests*"]),
35
35
  include_package_data=True,
36
36
  install_requires=[
37
- "dagster==1.2.1",
38
- "dagster-duckdb==0.18.1",
37
+ "dagster==1.2.3",
38
+ "dagster-duckdb==0.18.3",
39
39
  # Pyspark 2.x is incompatible with Python 3.8+
40
40
  'pyspark>=3.0.0; python_version >= "3.8"',
41
41
  'pyspark>=2.0.2; python_version < "3.8"',
dagster-duckdb-pyspark-0.18.1/dagster_duckdb_pyspark/version.py DELETED
@@ -1 +0,0 @@
1
- __version__ = "0.18.1"