dagster-pyspark 0.25.5__py3-none-any.whl → 0.25.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dagster-pyspark might be problematic. Click here for more details.
- dagster_pyspark/resources.py +4 -4
- dagster_pyspark/version.py +1 -1
- {dagster_pyspark-0.25.5.dist-info → dagster_pyspark-0.25.6.dist-info}/METADATA +3 -3
- dagster_pyspark-0.25.6.dist-info/RECORD +11 -0
- dagster_pyspark-0.25.5.dist-info/RECORD +0 -11
- {dagster_pyspark-0.25.5.dist-info → dagster_pyspark-0.25.6.dist-info}/LICENSE +0 -0
- {dagster_pyspark-0.25.5.dist-info → dagster_pyspark-0.25.6.dist-info}/WHEEL +0 -0
- {dagster_pyspark-0.25.5.dist-info → dagster_pyspark-0.25.6.dist-info}/top_level.txt +0 -0
dagster_pyspark/resources.py
CHANGED
|
@@ -15,9 +15,9 @@ def spark_session_from_config(spark_conf=None):
|
|
|
15
15
|
builder = SparkSession.builder
|
|
16
16
|
flat = flatten_dict(spark_conf)
|
|
17
17
|
for key, value in flat:
|
|
18
|
-
builder = builder.config(key, value)
|
|
18
|
+
builder = builder.config(key, value) # pyright: ignore[reportAttributeAccessIssue]
|
|
19
19
|
|
|
20
|
-
return builder.getOrCreate()
|
|
20
|
+
return builder.getOrCreate() # pyright: ignore[reportAttributeAccessIssue]
|
|
21
21
|
|
|
22
22
|
|
|
23
23
|
class PySparkResource(ConfigurableResource):
|
|
@@ -132,12 +132,12 @@ class LazyPySparkResource(ConfigurableResource):
|
|
|
132
132
|
@property
|
|
133
133
|
def spark_session(self) -> Any:
|
|
134
134
|
self._init_session()
|
|
135
|
-
return self._spark_session
|
|
135
|
+
return check.not_none(self._spark_session)
|
|
136
136
|
|
|
137
137
|
@property
|
|
138
138
|
def spark_context(self) -> Any:
|
|
139
139
|
self._init_session()
|
|
140
|
-
return self._spark_session.sparkContext
|
|
140
|
+
return check.not_none(self._spark_session).sparkContext
|
|
141
141
|
|
|
142
142
|
|
|
143
143
|
@dagster_maintained_resource
|
dagster_pyspark/version.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
__version__ = "0.25.
|
|
1
|
+
__version__ = "0.25.6"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: dagster-pyspark
|
|
3
|
-
Version: 0.25.
|
|
3
|
+
Version: 0.25.6
|
|
4
4
|
Summary: Package for PySpark Dagster framework components.
|
|
5
5
|
Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/dagster-framework/pyspark
|
|
6
6
|
Author: Dagster Labs
|
|
@@ -12,7 +12,7 @@ Classifier: License :: OSI Approved :: Apache Software License
|
|
|
12
12
|
Classifier: Operating System :: OS Independent
|
|
13
13
|
Requires-Python: >=3.9,<3.13
|
|
14
14
|
License-File: LICENSE
|
|
15
|
-
Requires-Dist: dagster ==1.9.
|
|
16
|
-
Requires-Dist: dagster-spark ==0.25.
|
|
15
|
+
Requires-Dist: dagster ==1.9.6
|
|
16
|
+
Requires-Dist: dagster-spark ==0.25.6
|
|
17
17
|
Requires-Dist: pyspark >=3
|
|
18
18
|
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
dagster_pyspark/__init__.py,sha256=pJl31s-9MK0MiqZDusthq37tTvgAsCtINwFDQaZjYKg,489
|
|
2
|
+
dagster_pyspark/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
|
|
3
|
+
dagster_pyspark/resources.py,sha256=xDHMta4jw6tEhZagBUKZdenY_X_ZRpwUm7i7BHuvi2M,6079
|
|
4
|
+
dagster_pyspark/types.py,sha256=pSc9mTMkpWQbkzxc3DhstUtZyUx5dFGORaqnGzGqf2o,31132
|
|
5
|
+
dagster_pyspark/utils.py,sha256=3DKp8guR52IlUwfBbN-54aMp6qz1JiM9nH7YTVpdPJo,1204
|
|
6
|
+
dagster_pyspark/version.py,sha256=mgi0NAwnQWP7UdrUf2L9Xln12B8NoCQbqDnalCRZMpE,23
|
|
7
|
+
dagster_pyspark-0.25.6.dist-info/LICENSE,sha256=TMatHW4_G9ldRdodEAp-l2Xa2WvsdeOh60E3v1R2jis,11349
|
|
8
|
+
dagster_pyspark-0.25.6.dist-info/METADATA,sha256=uyra6ERccC4Aum2fBzwAh_hHhyuLZ9ov4xtS96QC3xE,657
|
|
9
|
+
dagster_pyspark-0.25.6.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
|
|
10
|
+
dagster_pyspark-0.25.6.dist-info/top_level.txt,sha256=GgRyO5yEM3TjS5GzHHCvW_aqA41ryZah_yAtNSeorUw,16
|
|
11
|
+
dagster_pyspark-0.25.6.dist-info/RECORD,,
|
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
dagster_pyspark/__init__.py,sha256=pJl31s-9MK0MiqZDusthq37tTvgAsCtINwFDQaZjYKg,489
|
|
2
|
-
dagster_pyspark/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
|
|
3
|
-
dagster_pyspark/resources.py,sha256=weaH7Spi5JeE25i0UJGcIyfaH_f2-Rl3mCPF7vBitQM,5953
|
|
4
|
-
dagster_pyspark/types.py,sha256=pSc9mTMkpWQbkzxc3DhstUtZyUx5dFGORaqnGzGqf2o,31132
|
|
5
|
-
dagster_pyspark/utils.py,sha256=3DKp8guR52IlUwfBbN-54aMp6qz1JiM9nH7YTVpdPJo,1204
|
|
6
|
-
dagster_pyspark/version.py,sha256=9zCh_8lz8S_e8JlXmwkSbkp_uQhvRgQSEOH1abFGMw8,23
|
|
7
|
-
dagster_pyspark-0.25.5.dist-info/LICENSE,sha256=TMatHW4_G9ldRdodEAp-l2Xa2WvsdeOh60E3v1R2jis,11349
|
|
8
|
-
dagster_pyspark-0.25.5.dist-info/METADATA,sha256=8h3yz7126CK7pFUJO9wHdSNsjN_Z0u94YhpN4bmhRS8,657
|
|
9
|
-
dagster_pyspark-0.25.5.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
|
|
10
|
-
dagster_pyspark-0.25.5.dist-info/top_level.txt,sha256=GgRyO5yEM3TjS5GzHHCvW_aqA41ryZah_yAtNSeorUw,16
|
|
11
|
-
dagster_pyspark-0.25.5.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|