dagster-pyspark 0.20.17__py3-none-any.whl → 0.27.16__py3-none-any.whl

dagster_pyspark/__init__.py

@@ -1,15 +1,20 @@
-from dagster._core.libraries import DagsterLibraryRegistry
+from dagster_shared.libraries import DagsterLibraryRegistry
 
-from .resources import LazyPySparkResource, PySparkResource, lazy_pyspark_resource, pyspark_resource
-from .types import DataFrame
-from .version import __version__
+from dagster_pyspark.resources import (
+    LazyPySparkResource,
+    PySparkResource,
+    lazy_pyspark_resource,
+    pyspark_resource,
+)
+from dagster_pyspark.types import DataFrame
+from dagster_pyspark.version import __version__
 
 DagsterLibraryRegistry.register("dagster-pyspark", __version__)
 
 __all__ = [
     "DataFrame",
-    "pyspark_resource",
-    "lazy_pyspark_resource",
-    "PySparkResource",
     "LazyPySparkResource",
+    "PySparkResource",
+    "lazy_pyspark_resource",
+    "pyspark_resource",
 ]

dagster_pyspark/py.typed

@@ -0,0 +1 @@
+partial

dagster_pyspark/resources.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict
+from typing import Any
 
 import dagster._check as check
 from dagster import ConfigurableResource, resource
@@ -15,9 +15,9 @@ def spark_session_from_config(spark_conf=None):
     builder = SparkSession.builder
     flat = flatten_dict(spark_conf)
     for key, value in flat:
-        builder = builder.config(key, value)
+        builder = builder.config(key, value)  # pyright: ignore[reportAttributeAccessIssue]
 
-    return builder.getOrCreate()
+    return builder.getOrCreate()  # pyright: ignore[reportAttributeAccessIssue]
 
 
 class PySparkResource(ConfigurableResource):
@@ -45,7 +45,7 @@ class PySparkResource(ConfigurableResource):
                 my_op()
     """
 
-    spark_config: Dict[str, Any]
+    spark_config: dict[str, Any]
     _spark_session = PrivateAttr(default=None)
 
     @classmethod
@@ -118,7 +118,7 @@ class LazyPySparkResource(ConfigurableResource):
                 my_op()
     """
 
-    spark_config: Dict[str, Any]
+    spark_config: dict[str, Any]
     _spark_session = PrivateAttr(default=None)
 
     @classmethod
@@ -132,12 +132,12 @@ class LazyPySparkResource(ConfigurableResource):
     @property
     def spark_session(self) -> Any:
         self._init_session()
-        return self._spark_session
+        return check.not_none(self._spark_session)
 
     @property
     def spark_context(self) -> Any:
         self._init_session()
-        return self._spark_session.sparkContext
+        return check.not_none(self._spark_session).sparkContext
 
 
 @dagster_maintained_resource

dagster_pyspark/utils.py

@@ -1,11 +1,29 @@
 import os
+import re
 import zipfile
 
 import dagster._check as check
 
+DEFAULT_EXCLUDE = [
+    r".*pytest.*",
+    r".*__pycache__.*",
+    r".*pyc$",
+    r".*\/venv\/.*",
+    r".*\.egg-info$",
+    r".*\/logs\/.*",
+]
 
-def build_pyspark_zip(zip_file, path):
-    """Archives the current path into a file named `zip_file`."""
+
+def build_pyspark_zip(zip_file, path, exclude=DEFAULT_EXCLUDE) -> None:
+    """Archives the current path into a file named `zip_file`.
+
+    Args:
+        zip_file (str): The name of the zip file to create.
+        path (str): The path to archive.
+        exclude (Optional[List[str]]): A list of regular expression patterns to exclude paths from
+            the archive. Regular expressions will be matched against the absolute filepath with
+            `re.search`.
+    """
     check.str_param(zip_file, "zip_file")
     check.str_param(path, "path")
 
@@ -15,7 +33,7 @@ def build_pyspark_zip(zip_file, path):
                 abs_fname = os.path.join(root, fname)
 
                 # Skip various artifacts
-                if "pytest" in abs_fname or "__pycache__" in abs_fname or "pyc" in abs_fname:
+                if any([re.search(pattern, abs_fname) for pattern in exclude]):
                     continue
 
                 zf.write(abs_fname, os.path.relpath(os.path.join(root, fname), path))

dagster_pyspark/version.py

@@ -1 +1 @@
-__version__ = "0.20.17"
+__version__ = "0.27.16"

{dagster_pyspark-0.20.17.dist-info → dagster_pyspark-0.27.16.dist-info}/METADATA

@@ -1,18 +1,28 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: dagster-pyspark
-Version: 0.20.17
+Version: 0.27.16
 Summary: Package for PySpark Dagster framework components.
 Home-page: https://github.com/dagster-io/dagster/tree/master/python_modules/dagster-framework/pyspark
 Author: Dagster Labs
 Author-email: hello@dagsterlabs.com
 License: Apache-2.0
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9,<3.14
 License-File: LICENSE
-Requires-Dist: dagster (==1.4.17)
-Requires-Dist: dagster-spark (==0.20.17)
-Requires-Dist: pyspark (>=3)
-
+Requires-Dist: dagster==1.11.16
+Requires-Dist: dagster_spark==0.27.16
+Requires-Dist: pyspark<5,>=3
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: home-page
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

dagster_pyspark-0.27.16.dist-info/RECORD

@@ -0,0 +1,11 @@
+dagster_pyspark/__init__.py,sha256=peKeESvPFAqt8aeyT4Px_SL-rALE3Yr2cNxG7cdng5Q,490
+dagster_pyspark/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
+dagster_pyspark/resources.py,sha256=hXkBHQMgzUyHmZN8u4UIYLwpuI4lXmB1UVg1Z5XCDhs,6073
+dagster_pyspark/types.py,sha256=pSc9mTMkpWQbkzxc3DhstUtZyUx5dFGORaqnGzGqf2o,31132
+dagster_pyspark/utils.py,sha256=3DKp8guR52IlUwfBbN-54aMp6qz1JiM9nH7YTVpdPJo,1204
+dagster_pyspark/version.py,sha256=gNIskoVbgqFeY6KiyOzkhfP7GLwnNVmVeil7tBvkL44,24
+dagster_pyspark-0.27.16.dist-info/licenses/LICENSE,sha256=4lsMW-RCvfVD4_F57wrmpe3vX1xwUk_OAKKmV_XT7Z0,11348
+dagster_pyspark-0.27.16.dist-info/METADATA,sha256=F9zdsnU-d0KOYrOiJF5sw0kSvNiC7CX3ZrU2DX_Er_4,943
+dagster_pyspark-0.27.16.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dagster_pyspark-0.27.16.dist-info/top_level.txt,sha256=GgRyO5yEM3TjS5GzHHCvW_aqA41ryZah_yAtNSeorUw,16
+dagster_pyspark-0.27.16.dist-info/RECORD,,

{dagster_pyspark-0.20.17.dist-info → dagster_pyspark-0.27.16.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.33.6)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{dagster_pyspark-0.20.17.dist-info → dagster_pyspark-0.27.16.dist-info/licenses}/LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright 2023 Dagster Labs, Inc".
+   Copyright 2025 Dagster Labs, Inc.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

dagster_pyspark-0.20.17.dist-info/RECORD

@@ -1,10 +0,0 @@
-dagster_pyspark/__init__.py,sha256=Jf745Ecqp5duFzOoFvnitmmj14s3TbQqk6gA126BLEE,423
-dagster_pyspark/resources.py,sha256=weaH7Spi5JeE25i0UJGcIyfaH_f2-Rl3mCPF7vBitQM,5953
-dagster_pyspark/types.py,sha256=pSc9mTMkpWQbkzxc3DhstUtZyUx5dFGORaqnGzGqf2o,31132
-dagster_pyspark/utils.py,sha256=RmhaVR9S_QhnPffBIuwlZBSO-0Sb2RcRLemk3VBrxDc,693
-dagster_pyspark/version.py,sha256=eM75ih3rRMeRs2KAFT1C-wtFC_fhUYsPhS2X2Aa8ILk,24
-dagster_pyspark-0.20.17.dist-info/LICENSE,sha256=TMatHW4_G9ldRdodEAp-l2Xa2WvsdeOh60E3v1R2jis,11349
-dagster_pyspark-0.20.17.dist-info/METADATA,sha256=58Y-MKz48BUeo4Qx27BwjeR7m9kxZVJ4QbJzpgC4Q6g,687
-dagster_pyspark-0.20.17.dist-info/WHEEL,sha256=p46_5Uhzqz6AzeSosiOnxK-zmFja1i22CrQCjmYe8ec,92
-dagster_pyspark-0.20.17.dist-info/top_level.txt,sha256=GgRyO5yEM3TjS5GzHHCvW_aqA41ryZah_yAtNSeorUw,16
-dagster_pyspark-0.20.17.dist-info/RECORD,,