dagster-pandas 0.13.18rc0__py3-none-any.whl → 0.27.15__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dagster_pandas/__init__.py +13 -13
- dagster_pandas/constraints.py +144 -181
- dagster_pandas/data_frame.py +82 -129
- dagster_pandas/examples/__init__.py +9 -9
- dagster_pandas/examples/pandas_hello_world/environments/pandas_hello_world_prod.yaml +1 -1
- dagster_pandas/examples/pandas_hello_world/environments/pandas_hello_world_test.yaml +1 -1
- dagster_pandas/examples/pandas_hello_world/environments/papermill_pandas_hello_world_prod.yaml +1 -1
- dagster_pandas/examples/pandas_hello_world/environments/papermill_pandas_hello_world_test.yaml +1 -1
- dagster_pandas/examples/pandas_hello_world/ops.py +7 -7
- dagster_pandas/examples/pandas_hello_world.yaml +1 -1
- dagster_pandas/py.typed +1 -0
- dagster_pandas/validation.py +26 -31
- dagster_pandas/version.py +1 -1
- dagster_pandas-0.27.15.dist-info/METADATA +36 -0
- dagster_pandas-0.27.15.dist-info/RECORD +22 -0
- {dagster_pandas-0.13.18rc0.dist-info → dagster_pandas-0.27.15.dist-info}/WHEEL +1 -1
- {dagster_pandas-0.13.18rc0.dist-info → dagster_pandas-0.27.15.dist-info/licenses}/LICENSE +1 -1
- dagster_pandas-0.27.15.dist-info/top_level.txt +1 -0
- dagster_pandas-0.13.18rc0.dist-info/METADATA +0 -24
- dagster_pandas-0.13.18rc0.dist-info/RECORD +0 -23
- dagster_pandas-0.13.18rc0.dist-info/top_level.txt +0 -2
- dagster_pandas_tests/pandas_hello_world/__init__.py +0 -0
- dagster_pandas_tests/pandas_hello_world/test_pandas_hello_world.py +0 -93
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
dagster_pandas/__init__.py,sha256=Mfg1pqLMP8Zud0kTpqKFTLPsdYX7dU1v6-JOiICVMpQ,1521
|
|
2
|
+
dagster_pandas/constraints.py,sha256=rP_S-Ap8lfZpj8Utrx9vRLRCZSoeHeS-gMSvui0YnXY,47734
|
|
3
|
+
dagster_pandas/data_frame.py,sha256=NoB-Laxz-Wx6dVKgCqQhPJmo9kN-DVcHyiMmHwXAk14,11097
|
|
4
|
+
dagster_pandas/py.typed,sha256=la67KBlbjXN-_-DfGNcdOcjYumVpKG_Tkw-8n5dnGB4,8
|
|
5
|
+
dagster_pandas/validation.py,sha256=oPODshNhi5o-q1Lxb9yxZPFIvqVrpzZTdiWBHiMabqU,19350
|
|
6
|
+
dagster_pandas/version.py,sha256=nh4rkWKwcS4_2NEvEKWozrztgCUEn_1ODIolwNpGd8g,24
|
|
7
|
+
dagster_pandas/examples/__init__.py,sha256=A0ecEVe4z9X1yGD9yfrKIDXq2zHhTgqAjqkLV6POPjg,2133
|
|
8
|
+
dagster_pandas/examples/pandas_hello_world.yaml,sha256=9CNhtFRuutBPsBpAsaTAncMwBnLzzN9H-ZkU_DaxtEw,82
|
|
9
|
+
dagster_pandas/examples/workspace.yaml,sha256=67d6PErFNq5l7DlRSFTGUwLUwFP1QTd8f8TOCGmbPnU,73
|
|
10
|
+
dagster_pandas/examples/pandas_hello_world/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
11
|
+
dagster_pandas/examples/pandas_hello_world/env.yaml,sha256=pk7REnOLuw5KoyjUhdBdUOXQbYu182Jz74y9J4CIkqE,54
|
|
12
|
+
dagster_pandas/examples/pandas_hello_world/ops.py,sha256=ds9StT4uT0C5AEIAH2OB8CdIdmDY8CNkzQYDeIurSk0,1093
|
|
13
|
+
dagster_pandas/examples/pandas_hello_world/solids.yaml,sha256=43Qv-8OLGsSsWPZ1DPIwdkl1lwLafJe49yLhspPCJmg,118
|
|
14
|
+
dagster_pandas/examples/pandas_hello_world/environments/pandas_hello_world_prod.yaml,sha256=9TAZL3-HA_6KqUkS8QWxAiqyQoBQjDAZdCH0Ee35PRs,87
|
|
15
|
+
dagster_pandas/examples/pandas_hello_world/environments/pandas_hello_world_test.yaml,sha256=9CNhtFRuutBPsBpAsaTAncMwBnLzzN9H-ZkU_DaxtEw,82
|
|
16
|
+
dagster_pandas/examples/pandas_hello_world/environments/papermill_pandas_hello_world_prod.yaml,sha256=C6WtgRQBtvNUk-uheCPA-PhBxxRh46hP6Y_cD8xr2BE,108
|
|
17
|
+
dagster_pandas/examples/pandas_hello_world/environments/papermill_pandas_hello_world_test.yaml,sha256=LWTGTo1qv_Ph0tqefoXv7x62PD6L-8Hm4GxSeMW6NGk,103
|
|
18
|
+
dagster_pandas-0.27.15.dist-info/licenses/LICENSE,sha256=4lsMW-RCvfVD4_F57wrmpe3vX1xwUk_OAKKmV_XT7Z0,11348
|
|
19
|
+
dagster_pandas-0.27.15.dist-info/METADATA,sha256=0WYFsfcZc5rbXt7sfCMzKw55WOZ528zOTtySHHGInNA,1190
|
|
20
|
+
dagster_pandas-0.27.15.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
21
|
+
dagster_pandas-0.27.15.dist-info/top_level.txt,sha256=pbqh1vicDL3g55vtJyxAgEshkdrTsnt_WeUPk8Uemq8,15
|
|
22
|
+
dagster_pandas-0.27.15.dist-info/RECORD,,
|
|
@@ -186,7 +186,7 @@
|
|
|
186
186
|
same "printed page" as the copyright notice for easier
|
|
187
187
|
identification within third-party archives.
|
|
188
188
|
|
|
189
|
-
Copyright
|
|
189
|
+
Copyright 2025 Dagster Labs, Inc.
|
|
190
190
|
|
|
191
191
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
192
192
|
you may not use this file except in compliance with the License.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
dagster_pandas
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.1
|
|
2
|
-
Name: dagster-pandas
|
|
3
|
-
Version: 0.13.18rc0
|
|
4
|
-
Summary: Utilities and examples for working with pandas and dagster, an opinionated framework for expressing data pipelines
|
|
5
|
-
Home-page: https://github.com/dagster-io/dagster
|
|
6
|
-
Author: Elementl
|
|
7
|
-
Author-email: hello@elementl.com
|
|
8
|
-
License: Apache-2.0
|
|
9
|
-
Platform: UNKNOWN
|
|
10
|
-
Classifier: Programming Language :: Python :: 3.6
|
|
11
|
-
Classifier: Programming Language :: Python :: 3.7
|
|
12
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
13
|
-
Classifier: License :: OSI Approved :: Apache Software License
|
|
14
|
-
Classifier: Operating System :: OS Independent
|
|
15
|
-
Description-Content-Type: text/markdown
|
|
16
|
-
Requires-Dist: dagster (==0.13.18rc0)
|
|
17
|
-
Requires-Dist: pandas
|
|
18
|
-
|
|
19
|
-
# dagster-pandas
|
|
20
|
-
|
|
21
|
-
The docs for `dagster-pandas` can be found
|
|
22
|
-
[here](https://docs.dagster.io/_apidocs/libraries/dagster-pandas).
|
|
23
|
-
|
|
24
|
-
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
dagster_pandas/__init__.py,sha256=9XhXDj-p3BRPPSggTF7G6LfJfGd52XjP2apYOTBJw2g,1464
|
|
2
|
-
dagster_pandas/constraints.py,sha256=Q7Lr6Yr4_at8C4biyvptIw8Qxk7zANfxx3arFRJkcSA,49138
|
|
3
|
-
dagster_pandas/data_frame.py,sha256=7_AP7quOEB-eufR3PClfTn7pNpQnxPy0OHRlfe_RYYU,13371
|
|
4
|
-
dagster_pandas/validation.py,sha256=xjarvC3YxpJ-48EV_ouOJnDLtAoXFDCCSgKvgCftAxY,19426
|
|
5
|
-
dagster_pandas/version.py,sha256=Xf2lFB6wcfFhFQnDepuVAvU2IU6bic1xrUJKMR4qaag,27
|
|
6
|
-
dagster_pandas/examples/__init__.py,sha256=o3Nd0FCXUVu1_dv5QOTLT5xLrA5inn3DwAj5vY5XDJc,2162
|
|
7
|
-
dagster_pandas/examples/pandas_hello_world.yaml,sha256=Q_4CnkXW5CbgzH239Vv6HH6uU37ysGoCuQ2BW_RWOmQ,82
|
|
8
|
-
dagster_pandas/examples/workspace.yaml,sha256=67d6PErFNq5l7DlRSFTGUwLUwFP1QTd8f8TOCGmbPnU,73
|
|
9
|
-
dagster_pandas/examples/pandas_hello_world/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
10
|
-
dagster_pandas/examples/pandas_hello_world/env.yaml,sha256=pk7REnOLuw5KoyjUhdBdUOXQbYu182Jz74y9J4CIkqE,54
|
|
11
|
-
dagster_pandas/examples/pandas_hello_world/ops.py,sha256=xq1q_BdI8Q7E20z1rjBPYQeQL9wZfEcoSTAfR75vxVo,1176
|
|
12
|
-
dagster_pandas/examples/pandas_hello_world/solids.yaml,sha256=43Qv-8OLGsSsWPZ1DPIwdkl1lwLafJe49yLhspPCJmg,118
|
|
13
|
-
dagster_pandas/examples/pandas_hello_world/environments/pandas_hello_world_prod.yaml,sha256=Y9piDkzJaEpt5z0VAbi_D0-p7onOUIv_4ES894zQnYU,87
|
|
14
|
-
dagster_pandas/examples/pandas_hello_world/environments/pandas_hello_world_test.yaml,sha256=Q_4CnkXW5CbgzH239Vv6HH6uU37ysGoCuQ2BW_RWOmQ,82
|
|
15
|
-
dagster_pandas/examples/pandas_hello_world/environments/papermill_pandas_hello_world_prod.yaml,sha256=1AqRMNMeh4A5OQP7j6C88mYQOIwAHJ7WqvAjyOcPRKQ,108
|
|
16
|
-
dagster_pandas/examples/pandas_hello_world/environments/papermill_pandas_hello_world_test.yaml,sha256=w0AOD6ZUGe0wBK7J5rSJ-x9byxb_lecrB8lQc8ugYvE,103
|
|
17
|
-
dagster_pandas_tests/pandas_hello_world/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
18
|
-
dagster_pandas_tests/pandas_hello_world/test_pandas_hello_world.py,sha256=2ChjqkbWaQ9dj4UlxTYQqfFEYPg_jsf0B3pC_93WBhY,2910
|
|
19
|
-
dagster_pandas-0.13.18rc0.dist-info/LICENSE,sha256=tAkwu8-AdEyGxGoSvJ2gVmQdcicWw3j1ZZueVV74M-E,11357
|
|
20
|
-
dagster_pandas-0.13.18rc0.dist-info/METADATA,sha256=LDi8e_XOkETIsfzjo3TkyXDjClcG5ZAAwfuBbjXHaVE,815
|
|
21
|
-
dagster_pandas-0.13.18rc0.dist-info/WHEEL,sha256=p46_5Uhzqz6AzeSosiOnxK-zmFja1i22CrQCjmYe8ec,92
|
|
22
|
-
dagster_pandas-0.13.18rc0.dist-info/top_level.txt,sha256=5R7_CoEF2uk6vs4x_OnpSj1heVJBCrjypWldvCmHEaI,36
|
|
23
|
-
dagster_pandas-0.13.18rc0.dist-info/RECORD,,
|
|
File without changes
|
|
@@ -1,93 +0,0 @@
|
|
|
1
|
-
import os
|
|
2
|
-
|
|
3
|
-
from dagster import execute_pipeline
|
|
4
|
-
from dagster.cli.pipeline import do_execute_command
|
|
5
|
-
from dagster.core.definitions.reconstructable import ReconstructablePipeline
|
|
6
|
-
from dagster.core.test_utils import instance_for_test
|
|
7
|
-
from dagster.utils import file_relative_path
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
def test_execute_pipeline():
|
|
11
|
-
environment = {
|
|
12
|
-
"ops": {
|
|
13
|
-
"sum_op": {
|
|
14
|
-
"inputs": {"num": {"csv": {"path": file_relative_path(__file__, "num.csv")}}}
|
|
15
|
-
}
|
|
16
|
-
}
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
result = execute_pipeline(
|
|
20
|
-
ReconstructablePipeline.for_module("dagster_pandas.examples", "pandas_hello_world_test"),
|
|
21
|
-
run_config=environment,
|
|
22
|
-
)
|
|
23
|
-
|
|
24
|
-
assert result.success
|
|
25
|
-
|
|
26
|
-
assert result.result_for_solid("sum_op").output_value().to_dict("list") == {
|
|
27
|
-
"num1": [1, 3],
|
|
28
|
-
"num2": [2, 4],
|
|
29
|
-
"sum": [3, 7],
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
assert result.result_for_solid("sum_sq_op").output_value().to_dict("list") == {
|
|
33
|
-
"num1": [1, 3],
|
|
34
|
-
"num2": [2, 4],
|
|
35
|
-
"sum": [3, 7],
|
|
36
|
-
"sum_sq": [9, 49],
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
def test_cli_execute():
|
|
41
|
-
|
|
42
|
-
# currently paths in env files have to be relative to where the
|
|
43
|
-
# script has launched so we have to simulate that
|
|
44
|
-
cwd = os.getcwd()
|
|
45
|
-
try:
|
|
46
|
-
|
|
47
|
-
os.chdir(file_relative_path(__file__, "../.."))
|
|
48
|
-
with instance_for_test() as instance:
|
|
49
|
-
do_execute_command(
|
|
50
|
-
pipeline=ReconstructablePipeline.for_module(
|
|
51
|
-
"dagster_pandas.examples", "pandas_hello_world_test"
|
|
52
|
-
),
|
|
53
|
-
instance=instance,
|
|
54
|
-
config=[
|
|
55
|
-
file_relative_path(
|
|
56
|
-
__file__, "../../dagster_pandas/examples/pandas_hello_world/*.yaml"
|
|
57
|
-
)
|
|
58
|
-
],
|
|
59
|
-
)
|
|
60
|
-
finally:
|
|
61
|
-
# restore cwd
|
|
62
|
-
os.chdir(cwd)
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
def test_cli_execute_failure():
|
|
66
|
-
|
|
67
|
-
# currently paths in env files have to be relative to where the
|
|
68
|
-
# script has launched so we have to simulate that
|
|
69
|
-
# with pytest.raises(DagsterExecutionStepExecutionError) as e_info:
|
|
70
|
-
cwd = os.getcwd()
|
|
71
|
-
try:
|
|
72
|
-
|
|
73
|
-
os.chdir(file_relative_path(__file__, "../.."))
|
|
74
|
-
with instance_for_test() as instance:
|
|
75
|
-
result = do_execute_command(
|
|
76
|
-
pipeline=ReconstructablePipeline.for_module(
|
|
77
|
-
"dagster_pandas.examples",
|
|
78
|
-
"pandas_hello_world_fails_test",
|
|
79
|
-
),
|
|
80
|
-
instance=instance,
|
|
81
|
-
config=[
|
|
82
|
-
file_relative_path(
|
|
83
|
-
__file__, "../../dagster_pandas/examples/pandas_hello_world/*.yaml"
|
|
84
|
-
)
|
|
85
|
-
],
|
|
86
|
-
)
|
|
87
|
-
failures = [event for event in result.step_event_list if event.is_failure]
|
|
88
|
-
finally:
|
|
89
|
-
# restore cwd
|
|
90
|
-
os.chdir(cwd)
|
|
91
|
-
|
|
92
|
-
assert len(failures) == 1
|
|
93
|
-
assert "I am a programmer and I make error" in failures[0].step_failure_data.error.cause.message
|