dagster-pandas 0.25.11__tar.gz → 0.27.9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dagster-pandas might be problematic. Click here for more details.
- {dagster-pandas-0.25.11 → dagster_pandas-0.27.9}/LICENSE +1 -1
- {dagster-pandas-0.25.11/dagster_pandas.egg-info → dagster_pandas-0.27.9}/PKG-INFO +18 -4
- dagster_pandas-0.27.9/README.md +4 -0
- {dagster-pandas-0.25.11 → dagster_pandas-0.27.9}/dagster_pandas/__init__.py +1 -1
- {dagster-pandas-0.25.11 → dagster_pandas-0.27.9}/dagster_pandas/constraints.py +16 -2
- {dagster-pandas-0.25.11 → dagster_pandas-0.27.9}/dagster_pandas/data_frame.py +3 -2
- {dagster-pandas-0.25.11 → dagster_pandas-0.27.9}/dagster_pandas/validation.py +2 -0
- dagster_pandas-0.27.9/dagster_pandas/version.py +1 -0
- {dagster-pandas-0.25.11 → dagster_pandas-0.27.9/dagster_pandas.egg-info}/PKG-INFO +18 -4
- dagster_pandas-0.27.9/dagster_pandas.egg-info/requires.txt +2 -0
- {dagster-pandas-0.25.11 → dagster_pandas-0.27.9}/setup.py +3 -2
- dagster-pandas-0.25.11/README.md +0 -4
- dagster-pandas-0.25.11/dagster_pandas/version.py +0 -1
- dagster-pandas-0.25.11/dagster_pandas.egg-info/requires.txt +0 -2
- {dagster-pandas-0.25.11 → dagster_pandas-0.27.9}/MANIFEST.in +0 -0
- {dagster-pandas-0.25.11 → dagster_pandas-0.27.9}/dagster_pandas/examples/__init__.py +0 -0
- {dagster-pandas-0.25.11 → dagster_pandas-0.27.9}/dagster_pandas/examples/pandas_hello_world/__init__.py +0 -0
- {dagster-pandas-0.25.11 → dagster_pandas-0.27.9}/dagster_pandas/examples/pandas_hello_world/env.yaml +0 -0
- {dagster-pandas-0.25.11 → dagster_pandas-0.27.9}/dagster_pandas/examples/pandas_hello_world/environments/pandas_hello_world_prod.yaml +0 -0
- {dagster-pandas-0.25.11 → dagster_pandas-0.27.9}/dagster_pandas/examples/pandas_hello_world/environments/pandas_hello_world_test.yaml +0 -0
- {dagster-pandas-0.25.11 → dagster_pandas-0.27.9}/dagster_pandas/examples/pandas_hello_world/environments/papermill_pandas_hello_world_prod.yaml +0 -0
- {dagster-pandas-0.25.11 → dagster_pandas-0.27.9}/dagster_pandas/examples/pandas_hello_world/environments/papermill_pandas_hello_world_test.yaml +0 -0
- {dagster-pandas-0.25.11 → dagster_pandas-0.27.9}/dagster_pandas/examples/pandas_hello_world/ops.py +0 -0
- {dagster-pandas-0.25.11 → dagster_pandas-0.27.9}/dagster_pandas/examples/pandas_hello_world/solids.yaml +0 -0
- {dagster-pandas-0.25.11 → dagster_pandas-0.27.9}/dagster_pandas/examples/pandas_hello_world.yaml +0 -0
- {dagster-pandas-0.25.11 → dagster_pandas-0.27.9}/dagster_pandas/examples/workspace.yaml +0 -0
- {dagster-pandas-0.25.11 → dagster_pandas-0.27.9}/dagster_pandas/py.typed +0 -0
- {dagster-pandas-0.25.11 → dagster_pandas-0.27.9}/dagster_pandas.egg-info/SOURCES.txt +0 -0
- {dagster-pandas-0.25.11 → dagster_pandas-0.27.9}/dagster_pandas.egg-info/dependency_links.txt +0 -0
- {dagster-pandas-0.25.11 → dagster_pandas-0.27.9}/dagster_pandas.egg-info/top_level.txt +0 -0
- {dagster-pandas-0.25.11 → dagster_pandas-0.27.9}/setup.cfg +0 -0
|
@@ -186,7 +186,7 @@
|
|
|
186
186
|
same "printed page" as the copyright notice for easier
|
|
187
187
|
identification within third-party archives.
|
|
188
188
|
|
|
189
|
-
Copyright
|
|
189
|
+
Copyright 2025 Dagster Labs, Inc.
|
|
190
190
|
|
|
191
191
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
192
192
|
you may not use this file except in compliance with the License.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: dagster-pandas
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.27.9
|
|
4
4
|
Summary: Utilities and examples for working with pandas and dagster, an opinionated framework for expressing data pipelines
|
|
5
5
|
Home-page: https://github.com/dagster-io/dagster
|
|
6
6
|
Author: Dagster Labs
|
|
@@ -10,13 +10,27 @@ Classifier: Programming Language :: Python :: 3.9
|
|
|
10
10
|
Classifier: Programming Language :: Python :: 3.10
|
|
11
11
|
Classifier: Programming Language :: Python :: 3.11
|
|
12
12
|
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
13
14
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
14
15
|
Classifier: Operating System :: OS Independent
|
|
15
|
-
Requires-Python: >=3.9,<3.
|
|
16
|
+
Requires-Python: >=3.9,<3.14
|
|
16
17
|
Description-Content-Type: text/markdown
|
|
17
18
|
License-File: LICENSE
|
|
19
|
+
Requires-Dist: dagster==1.11.9
|
|
20
|
+
Requires-Dist: pandas
|
|
21
|
+
Dynamic: author
|
|
22
|
+
Dynamic: author-email
|
|
23
|
+
Dynamic: classifier
|
|
24
|
+
Dynamic: description
|
|
25
|
+
Dynamic: description-content-type
|
|
26
|
+
Dynamic: home-page
|
|
27
|
+
Dynamic: license
|
|
28
|
+
Dynamic: license-file
|
|
29
|
+
Dynamic: requires-dist
|
|
30
|
+
Dynamic: requires-python
|
|
31
|
+
Dynamic: summary
|
|
18
32
|
|
|
19
33
|
# dagster-pandas
|
|
20
34
|
|
|
21
35
|
The docs for `dagster-pandas` can be found
|
|
22
|
-
[here](https://docs.dagster.io/
|
|
36
|
+
[here](https://docs.dagster.io/api/python-api/libraries/dagster-pandas).
|
|
@@ -10,7 +10,7 @@ from dagster import (
|
|
|
10
10
|
TypeCheck,
|
|
11
11
|
_check as check,
|
|
12
12
|
)
|
|
13
|
-
from dagster._annotations import
|
|
13
|
+
from dagster._annotations import beta
|
|
14
14
|
from pandas import DataFrame
|
|
15
15
|
|
|
16
16
|
CONSTRAINT_METADATA_KEY: Final = "constraint_metadata"
|
|
@@ -20,6 +20,7 @@ class ConstraintViolationException(Exception):
|
|
|
20
20
|
"""Indicates that a constraint has been violated."""
|
|
21
21
|
|
|
22
22
|
|
|
23
|
+
@beta
|
|
23
24
|
class ConstraintWithMetadataException(Exception):
|
|
24
25
|
"""This class defines the response generated when a pandas DF fails validation -- it can be used to generate either a
|
|
25
26
|
failed typecheck or an exception.
|
|
@@ -105,6 +106,7 @@ class ColumnConstraintViolationException(ConstraintViolationException):
|
|
|
105
106
|
return base_message
|
|
106
107
|
|
|
107
108
|
|
|
109
|
+
@beta
|
|
108
110
|
class ColumnWithMetadataException(ConstraintWithMetadataException):
|
|
109
111
|
def __init__(self, constraint_name, constraint_description, expectation, offending, actual):
|
|
110
112
|
super().__init__(
|
|
@@ -130,7 +132,7 @@ class Constraint:
|
|
|
130
132
|
self.error_description = check.str_param(error_description, "error_description")
|
|
131
133
|
|
|
132
134
|
|
|
133
|
-
@
|
|
135
|
+
@beta
|
|
134
136
|
class ConstraintWithMetadata:
|
|
135
137
|
"""This class defines a base constraint over pandas DFs with organized metadata.
|
|
136
138
|
|
|
@@ -192,6 +194,7 @@ class ConstraintWithMetadata:
|
|
|
192
194
|
)
|
|
193
195
|
|
|
194
196
|
|
|
197
|
+
@beta
|
|
195
198
|
class MultiConstraintWithMetadata(ConstraintWithMetadata):
|
|
196
199
|
"""Use this class if you have multiple constraints to check over the entire dataframe.
|
|
197
200
|
|
|
@@ -237,6 +240,7 @@ class MultiConstraintWithMetadata(ConstraintWithMetadata):
|
|
|
237
240
|
)
|
|
238
241
|
|
|
239
242
|
|
|
243
|
+
@beta
|
|
240
244
|
class StrictColumnsWithMetadata(ConstraintWithMetadata):
|
|
241
245
|
def __init__(self, column_list, enforce_ordering=False, raise_or_typecheck=True, name=None):
|
|
242
246
|
self.enforce_ordering = check.bool_param(enforce_ordering, "enforce_ordering")
|
|
@@ -290,6 +294,7 @@ class DataFrameConstraint(Constraint):
|
|
|
290
294
|
raise NotImplementedError()
|
|
291
295
|
|
|
292
296
|
|
|
297
|
+
@beta
|
|
293
298
|
class StrictColumnsConstraint(DataFrameConstraint):
|
|
294
299
|
"""A dataframe constraint that validates column existence and ordering.
|
|
295
300
|
|
|
@@ -329,6 +334,7 @@ class StrictColumnsConstraint(DataFrameConstraint):
|
|
|
329
334
|
)
|
|
330
335
|
|
|
331
336
|
|
|
337
|
+
@beta
|
|
332
338
|
class RowCountConstraint(DataFrameConstraint):
|
|
333
339
|
"""A dataframe constraint that validates the expected count of rows.
|
|
334
340
|
|
|
@@ -470,6 +476,7 @@ class ColumnConstraintWithMetadata(ConstraintWithMetadata):
|
|
|
470
476
|
return exc.return_as_typecheck()
|
|
471
477
|
|
|
472
478
|
|
|
479
|
+
@beta
|
|
473
480
|
class MultiColumnConstraintWithMetadata(ColumnConstraintWithMetadata):
|
|
474
481
|
"""This class is useful for constructing more complicated relationships between columns
|
|
475
482
|
and expectations -- i.e. you want some validations on column A, others on column B, etc.
|
|
@@ -552,6 +559,7 @@ class MultiColumnConstraintWithMetadata(ColumnConstraintWithMetadata):
|
|
|
552
559
|
return ConstraintWithMetadata.validate(self, data, *args, **kwargs)
|
|
553
560
|
|
|
554
561
|
|
|
562
|
+
@beta
|
|
555
563
|
class MultiAggregateConstraintWithMetadata(MultiColumnConstraintWithMetadata):
|
|
556
564
|
"""This class is similar to multicolumn, but takes in functions that operate on the whole column at once
|
|
557
565
|
rather than ones that operate on each value --
|
|
@@ -589,6 +597,7 @@ class MultiAggregateConstraintWithMetadata(MultiColumnConstraintWithMetadata):
|
|
|
589
597
|
)
|
|
590
598
|
|
|
591
599
|
|
|
600
|
+
@beta
|
|
592
601
|
def non_null_validation(x):
|
|
593
602
|
"""Validates that a particular value in a column is not null.
|
|
594
603
|
|
|
@@ -602,6 +611,7 @@ def non_null_validation(x):
|
|
|
602
611
|
return not pd.isnull(x), {}
|
|
603
612
|
|
|
604
613
|
|
|
614
|
+
@beta
|
|
605
615
|
def all_unique_validator(column, ignore_missing_vals=False):
|
|
606
616
|
"""Validates that all values in an iterable are unique.
|
|
607
617
|
|
|
@@ -640,6 +650,7 @@ def all_unique_validator(column, ignore_missing_vals=False):
|
|
|
640
650
|
return not duplicated.any(), {"actual": column[duplicated]}
|
|
641
651
|
|
|
642
652
|
|
|
653
|
+
@beta
|
|
643
654
|
def nonnull(func):
|
|
644
655
|
"""Decorator for column validation functions to make them error on nulls.
|
|
645
656
|
|
|
@@ -663,6 +674,7 @@ def nonnull(func):
|
|
|
663
674
|
return nvalidator
|
|
664
675
|
|
|
665
676
|
|
|
677
|
+
@beta
|
|
666
678
|
def column_range_validation_factory(minim=None, maxim=None, ignore_missing_vals=False):
|
|
667
679
|
"""Factory for validators testing if column values are within a range.
|
|
668
680
|
|
|
@@ -723,6 +735,7 @@ def column_range_validation_factory(minim=None, maxim=None, ignore_missing_vals=
|
|
|
723
735
|
return in_range_validation_fn
|
|
724
736
|
|
|
725
737
|
|
|
738
|
+
@beta
|
|
726
739
|
def categorical_column_validator_factory(categories, ignore_missing_vals=False):
|
|
727
740
|
"""Factory for validators testing if all values are in some set.
|
|
728
741
|
|
|
@@ -777,6 +790,7 @@ def categorical_column_validator_factory(categories, ignore_missing_vals=False):
|
|
|
777
790
|
return categorical_validation_fn
|
|
778
791
|
|
|
779
792
|
|
|
793
|
+
@beta
|
|
780
794
|
def dtype_in_set_validation_factory(datatypes, ignore_missing_vals=False):
|
|
781
795
|
"""Factory for testing if the dtype of a val falls within some allowed set.
|
|
782
796
|
|
|
@@ -12,7 +12,7 @@ from dagster import (
|
|
|
12
12
|
_check as check,
|
|
13
13
|
dagster_type_loader,
|
|
14
14
|
)
|
|
15
|
-
from dagster._annotations import
|
|
15
|
+
from dagster._annotations import beta
|
|
16
16
|
from dagster._config import Selector
|
|
17
17
|
from dagster._core.definitions.metadata import normalize_metadata
|
|
18
18
|
from dagster._utils import dict_without_keys
|
|
@@ -135,6 +135,7 @@ def create_table_schema_metadata_from_dataframe(
|
|
|
135
135
|
)
|
|
136
136
|
|
|
137
137
|
|
|
138
|
+
@beta
|
|
138
139
|
def create_dagster_pandas_dataframe_type(
|
|
139
140
|
name,
|
|
140
141
|
description=None,
|
|
@@ -201,7 +202,7 @@ def create_dagster_pandas_dataframe_type(
|
|
|
201
202
|
)
|
|
202
203
|
|
|
203
204
|
|
|
204
|
-
@
|
|
205
|
+
@beta
|
|
205
206
|
def create_structured_dataframe_type(
|
|
206
207
|
name,
|
|
207
208
|
description=None,
|
|
@@ -2,6 +2,7 @@ from dagster import (
|
|
|
2
2
|
DagsterInvariantViolationError,
|
|
3
3
|
_check as check,
|
|
4
4
|
)
|
|
5
|
+
from dagster._annotations import beta
|
|
5
6
|
from pandas import DataFrame, Timestamp
|
|
6
7
|
from pandas.core.dtypes.common import (
|
|
7
8
|
is_bool_dtype,
|
|
@@ -42,6 +43,7 @@ def _construct_keyword_constraints(non_nullable, unique, ignore_missing_vals):
|
|
|
42
43
|
return constraints
|
|
43
44
|
|
|
44
45
|
|
|
46
|
+
@beta
|
|
45
47
|
class PandasColumn:
|
|
46
48
|
"""The main API for expressing column level schemas and constraints for your custom dataframe
|
|
47
49
|
types.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.27.9"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: dagster-pandas
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.27.9
|
|
4
4
|
Summary: Utilities and examples for working with pandas and dagster, an opinionated framework for expressing data pipelines
|
|
5
5
|
Home-page: https://github.com/dagster-io/dagster
|
|
6
6
|
Author: Dagster Labs
|
|
@@ -10,13 +10,27 @@ Classifier: Programming Language :: Python :: 3.9
|
|
|
10
10
|
Classifier: Programming Language :: Python :: 3.10
|
|
11
11
|
Classifier: Programming Language :: Python :: 3.11
|
|
12
12
|
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
13
14
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
14
15
|
Classifier: Operating System :: OS Independent
|
|
15
|
-
Requires-Python: >=3.9,<3.
|
|
16
|
+
Requires-Python: >=3.9,<3.14
|
|
16
17
|
Description-Content-Type: text/markdown
|
|
17
18
|
License-File: LICENSE
|
|
19
|
+
Requires-Dist: dagster==1.11.9
|
|
20
|
+
Requires-Dist: pandas
|
|
21
|
+
Dynamic: author
|
|
22
|
+
Dynamic: author-email
|
|
23
|
+
Dynamic: classifier
|
|
24
|
+
Dynamic: description
|
|
25
|
+
Dynamic: description-content-type
|
|
26
|
+
Dynamic: home-page
|
|
27
|
+
Dynamic: license
|
|
28
|
+
Dynamic: license-file
|
|
29
|
+
Dynamic: requires-dist
|
|
30
|
+
Dynamic: requires-python
|
|
31
|
+
Dynamic: summary
|
|
18
32
|
|
|
19
33
|
# dagster-pandas
|
|
20
34
|
|
|
21
35
|
The docs for `dagster-pandas` can be found
|
|
22
|
-
[here](https://docs.dagster.io/
|
|
36
|
+
[here](https://docs.dagster.io/api/python-api/libraries/dagster-pandas).
|
|
@@ -39,14 +39,15 @@ setup(
|
|
|
39
39
|
"Programming Language :: Python :: 3.10",
|
|
40
40
|
"Programming Language :: Python :: 3.11",
|
|
41
41
|
"Programming Language :: Python :: 3.12",
|
|
42
|
+
"Programming Language :: Python :: 3.13",
|
|
42
43
|
"License :: OSI Approved :: Apache Software License",
|
|
43
44
|
"Operating System :: OS Independent",
|
|
44
45
|
],
|
|
45
46
|
packages=find_packages(exclude=["dagster_pandas_tests*"]),
|
|
46
47
|
include_package_data=True,
|
|
47
|
-
python_requires=">=3.9,<3.
|
|
48
|
+
python_requires=">=3.9,<3.14",
|
|
48
49
|
install_requires=[
|
|
49
|
-
"dagster==1.9
|
|
50
|
+
"dagster==1.11.9",
|
|
50
51
|
"pandas",
|
|
51
52
|
],
|
|
52
53
|
)
|
dagster-pandas-0.25.11/README.md
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.25.11"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{dagster-pandas-0.25.11 → dagster_pandas-0.27.9}/dagster_pandas/examples/pandas_hello_world/env.yaml
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{dagster-pandas-0.25.11 → dagster_pandas-0.27.9}/dagster_pandas/examples/pandas_hello_world/ops.py
RENAMED
|
File without changes
|
|
File without changes
|
{dagster-pandas-0.25.11 → dagster_pandas-0.27.9}/dagster_pandas/examples/pandas_hello_world.yaml
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{dagster-pandas-0.25.11 → dagster_pandas-0.27.9}/dagster_pandas.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|