dataframely 1.2.0__tar.gz → 1.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dataframely-1.2.0 → dataframely-1.3.0}/.copier-answers.yml +1 -1
- {dataframely-1.2.0 → dataframely-1.3.0}/.github/workflows/build.yml +1 -1
- {dataframely-1.2.0 → dataframely-1.3.0}/Cargo.toml +4 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/PKG-INFO +1 -1
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/__init__.py +2 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/_base_schema.py +6 -6
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/_rule.py +3 -1
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/columns/__init__.py +2 -0
- dataframely-1.3.0/dataframely/columns/array.py +105 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/columns/enum.py +1 -1
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/columns/list.py +4 -1
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/columns/struct.py +5 -1
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/mypy.py +23 -1
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/random.py +4 -2
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/testing/__init__.py +2 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/testing/const.py +5 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/pyproject.toml +1 -1
- dataframely-1.3.0/tests/column_types/test_array.py +159 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/column_types/test_enum.py +9 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/columns/test_default_dtypes.py +1 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/columns/test_pyarrow.py +27 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/columns/test_sample.py +9 -1
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/columns/test_sql_schema.py +8 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/columns/test_str.py +5 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/schema/test_base.py +14 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/schema/test_validate.py +9 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/test_typing.py +15 -0
- dataframely-1.2.0/.github/workflows/update-lockfiles.yml +0 -34
- {dataframely-1.2.0 → dataframely-1.3.0}/.envrc +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/.gitattributes +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/.github/CODEOWNERS +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/.github/dependabot.yml +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/.github/release-drafter.yml +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/.github/workflows/chore.yml +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/.github/workflows/ci.yml +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/.gitignore +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/.pre-commit-config.yaml +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/.prettierignore +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/.prettierrc +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/.readthedocs.yml +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/Cargo.lock +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/LICENSE +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/README.md +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/_base_collection.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/_compat.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/_extre.pyi +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/_filter.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/_polars.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/_typing.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/_validation.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/collection.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/columns/_base.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/columns/_mixins.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/columns/_utils.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/columns/any.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/columns/bool.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/columns/datetime.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/columns/decimal.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/columns/float.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/columns/integer.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/columns/object.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/columns/string.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/config.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/exc.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/failure.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/functional.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/py.typed +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/schema.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/testing/factory.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/testing/mask.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/testing/rules.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/testing/typing.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docker-compose.yml +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/Makefile +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.collection.rst +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.columns.any.rst +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.columns.bool.rst +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.columns.datetime.rst +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.columns.decimal.rst +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.columns.enum.rst +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.columns.float.rst +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.columns.integer.rst +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.columns.list.rst +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.columns.rst +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.columns.string.rst +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.columns.struct.rst +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.config.rst +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.exc.rst +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.failure.rst +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.functional.rst +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.mypy.rst +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.random.rst +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.rst +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.schema.rst +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.testing.const.rst +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.testing.factory.rst +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.testing.mask.rst +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.testing.rst +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.testing.rules.rst +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.testing.typing.rst +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/modules.rst +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/_static/custom.css +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/_static/favicon.ico +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/conf.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/index.rst +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/make.bat +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/sites/development.rst +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/sites/examples/real-world.ipynb +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/sites/faq.rst +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/sites/installation.rst +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/docs/sites/quickstart.rst +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/pixi.lock +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/pixi.toml +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/src/errdefs.rs +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/src/lib.rs +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/src/regex_repr.rs +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/collection/test_base.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/collection/test_cast.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/collection/test_create_empty.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/collection/test_filter_one_to_n.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/collection/test_filter_validate.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/collection/test_ignore_in_filter.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/collection/test_implementation.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/collection/test_optional_members.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/collection/test_sample.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/collection/test_validate_input.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/column_types/__init__.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/column_types/test_any.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/column_types/test_datetime.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/column_types/test_decimal.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/column_types/test_float.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/column_types/test_integer.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/column_types/test_list.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/column_types/test_object.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/column_types/test_string.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/column_types/test_struct.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/columns/__init__.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/columns/test_alias.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/columns/test_check.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/columns/test_metadata.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/columns/test_polars_schema.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/columns/test_rules.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/columns/test_utils.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/core_validation/__init__.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/core_validation/test_column_validation.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/core_validation/test_dtype_validation.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/core_validation/test_rule_evaluation.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/functional/test_concat.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/functional/test_relationships.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/schema/test_cast.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/schema/test_create_empty.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/schema/test_create_empty_if_none.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/schema/test_filter.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/schema/test_inheritance.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/schema/test_rule_implementation.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/schema/test_sample.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/test_compat.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/test_config.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/test_exc.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/test_extre.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/test_failure_info.py +0 -0
- {dataframely-1.2.0 → dataframely-1.3.0}/tests/test_random.py +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# This file is managed by Copier; DO NOT EDIT OR REMOVE.
|
|
2
2
|
_commit: v0.3.0
|
|
3
3
|
_src_path: https://github.com/quantco/copier-template-python-open-source
|
|
4
|
-
add_autobump_workflow:
|
|
4
|
+
add_autobump_workflow: false
|
|
5
5
|
author_email: oliver.borchert@quantco.com
|
|
6
6
|
author_name: Oliver Borchert
|
|
7
7
|
github_url: https://github.com/quantco/dataframely
|
|
@@ -78,7 +78,7 @@ jobs:
|
|
|
78
78
|
id-token: write
|
|
79
79
|
environment: pypi
|
|
80
80
|
steps:
|
|
81
|
-
- uses: actions/download-artifact@
|
|
81
|
+
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
|
82
82
|
with:
|
|
83
83
|
path: dist
|
|
84
84
|
merge-multiple: true
|
|
@@ -18,6 +18,7 @@ from ._typing import DataFrame, LazyFrame
|
|
|
18
18
|
from .collection import Collection
|
|
19
19
|
from .columns import (
|
|
20
20
|
Any,
|
|
21
|
+
Array,
|
|
21
22
|
Bool,
|
|
22
23
|
Column,
|
|
23
24
|
Date,
|
|
@@ -90,5 +91,6 @@ __all__ = [
|
|
|
90
91
|
"String",
|
|
91
92
|
"Struct",
|
|
92
93
|
"List",
|
|
94
|
+
"Array",
|
|
93
95
|
"Object",
|
|
94
96
|
]
|
|
@@ -98,7 +98,7 @@ class SchemaMeta(ABCMeta):
|
|
|
98
98
|
)
|
|
99
99
|
|
|
100
100
|
# 2) Check that the columns referenced in the group rules exist.
|
|
101
|
-
for
|
|
101
|
+
for rule_name, rule in result.rules.items():
|
|
102
102
|
if isinstance(rule, GroupRule):
|
|
103
103
|
missing_columns = set(rule.group_columns) - set(result.columns)
|
|
104
104
|
if len(missing_columns) > 0:
|
|
@@ -106,7 +106,7 @@ class SchemaMeta(ABCMeta):
|
|
|
106
106
|
sorted(f"'{col}'" for col in missing_columns)
|
|
107
107
|
)
|
|
108
108
|
raise ImplementationError(
|
|
109
|
-
f"Group validation rule '{
|
|
109
|
+
f"Group validation rule '{rule_name}' has been implemented "
|
|
110
110
|
f"incorrectly. It references {len(missing_columns)} columns "
|
|
111
111
|
f"which are not in the schema: {missing_list}."
|
|
112
112
|
)
|
|
@@ -116,16 +116,16 @@ class SchemaMeta(ABCMeta):
|
|
|
116
116
|
# empty data frame and checking for the evaluated dtypes.
|
|
117
117
|
if len(result.rules) > 0:
|
|
118
118
|
lf_empty = pl.LazyFrame(
|
|
119
|
-
schema={
|
|
119
|
+
schema={col_name: col.dtype for col_name, col in result.columns.items()}
|
|
120
120
|
)
|
|
121
121
|
# NOTE: For some reason, `polars` does not yield correct dtypes when calling
|
|
122
122
|
# `collect_schema()`
|
|
123
123
|
schema = with_evaluation_rules(lf_empty, result.rules).collect().schema
|
|
124
|
-
for
|
|
125
|
-
dtype = schema[
|
|
124
|
+
for rule_name, rule in result.rules.items():
|
|
125
|
+
dtype = schema[rule_name]
|
|
126
126
|
if not isinstance(dtype, pl.Boolean):
|
|
127
127
|
raise RuleImplementationError(
|
|
128
|
-
|
|
128
|
+
rule_name, dtype, isinstance(rule, GroupRule)
|
|
129
129
|
)
|
|
130
130
|
|
|
131
131
|
return super().__new__(mcs, name, bases, namespace, *args, **kwargs)
|
|
@@ -126,5 +126,7 @@ def _with_group_rules(lf: pl.LazyFrame, rules: dict[str, GroupRule]) -> pl.LazyF
|
|
|
126
126
|
# preserves the order of the left data frame.
|
|
127
127
|
result = lf
|
|
128
128
|
for group_columns, frame in group_evaluations.items():
|
|
129
|
-
result = result.join(
|
|
129
|
+
result = result.join(
|
|
130
|
+
frame, on=list(group_columns), how="left", nulls_equal=True
|
|
131
|
+
)
|
|
130
132
|
return result
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
|
|
4
4
|
from ._base import Column
|
|
5
5
|
from .any import Any
|
|
6
|
+
from .array import Array
|
|
6
7
|
from .bool import Bool
|
|
7
8
|
from .datetime import Date, Datetime, Duration, Time
|
|
8
9
|
from .decimal import Decimal
|
|
@@ -17,6 +18,7 @@ from .struct import Struct
|
|
|
17
18
|
__all__ = [
|
|
18
19
|
"Column",
|
|
19
20
|
"Any",
|
|
21
|
+
"Array",
|
|
20
22
|
"Bool",
|
|
21
23
|
"Date",
|
|
22
24
|
"Datetime",
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
# Copyright (c) QuantCo 2025-2025
|
|
2
|
+
# SPDX-License-Identifier: BSD-3-Clause
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import math
|
|
7
|
+
from collections.abc import Callable, Sequence
|
|
8
|
+
from typing import Any, Literal
|
|
9
|
+
|
|
10
|
+
import polars as pl
|
|
11
|
+
|
|
12
|
+
from dataframely._compat import pa, sa, sa_TypeEngine
|
|
13
|
+
from dataframely.random import Generator
|
|
14
|
+
|
|
15
|
+
from ._base import Column
|
|
16
|
+
from .struct import Struct
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class Array(Column):
|
|
20
|
+
"""A fixed-shape array column."""
|
|
21
|
+
|
|
22
|
+
def __init__(
|
|
23
|
+
self,
|
|
24
|
+
inner: Column,
|
|
25
|
+
shape: int | tuple[int, ...],
|
|
26
|
+
*,
|
|
27
|
+
nullable: bool = True,
|
|
28
|
+
# polars doesn't yet support grouping by arrays,
|
|
29
|
+
# see https://github.com/pola-rs/polars/issues/22574
|
|
30
|
+
primary_key: Literal[False] = False,
|
|
31
|
+
check: Callable[[pl.Expr], pl.Expr] | None = None,
|
|
32
|
+
alias: str | None = None,
|
|
33
|
+
metadata: dict[str, Any] | None = None,
|
|
34
|
+
):
|
|
35
|
+
"""
|
|
36
|
+
Args:
|
|
37
|
+
inner: The inner column type. No validation rules on the inner type are supported yet.
|
|
38
|
+
shape: The shape of the array.
|
|
39
|
+
nullable: Whether this column may contain null values.
|
|
40
|
+
primary_key: Whether this column is part of the primary key of the schema.
|
|
41
|
+
Not yet supported for the Array type.
|
|
42
|
+
check: A custom check to run for this column. Must return a non-aggregated
|
|
43
|
+
boolean expression.
|
|
44
|
+
alias: An overwrite for this column's name which allows for using a column
|
|
45
|
+
name that is not a valid Python identifier. Especially note that setting
|
|
46
|
+
this option does _not_ allow to refer to the column with two different
|
|
47
|
+
names, the specified alias is the only valid name.
|
|
48
|
+
metadata: A dictionary of metadata to attach to the column.
|
|
49
|
+
"""
|
|
50
|
+
if inner.primary_key or (
|
|
51
|
+
isinstance(inner, Struct)
|
|
52
|
+
and any(col.primary_key for col in inner.inner.values())
|
|
53
|
+
):
|
|
54
|
+
raise ValueError(
|
|
55
|
+
"`primary_key=True` is not yet supported for inner types of the Array type."
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
# We disallow validation rules on the inner type since Polars arrays currently don't support .eval(). Converting
|
|
59
|
+
# to a list and calling .list.eval() is possible, however, since the shape can have multiple axes, the recursive
|
|
60
|
+
# conversion could have significant performance impact. Hence, we simply disallow inner validation rules.
|
|
61
|
+
# Another option would be to allow validation rules only for sampling, but not enforce them.
|
|
62
|
+
if inner.validation_rules(pl.lit(None)):
|
|
63
|
+
raise ValueError(
|
|
64
|
+
"Validation rules on the inner type of Array are not yet supported."
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
super().__init__(
|
|
68
|
+
nullable=nullable,
|
|
69
|
+
primary_key=False,
|
|
70
|
+
check=check,
|
|
71
|
+
alias=alias,
|
|
72
|
+
metadata=metadata,
|
|
73
|
+
)
|
|
74
|
+
self.inner = inner
|
|
75
|
+
self.shape = shape if isinstance(shape, tuple) else (shape,)
|
|
76
|
+
|
|
77
|
+
@property
|
|
78
|
+
def dtype(self) -> pl.DataType:
|
|
79
|
+
return pl.Array(self.inner.dtype, self.shape)
|
|
80
|
+
|
|
81
|
+
def sqlalchemy_dtype(self, dialect: sa.Dialect) -> sa_TypeEngine:
|
|
82
|
+
# NOTE: We might want to add support for PostgreSQL's ARRAY type or use JSON in the future.
|
|
83
|
+
raise NotImplementedError("SQL column cannot have 'Array' type.")
|
|
84
|
+
|
|
85
|
+
def _pyarrow_dtype_of_shape(self, shape: Sequence[int]) -> pa.DataType:
|
|
86
|
+
if shape:
|
|
87
|
+
size, *rest = shape
|
|
88
|
+
return pa.list_(self._pyarrow_dtype_of_shape(rest), size)
|
|
89
|
+
else:
|
|
90
|
+
return self.inner.pyarrow_dtype
|
|
91
|
+
|
|
92
|
+
@property
|
|
93
|
+
def pyarrow_dtype(self) -> pa.DataType:
|
|
94
|
+
return self._pyarrow_dtype_of_shape(self.shape)
|
|
95
|
+
|
|
96
|
+
def _sample_unchecked(self, generator: Generator, n: int) -> pl.Series:
|
|
97
|
+
# Sample the inner elements in a flat series
|
|
98
|
+
n_elements = n * math.prod(self.shape)
|
|
99
|
+
all_elements = self.inner.sample(generator, n_elements)
|
|
100
|
+
|
|
101
|
+
# Finally, apply a null mask
|
|
102
|
+
return generator._apply_null_mask(
|
|
103
|
+
all_elements.reshape((n, *self.shape)),
|
|
104
|
+
null_probability=self._null_probability,
|
|
105
|
+
)
|
|
@@ -147,4 +147,7 @@ class List(Column):
|
|
|
147
147
|
chain([0], element_lengths.cum_sum()), element_lengths
|
|
148
148
|
)
|
|
149
149
|
]
|
|
150
|
-
|
|
150
|
+
# Finally, apply a null mask
|
|
151
|
+
return generator._apply_null_mask(
|
|
152
|
+
pl.Series(list_elements), null_probability=self._null_probability
|
|
153
|
+
)
|
|
@@ -96,10 +96,14 @@ class Struct(Column):
|
|
|
96
96
|
return pa.struct({name: col.pyarrow_dtype for name, col in self.inner.items()})
|
|
97
97
|
|
|
98
98
|
def _sample_unchecked(self, generator: Generator, n: int) -> pl.Series:
|
|
99
|
-
|
|
99
|
+
series = (
|
|
100
100
|
pl.DataFrame(
|
|
101
101
|
{name: col.sample(generator, n) for name, col in self.inner.items()}
|
|
102
102
|
)
|
|
103
103
|
.select(pl.struct(pl.all()))
|
|
104
104
|
.to_series()
|
|
105
105
|
)
|
|
106
|
+
# Apply a null mask.
|
|
107
|
+
return generator._apply_null_mask(
|
|
108
|
+
series, null_probability=self._null_probability
|
|
109
|
+
)
|
|
@@ -16,6 +16,7 @@ from mypy.nodes import (
|
|
|
16
16
|
MemberExpr,
|
|
17
17
|
NameExpr,
|
|
18
18
|
StrExpr,
|
|
19
|
+
TupleExpr,
|
|
19
20
|
TypeInfo,
|
|
20
21
|
Var,
|
|
21
22
|
)
|
|
@@ -189,7 +190,28 @@ def _convert_dy_column_to_dtype(
|
|
|
189
190
|
],
|
|
190
191
|
)
|
|
191
192
|
return api.named_type("builtins.list")
|
|
192
|
-
if column_type == "
|
|
193
|
+
if column_type == "Array":
|
|
194
|
+
if isinstance(column_args[0], CallExpr) and isinstance(
|
|
195
|
+
column_args[0].callee, MemberExpr | NameExpr
|
|
196
|
+
):
|
|
197
|
+
inner_type = _convert_dy_column_to_dtype(
|
|
198
|
+
api,
|
|
199
|
+
column_args[0].callee.name,
|
|
200
|
+
column_args[0].args,
|
|
201
|
+
)
|
|
202
|
+
# If the array has more than one dimension, return a list of lists of the inner type.
|
|
203
|
+
if len(column_args) > 1 and isinstance(column_args[1], TupleExpr):
|
|
204
|
+
for _ in range(len(column_args[1].items) - 1):
|
|
205
|
+
inner_type = api.named_type(
|
|
206
|
+
"builtins.list",
|
|
207
|
+
[inner_type],
|
|
208
|
+
)
|
|
209
|
+
return api.named_type(
|
|
210
|
+
"builtins.list",
|
|
211
|
+
[inner_type],
|
|
212
|
+
)
|
|
213
|
+
return api.named_type("builtins.list")
|
|
214
|
+
if column_type == "Any" or column_type == "Object":
|
|
193
215
|
return AnyType(TypeOfAny.explicit)
|
|
194
216
|
# If we can't infer the type, we default to `Any`.
|
|
195
217
|
# This is, for example, the case for self-defined types, e.g., via `functools.partial`.
|
|
@@ -392,8 +392,10 @@ class Generator:
|
|
|
392
392
|
def _apply_null_mask(self, series: pl.Series, null_probability: float) -> pl.Series:
|
|
393
393
|
if null_probability == 0:
|
|
394
394
|
return series
|
|
395
|
-
null_mask =
|
|
396
|
-
|
|
395
|
+
null_mask = (
|
|
396
|
+
pl.Series(self.numpy_generator.random(series.len())) > null_probability
|
|
397
|
+
)
|
|
398
|
+
return pl.select(pl.when(null_mask).then(series)).to_series()
|
|
397
399
|
|
|
398
400
|
|
|
399
401
|
# --------------------------------------- UTILS -------------------------------------- #
|
|
@@ -6,6 +6,7 @@ from .const import (
|
|
|
6
6
|
COLUMN_TYPES,
|
|
7
7
|
FLOAT_COLUMN_TYPES,
|
|
8
8
|
INTEGER_COLUMN_TYPES,
|
|
9
|
+
NO_VALIDATION_COLUMN_TYPES,
|
|
9
10
|
SUPERTYPE_COLUMN_TYPES,
|
|
10
11
|
)
|
|
11
12
|
from .factory import create_collection, create_collection_raw, create_schema
|
|
@@ -18,6 +19,7 @@ __all__ = [
|
|
|
18
19
|
"FLOAT_COLUMN_TYPES",
|
|
19
20
|
"INTEGER_COLUMN_TYPES",
|
|
20
21
|
"SUPERTYPE_COLUMN_TYPES",
|
|
22
|
+
"NO_VALIDATION_COLUMN_TYPES",
|
|
21
23
|
"create_collection",
|
|
22
24
|
"create_collection_raw",
|
|
23
25
|
"create_schema",
|
|
@@ -47,3 +47,8 @@ SUPERTYPE_COLUMN_TYPES: list[type[dc.Column]] = [
|
|
|
47
47
|
ALL_COLUMN_TYPES: list[type[dc.Column]] = (
|
|
48
48
|
[dc.Any] + COLUMN_TYPES + SUPERTYPE_COLUMN_TYPES
|
|
49
49
|
)
|
|
50
|
+
|
|
51
|
+
# The following is a list of column types that, when created with default parameter values, add no validation rules.
|
|
52
|
+
NO_VALIDATION_COLUMN_TYPES: list[type[dc.Column]] = [
|
|
53
|
+
t for t in ALL_COLUMN_TYPES if t not in FLOAT_COLUMN_TYPES
|
|
54
|
+
]
|
|
@@ -19,7 +19,7 @@ description = "A declarative, polars-native data frame validation library"
|
|
|
19
19
|
name = "dataframely"
|
|
20
20
|
readme = "README.md"
|
|
21
21
|
requires-python = ">=3.11"
|
|
22
|
-
version = "1.
|
|
22
|
+
version = "1.3.0"
|
|
23
23
|
|
|
24
24
|
[project.urls]
|
|
25
25
|
Repository = "https://github.com/quantco/dataframely"
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
# Copyright (c) QuantCo 2025-2025
|
|
2
|
+
# SPDX-License-Identifier: BSD-3-Clause
|
|
3
|
+
|
|
4
|
+
import polars as pl
|
|
5
|
+
import pytest
|
|
6
|
+
|
|
7
|
+
import dataframely as dy
|
|
8
|
+
from dataframely.columns._base import Column
|
|
9
|
+
from dataframely.testing import create_schema
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@pytest.mark.parametrize(
|
|
13
|
+
"inner",
|
|
14
|
+
[
|
|
15
|
+
(dy.Int64()),
|
|
16
|
+
(dy.Integer()),
|
|
17
|
+
],
|
|
18
|
+
)
|
|
19
|
+
def test_integer_array(inner: Column) -> None:
|
|
20
|
+
schema = create_schema("test", {"a": dy.Array(inner, 1)})
|
|
21
|
+
assert schema.is_valid(
|
|
22
|
+
pl.DataFrame(
|
|
23
|
+
{"a": [[1], [2], [3]]},
|
|
24
|
+
schema={
|
|
25
|
+
"a": pl.Array(pl.Int64, 1),
|
|
26
|
+
},
|
|
27
|
+
)
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def test_invalid_inner_type() -> None:
|
|
32
|
+
schema = create_schema("test", {"a": dy.Array(dy.Int64(), 1)})
|
|
33
|
+
assert not schema.is_valid(pl.DataFrame({"a": [["1"], ["2"], ["3"]]}))
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def test_invalid_shape() -> None:
|
|
37
|
+
schema = create_schema("test", {"a": dy.Array(dy.Int64(), 2)})
|
|
38
|
+
assert not schema.is_valid(
|
|
39
|
+
pl.DataFrame(
|
|
40
|
+
{"a": [[1], [2], [3]]},
|
|
41
|
+
schema={
|
|
42
|
+
"a": pl.Array(pl.Int64, 1),
|
|
43
|
+
},
|
|
44
|
+
)
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@pytest.mark.parametrize(
|
|
49
|
+
("column", "dtype", "is_valid"),
|
|
50
|
+
[
|
|
51
|
+
(
|
|
52
|
+
dy.Array(dy.Int64(), 1),
|
|
53
|
+
pl.Array(pl.Int64(), 1),
|
|
54
|
+
True,
|
|
55
|
+
),
|
|
56
|
+
(
|
|
57
|
+
dy.Array(dy.String(), 1),
|
|
58
|
+
pl.Array(pl.Int64(), 1),
|
|
59
|
+
False,
|
|
60
|
+
),
|
|
61
|
+
(
|
|
62
|
+
dy.Array(dy.String(), 1),
|
|
63
|
+
pl.Array(pl.Int64(), 2),
|
|
64
|
+
False,
|
|
65
|
+
),
|
|
66
|
+
(
|
|
67
|
+
dy.Array(dy.Int64(), (1,)),
|
|
68
|
+
pl.Array(pl.Int64(), (1,)),
|
|
69
|
+
True,
|
|
70
|
+
),
|
|
71
|
+
(
|
|
72
|
+
dy.Array(dy.Int64(), (1,)),
|
|
73
|
+
pl.Array(pl.Int64(), (2,)),
|
|
74
|
+
False,
|
|
75
|
+
),
|
|
76
|
+
(
|
|
77
|
+
dy.Array(dy.String(), 1),
|
|
78
|
+
dy.Array(dy.String(), 1),
|
|
79
|
+
False,
|
|
80
|
+
),
|
|
81
|
+
(
|
|
82
|
+
dy.Array(dy.String(), 1),
|
|
83
|
+
dy.String(),
|
|
84
|
+
False,
|
|
85
|
+
),
|
|
86
|
+
(
|
|
87
|
+
dy.Array(dy.String(), 1),
|
|
88
|
+
pl.String(),
|
|
89
|
+
False,
|
|
90
|
+
),
|
|
91
|
+
(
|
|
92
|
+
dy.Array(dy.Array(dy.String(), 1), 1),
|
|
93
|
+
pl.Array(pl.String(), (1, 1)),
|
|
94
|
+
True,
|
|
95
|
+
),
|
|
96
|
+
(
|
|
97
|
+
dy.Array(dy.String(), (1, 1)),
|
|
98
|
+
pl.Array(pl.Array(pl.String(), 1), 1),
|
|
99
|
+
True,
|
|
100
|
+
),
|
|
101
|
+
],
|
|
102
|
+
)
|
|
103
|
+
def test_validate_dtype(column: Column, dtype: pl.DataType, is_valid: bool) -> None:
|
|
104
|
+
assert column.validate_dtype(dtype) == is_valid
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def test_nested_arrays() -> None:
|
|
108
|
+
schema = create_schema("test", {"a": dy.Array(dy.Array(dy.Int64(), 1), 1)})
|
|
109
|
+
assert schema.is_valid(
|
|
110
|
+
pl.DataFrame(
|
|
111
|
+
{"a": [[[1]], [[2]], [[3]]]},
|
|
112
|
+
schema={
|
|
113
|
+
"a": pl.Array(pl.Int64, (1, 1)),
|
|
114
|
+
},
|
|
115
|
+
)
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def test_nested_array() -> None:
|
|
120
|
+
schema = create_schema("test", {"a": dy.Array(dy.Array(dy.Int64(), 1), 1)})
|
|
121
|
+
assert schema.is_valid(
|
|
122
|
+
pl.DataFrame(
|
|
123
|
+
{"a": [[[1]], [[2]], [[3]]]},
|
|
124
|
+
schema={
|
|
125
|
+
"a": pl.Array(pl.Int64, (1, 1)),
|
|
126
|
+
},
|
|
127
|
+
)
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def test_array_with_inner_pk() -> None:
|
|
132
|
+
with pytest.raises(ValueError):
|
|
133
|
+
column = dy.Array(dy.String(primary_key=True), 2)
|
|
134
|
+
create_schema(
|
|
135
|
+
"test",
|
|
136
|
+
{"a": column},
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def test_array_with_rules() -> None:
|
|
141
|
+
with pytest.raises(ValueError):
|
|
142
|
+
create_schema(
|
|
143
|
+
"test", {"a": dy.Array(dy.String(min_length=2, nullable=False), 1)}
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def test_outer_nullability() -> None:
|
|
148
|
+
schema = create_schema(
|
|
149
|
+
"test",
|
|
150
|
+
{
|
|
151
|
+
"nullable": dy.Array(
|
|
152
|
+
inner=dy.Integer(),
|
|
153
|
+
shape=1,
|
|
154
|
+
nullable=True,
|
|
155
|
+
)
|
|
156
|
+
},
|
|
157
|
+
)
|
|
158
|
+
df = pl.DataFrame({"nullable": [None, None]})
|
|
159
|
+
schema.validate(df, cast=True)
|
|
@@ -52,3 +52,12 @@ def test_valid_cast(
|
|
|
52
52
|
schema = create_schema("test", {"a": enum})
|
|
53
53
|
df = df_type(data)
|
|
54
54
|
assert schema.is_valid(df, cast=True) == valid
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@pytest.mark.parametrize("type1", [list, tuple])
|
|
58
|
+
@pytest.mark.parametrize("type2", [list, tuple])
|
|
59
|
+
def test_different_sequences(type1: type, type2: type) -> None:
|
|
60
|
+
allowed = ["a", "b"]
|
|
61
|
+
S = create_schema("test", {"x": dy.Enum(type1(allowed))})
|
|
62
|
+
df = pl.DataFrame({"x": pl.Series(["a", "b"], dtype=pl.Enum(type2(allowed)))})
|
|
63
|
+
S.validate(df)
|
|
@@ -36,6 +36,7 @@ from dataframely.testing import create_schema
|
|
|
36
36
|
(dy.UInt64(), pl.UInt64()),
|
|
37
37
|
(dy.String(), pl.String()),
|
|
38
38
|
(dy.List(dy.String()), pl.List(pl.String())),
|
|
39
|
+
(dy.Array(dy.String(), 1), pl.Array(pl.String(), 1)),
|
|
39
40
|
(dy.Struct({"a": dy.String()}), pl.Struct({"a": pl.String()})),
|
|
40
41
|
(dy.Enum(["a", "b"]), pl.Enum(["a", "b"])),
|
|
41
42
|
],
|
|
@@ -8,6 +8,7 @@ from dataframely.columns import Column
|
|
|
8
8
|
from dataframely.testing import (
|
|
9
9
|
ALL_COLUMN_TYPES,
|
|
10
10
|
COLUMN_TYPES,
|
|
11
|
+
NO_VALIDATION_COLUMN_TYPES,
|
|
11
12
|
SUPERTYPE_COLUMN_TYPES,
|
|
12
13
|
create_schema,
|
|
13
14
|
)
|
|
@@ -32,6 +33,7 @@ def test_equal_polars_schema_enum() -> None:
|
|
|
32
33
|
"inner",
|
|
33
34
|
[c() for c in ALL_COLUMN_TYPES]
|
|
34
35
|
+ [dy.List(t()) for t in ALL_COLUMN_TYPES]
|
|
36
|
+
+ [dy.Array(t(), 1) for t in NO_VALIDATION_COLUMN_TYPES]
|
|
35
37
|
+ [dy.Struct({"a": t()}) for t in ALL_COLUMN_TYPES],
|
|
36
38
|
)
|
|
37
39
|
def test_equal_polars_schema_list(inner: Column) -> None:
|
|
@@ -41,10 +43,33 @@ def test_equal_polars_schema_list(inner: Column) -> None:
|
|
|
41
43
|
assert actual == expected
|
|
42
44
|
|
|
43
45
|
|
|
46
|
+
@pytest.mark.parametrize(
|
|
47
|
+
"inner",
|
|
48
|
+
[c() for c in NO_VALIDATION_COLUMN_TYPES]
|
|
49
|
+
+ [dy.List(t()) for t in NO_VALIDATION_COLUMN_TYPES]
|
|
50
|
+
+ [dy.Array(t(), 1) for t in NO_VALIDATION_COLUMN_TYPES]
|
|
51
|
+
+ [dy.Struct({"a": t()}) for t in NO_VALIDATION_COLUMN_TYPES],
|
|
52
|
+
)
|
|
53
|
+
@pytest.mark.parametrize(
|
|
54
|
+
"shape",
|
|
55
|
+
[
|
|
56
|
+
1,
|
|
57
|
+
0,
|
|
58
|
+
(0, 0),
|
|
59
|
+
],
|
|
60
|
+
)
|
|
61
|
+
def test_equal_polars_schema_array(inner: Column, shape: int | tuple[int, ...]) -> None:
|
|
62
|
+
schema = create_schema("test", {"a": dy.Array(inner, shape)})
|
|
63
|
+
actual = schema.pyarrow_schema()
|
|
64
|
+
expected = schema.create_empty().to_arrow().schema
|
|
65
|
+
assert actual == expected
|
|
66
|
+
|
|
67
|
+
|
|
44
68
|
@pytest.mark.parametrize(
|
|
45
69
|
"inner",
|
|
46
70
|
[c() for c in ALL_COLUMN_TYPES]
|
|
47
71
|
+ [dy.Struct({"a": t()}) for t in ALL_COLUMN_TYPES]
|
|
72
|
+
+ [dy.Array(t(), 1) for t in NO_VALIDATION_COLUMN_TYPES]
|
|
48
73
|
+ [dy.List(t()) for t in ALL_COLUMN_TYPES],
|
|
49
74
|
)
|
|
50
75
|
def test_equal_polars_schema_struct(inner: Column) -> None:
|
|
@@ -71,6 +96,7 @@ def test_nullability_information_enum(nullable: bool) -> None:
|
|
|
71
96
|
"inner",
|
|
72
97
|
[c() for c in ALL_COLUMN_TYPES]
|
|
73
98
|
+ [dy.List(t()) for t in ALL_COLUMN_TYPES]
|
|
99
|
+
+ [dy.Array(t(), 1) for t in NO_VALIDATION_COLUMN_TYPES]
|
|
74
100
|
+ [dy.Struct({"a": t()}) for t in ALL_COLUMN_TYPES],
|
|
75
101
|
)
|
|
76
102
|
@pytest.mark.parametrize("nullable", [True, False])
|
|
@@ -83,6 +109,7 @@ def test_nullability_information_list(inner: Column, nullable: bool) -> None:
|
|
|
83
109
|
"inner",
|
|
84
110
|
[c() for c in ALL_COLUMN_TYPES]
|
|
85
111
|
+ [dy.Struct({"a": t()}) for t in ALL_COLUMN_TYPES]
|
|
112
|
+
+ [dy.Array(t(), 1) for t in NO_VALIDATION_COLUMN_TYPES]
|
|
86
113
|
+ [dy.List(t()) for t in ALL_COLUMN_TYPES],
|
|
87
114
|
)
|
|
88
115
|
@pytest.mark.parametrize("nullable", [True, False])
|
|
@@ -178,12 +178,20 @@ def test_sample_enum(generator: Generator) -> None:
|
|
|
178
178
|
def test_sample_list(generator: Generator) -> None:
|
|
179
179
|
column = dy.List(dy.String(regex="[abc]"), min_length=5, max_length=10)
|
|
180
180
|
samples = sample_and_validate(column, generator, n=10_000)
|
|
181
|
-
assert set(samples.list.len()) == set(range(5, 11))
|
|
181
|
+
assert set(samples.list.len()) == set(range(5, 11)) | {None}
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def test_sample_array(generator: Generator) -> None:
|
|
185
|
+
column = dy.Array(dy.Bool(), (2, 3))
|
|
186
|
+
samples = sample_and_validate(column, generator, n=10_000)
|
|
187
|
+
assert samples.is_null().any()
|
|
188
|
+
assert set(samples.arr.len()) == {2, None}
|
|
182
189
|
|
|
183
190
|
|
|
184
191
|
def test_sample_struct(generator: Generator) -> None:
|
|
185
192
|
column = dy.Struct({"a": dy.String(regex="[abc]"), "b": dy.String(regex="[a-z]xx")})
|
|
186
193
|
samples = sample_and_validate(column, generator, n=10_000)
|
|
194
|
+
assert samples.is_null().any()
|
|
187
195
|
assert len(samples) == 10_000
|
|
188
196
|
|
|
189
197
|
|
|
@@ -139,6 +139,14 @@ def test_raise_for_list_column(dialect: sa.Dialect) -> None:
|
|
|
139
139
|
dy.List(dy.String()).sqlalchemy_dtype(dialect)
|
|
140
140
|
|
|
141
141
|
|
|
142
|
+
@pytest.mark.parametrize("dialect", [MSDialect_pyodbc(), PGDialect_psycopg2()])
|
|
143
|
+
def test_raise_for_array_column(dialect: sa.Dialect) -> None:
|
|
144
|
+
with pytest.raises(
|
|
145
|
+
NotImplementedError, match="SQL column cannot have 'Array' type."
|
|
146
|
+
):
|
|
147
|
+
dy.Array(dy.String(), 1).sqlalchemy_dtype(dialect)
|
|
148
|
+
|
|
149
|
+
|
|
142
150
|
@pytest.mark.parametrize("dialect", [MSDialect_pyodbc(), PGDialect_psycopg2()])
|
|
143
151
|
def test_raise_for_struct_column(dialect: sa.Dialect) -> None:
|
|
144
152
|
with pytest.raises(
|
|
@@ -24,6 +24,11 @@ def test_string_representation_list() -> None:
|
|
|
24
24
|
assert str(column) == dy.List.__name__.lower()
|
|
25
25
|
|
|
26
26
|
|
|
27
|
+
def test_string_representation_array() -> None:
|
|
28
|
+
column = dy.Array(dy.String(), 1)
|
|
29
|
+
assert str(column) == dy.Array.__name__.lower()
|
|
30
|
+
|
|
31
|
+
|
|
27
32
|
def test_string_representation_struct() -> None:
|
|
28
33
|
column = dy.Struct({"a": dy.String()})
|
|
29
34
|
assert str(column) == dy.Struct.__name__.lower()
|
|
@@ -18,6 +18,12 @@ class MySchema(dy.Schema):
|
|
|
18
18
|
d = dy.Any(alias="e")
|
|
19
19
|
|
|
20
20
|
|
|
21
|
+
class MySchemaWithRule(MySchema):
|
|
22
|
+
@dy.rule()
|
|
23
|
+
def a_greater_than_c() -> pl.Expr:
|
|
24
|
+
return pl.col("a") > pl.col("c")
|
|
25
|
+
|
|
26
|
+
|
|
21
27
|
def test_column_names() -> None:
|
|
22
28
|
assert MySchema.column_names() == ["a", "b", "c", "e"]
|
|
23
29
|
|
|
@@ -75,3 +81,11 @@ def test_col_in_polars_expression() -> None:
|
|
|
75
81
|
.select(MySchema.a.col)
|
|
76
82
|
)
|
|
77
83
|
assert df.row(0) == (1,)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def test_dunder_name() -> None:
|
|
87
|
+
assert MySchema.__name__ == "MySchema"
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def test_dunder_name_with_rule() -> None:
|
|
91
|
+
assert MySchemaWithRule.__name__ == "MySchemaWithRule"
|