dataframely 1.2.0__tar.gz → 1.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (163) hide show
  1. {dataframely-1.2.0 → dataframely-1.3.0}/.copier-answers.yml +1 -1
  2. {dataframely-1.2.0 → dataframely-1.3.0}/.github/workflows/build.yml +1 -1
  3. {dataframely-1.2.0 → dataframely-1.3.0}/Cargo.toml +4 -0
  4. {dataframely-1.2.0 → dataframely-1.3.0}/PKG-INFO +1 -1
  5. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/__init__.py +2 -0
  6. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/_base_schema.py +6 -6
  7. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/_rule.py +3 -1
  8. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/columns/__init__.py +2 -0
  9. dataframely-1.3.0/dataframely/columns/array.py +105 -0
  10. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/columns/enum.py +1 -1
  11. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/columns/list.py +4 -1
  12. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/columns/struct.py +5 -1
  13. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/mypy.py +23 -1
  14. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/random.py +4 -2
  15. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/testing/__init__.py +2 -0
  16. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/testing/const.py +5 -0
  17. {dataframely-1.2.0 → dataframely-1.3.0}/pyproject.toml +1 -1
  18. dataframely-1.3.0/tests/column_types/test_array.py +159 -0
  19. {dataframely-1.2.0 → dataframely-1.3.0}/tests/column_types/test_enum.py +9 -0
  20. {dataframely-1.2.0 → dataframely-1.3.0}/tests/columns/test_default_dtypes.py +1 -0
  21. {dataframely-1.2.0 → dataframely-1.3.0}/tests/columns/test_pyarrow.py +27 -0
  22. {dataframely-1.2.0 → dataframely-1.3.0}/tests/columns/test_sample.py +9 -1
  23. {dataframely-1.2.0 → dataframely-1.3.0}/tests/columns/test_sql_schema.py +8 -0
  24. {dataframely-1.2.0 → dataframely-1.3.0}/tests/columns/test_str.py +5 -0
  25. {dataframely-1.2.0 → dataframely-1.3.0}/tests/schema/test_base.py +14 -0
  26. {dataframely-1.2.0 → dataframely-1.3.0}/tests/schema/test_validate.py +9 -0
  27. {dataframely-1.2.0 → dataframely-1.3.0}/tests/test_typing.py +15 -0
  28. dataframely-1.2.0/.github/workflows/update-lockfiles.yml +0 -34
  29. {dataframely-1.2.0 → dataframely-1.3.0}/.envrc +0 -0
  30. {dataframely-1.2.0 → dataframely-1.3.0}/.gitattributes +0 -0
  31. {dataframely-1.2.0 → dataframely-1.3.0}/.github/CODEOWNERS +0 -0
  32. {dataframely-1.2.0 → dataframely-1.3.0}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
  33. {dataframely-1.2.0 → dataframely-1.3.0}/.github/dependabot.yml +0 -0
  34. {dataframely-1.2.0 → dataframely-1.3.0}/.github/release-drafter.yml +0 -0
  35. {dataframely-1.2.0 → dataframely-1.3.0}/.github/workflows/chore.yml +0 -0
  36. {dataframely-1.2.0 → dataframely-1.3.0}/.github/workflows/ci.yml +0 -0
  37. {dataframely-1.2.0 → dataframely-1.3.0}/.gitignore +0 -0
  38. {dataframely-1.2.0 → dataframely-1.3.0}/.pre-commit-config.yaml +0 -0
  39. {dataframely-1.2.0 → dataframely-1.3.0}/.prettierignore +0 -0
  40. {dataframely-1.2.0 → dataframely-1.3.0}/.prettierrc +0 -0
  41. {dataframely-1.2.0 → dataframely-1.3.0}/.readthedocs.yml +0 -0
  42. {dataframely-1.2.0 → dataframely-1.3.0}/Cargo.lock +0 -0
  43. {dataframely-1.2.0 → dataframely-1.3.0}/LICENSE +0 -0
  44. {dataframely-1.2.0 → dataframely-1.3.0}/README.md +0 -0
  45. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/_base_collection.py +0 -0
  46. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/_compat.py +0 -0
  47. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/_extre.pyi +0 -0
  48. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/_filter.py +0 -0
  49. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/_polars.py +0 -0
  50. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/_typing.py +0 -0
  51. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/_validation.py +0 -0
  52. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/collection.py +0 -0
  53. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/columns/_base.py +0 -0
  54. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/columns/_mixins.py +0 -0
  55. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/columns/_utils.py +0 -0
  56. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/columns/any.py +0 -0
  57. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/columns/bool.py +0 -0
  58. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/columns/datetime.py +0 -0
  59. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/columns/decimal.py +0 -0
  60. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/columns/float.py +0 -0
  61. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/columns/integer.py +0 -0
  62. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/columns/object.py +0 -0
  63. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/columns/string.py +0 -0
  64. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/config.py +0 -0
  65. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/exc.py +0 -0
  66. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/failure.py +0 -0
  67. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/functional.py +0 -0
  68. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/py.typed +0 -0
  69. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/schema.py +0 -0
  70. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/testing/factory.py +0 -0
  71. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/testing/mask.py +0 -0
  72. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/testing/rules.py +0 -0
  73. {dataframely-1.2.0 → dataframely-1.3.0}/dataframely/testing/typing.py +0 -0
  74. {dataframely-1.2.0 → dataframely-1.3.0}/docker-compose.yml +0 -0
  75. {dataframely-1.2.0 → dataframely-1.3.0}/docs/Makefile +0 -0
  76. {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.collection.rst +0 -0
  77. {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.columns.any.rst +0 -0
  78. {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.columns.bool.rst +0 -0
  79. {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.columns.datetime.rst +0 -0
  80. {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.columns.decimal.rst +0 -0
  81. {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.columns.enum.rst +0 -0
  82. {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.columns.float.rst +0 -0
  83. {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.columns.integer.rst +0 -0
  84. {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.columns.list.rst +0 -0
  85. {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.columns.rst +0 -0
  86. {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.columns.string.rst +0 -0
  87. {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.columns.struct.rst +0 -0
  88. {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.config.rst +0 -0
  89. {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.exc.rst +0 -0
  90. {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.failure.rst +0 -0
  91. {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.functional.rst +0 -0
  92. {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.mypy.rst +0 -0
  93. {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.random.rst +0 -0
  94. {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.rst +0 -0
  95. {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.schema.rst +0 -0
  96. {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.testing.const.rst +0 -0
  97. {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.testing.factory.rst +0 -0
  98. {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.testing.mask.rst +0 -0
  99. {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.testing.rst +0 -0
  100. {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.testing.rules.rst +0 -0
  101. {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/dataframely.testing.typing.rst +0 -0
  102. {dataframely-1.2.0 → dataframely-1.3.0}/docs/_api/modules.rst +0 -0
  103. {dataframely-1.2.0 → dataframely-1.3.0}/docs/_static/custom.css +0 -0
  104. {dataframely-1.2.0 → dataframely-1.3.0}/docs/_static/favicon.ico +0 -0
  105. {dataframely-1.2.0 → dataframely-1.3.0}/docs/conf.py +0 -0
  106. {dataframely-1.2.0 → dataframely-1.3.0}/docs/index.rst +0 -0
  107. {dataframely-1.2.0 → dataframely-1.3.0}/docs/make.bat +0 -0
  108. {dataframely-1.2.0 → dataframely-1.3.0}/docs/sites/development.rst +0 -0
  109. {dataframely-1.2.0 → dataframely-1.3.0}/docs/sites/examples/real-world.ipynb +0 -0
  110. {dataframely-1.2.0 → dataframely-1.3.0}/docs/sites/faq.rst +0 -0
  111. {dataframely-1.2.0 → dataframely-1.3.0}/docs/sites/installation.rst +0 -0
  112. {dataframely-1.2.0 → dataframely-1.3.0}/docs/sites/quickstart.rst +0 -0
  113. {dataframely-1.2.0 → dataframely-1.3.0}/pixi.lock +0 -0
  114. {dataframely-1.2.0 → dataframely-1.3.0}/pixi.toml +0 -0
  115. {dataframely-1.2.0 → dataframely-1.3.0}/src/errdefs.rs +0 -0
  116. {dataframely-1.2.0 → dataframely-1.3.0}/src/lib.rs +0 -0
  117. {dataframely-1.2.0 → dataframely-1.3.0}/src/regex_repr.rs +0 -0
  118. {dataframely-1.2.0 → dataframely-1.3.0}/tests/collection/test_base.py +0 -0
  119. {dataframely-1.2.0 → dataframely-1.3.0}/tests/collection/test_cast.py +0 -0
  120. {dataframely-1.2.0 → dataframely-1.3.0}/tests/collection/test_create_empty.py +0 -0
  121. {dataframely-1.2.0 → dataframely-1.3.0}/tests/collection/test_filter_one_to_n.py +0 -0
  122. {dataframely-1.2.0 → dataframely-1.3.0}/tests/collection/test_filter_validate.py +0 -0
  123. {dataframely-1.2.0 → dataframely-1.3.0}/tests/collection/test_ignore_in_filter.py +0 -0
  124. {dataframely-1.2.0 → dataframely-1.3.0}/tests/collection/test_implementation.py +0 -0
  125. {dataframely-1.2.0 → dataframely-1.3.0}/tests/collection/test_optional_members.py +0 -0
  126. {dataframely-1.2.0 → dataframely-1.3.0}/tests/collection/test_sample.py +0 -0
  127. {dataframely-1.2.0 → dataframely-1.3.0}/tests/collection/test_validate_input.py +0 -0
  128. {dataframely-1.2.0 → dataframely-1.3.0}/tests/column_types/__init__.py +0 -0
  129. {dataframely-1.2.0 → dataframely-1.3.0}/tests/column_types/test_any.py +0 -0
  130. {dataframely-1.2.0 → dataframely-1.3.0}/tests/column_types/test_datetime.py +0 -0
  131. {dataframely-1.2.0 → dataframely-1.3.0}/tests/column_types/test_decimal.py +0 -0
  132. {dataframely-1.2.0 → dataframely-1.3.0}/tests/column_types/test_float.py +0 -0
  133. {dataframely-1.2.0 → dataframely-1.3.0}/tests/column_types/test_integer.py +0 -0
  134. {dataframely-1.2.0 → dataframely-1.3.0}/tests/column_types/test_list.py +0 -0
  135. {dataframely-1.2.0 → dataframely-1.3.0}/tests/column_types/test_object.py +0 -0
  136. {dataframely-1.2.0 → dataframely-1.3.0}/tests/column_types/test_string.py +0 -0
  137. {dataframely-1.2.0 → dataframely-1.3.0}/tests/column_types/test_struct.py +0 -0
  138. {dataframely-1.2.0 → dataframely-1.3.0}/tests/columns/__init__.py +0 -0
  139. {dataframely-1.2.0 → dataframely-1.3.0}/tests/columns/test_alias.py +0 -0
  140. {dataframely-1.2.0 → dataframely-1.3.0}/tests/columns/test_check.py +0 -0
  141. {dataframely-1.2.0 → dataframely-1.3.0}/tests/columns/test_metadata.py +0 -0
  142. {dataframely-1.2.0 → dataframely-1.3.0}/tests/columns/test_polars_schema.py +0 -0
  143. {dataframely-1.2.0 → dataframely-1.3.0}/tests/columns/test_rules.py +0 -0
  144. {dataframely-1.2.0 → dataframely-1.3.0}/tests/columns/test_utils.py +0 -0
  145. {dataframely-1.2.0 → dataframely-1.3.0}/tests/core_validation/__init__.py +0 -0
  146. {dataframely-1.2.0 → dataframely-1.3.0}/tests/core_validation/test_column_validation.py +0 -0
  147. {dataframely-1.2.0 → dataframely-1.3.0}/tests/core_validation/test_dtype_validation.py +0 -0
  148. {dataframely-1.2.0 → dataframely-1.3.0}/tests/core_validation/test_rule_evaluation.py +0 -0
  149. {dataframely-1.2.0 → dataframely-1.3.0}/tests/functional/test_concat.py +0 -0
  150. {dataframely-1.2.0 → dataframely-1.3.0}/tests/functional/test_relationships.py +0 -0
  151. {dataframely-1.2.0 → dataframely-1.3.0}/tests/schema/test_cast.py +0 -0
  152. {dataframely-1.2.0 → dataframely-1.3.0}/tests/schema/test_create_empty.py +0 -0
  153. {dataframely-1.2.0 → dataframely-1.3.0}/tests/schema/test_create_empty_if_none.py +0 -0
  154. {dataframely-1.2.0 → dataframely-1.3.0}/tests/schema/test_filter.py +0 -0
  155. {dataframely-1.2.0 → dataframely-1.3.0}/tests/schema/test_inheritance.py +0 -0
  156. {dataframely-1.2.0 → dataframely-1.3.0}/tests/schema/test_rule_implementation.py +0 -0
  157. {dataframely-1.2.0 → dataframely-1.3.0}/tests/schema/test_sample.py +0 -0
  158. {dataframely-1.2.0 → dataframely-1.3.0}/tests/test_compat.py +0 -0
  159. {dataframely-1.2.0 → dataframely-1.3.0}/tests/test_config.py +0 -0
  160. {dataframely-1.2.0 → dataframely-1.3.0}/tests/test_exc.py +0 -0
  161. {dataframely-1.2.0 → dataframely-1.3.0}/tests/test_extre.py +0 -0
  162. {dataframely-1.2.0 → dataframely-1.3.0}/tests/test_failure_info.py +0 -0
  163. {dataframely-1.2.0 → dataframely-1.3.0}/tests/test_random.py +0 -0
@@ -1,7 +1,7 @@
1
1
  # This file is managed by Copier; DO NOT EDIT OR REMOVE.
2
2
  _commit: v0.3.0
3
3
  _src_path: https://github.com/quantco/copier-template-python-open-source
4
- add_autobump_workflow: true
4
+ add_autobump_workflow: false
5
5
  author_email: oliver.borchert@quantco.com
6
6
  author_name: Oliver Borchert
7
7
  github_url: https://github.com/quantco/dataframely
@@ -78,7 +78,7 @@ jobs:
78
78
  id-token: write
79
79
  environment: pypi
80
80
  steps:
81
- - uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1
81
+ - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
82
82
  with:
83
83
  path: dist
84
84
  merge-multiple: true
@@ -12,3 +12,7 @@ pyo3 = { version = "0.24", features = ["abi3-py311", "extension-module"] }
12
12
  rand = { version = "0.9", features = ["std_rng"] }
13
13
  regex-syntax = "0.8"
14
14
  thiserror = "2.0"
15
+
16
+ [profile.release]
17
+ codegen-units = 1
18
+ lto = true
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dataframely
3
- Version: 1.2.0
3
+ Version: 1.3.0
4
4
  Classifier: Programming Language :: Python :: 3
5
5
  Classifier: Programming Language :: Python :: 3.11
6
6
  Classifier: Programming Language :: Python :: 3.12
@@ -18,6 +18,7 @@ from ._typing import DataFrame, LazyFrame
18
18
  from .collection import Collection
19
19
  from .columns import (
20
20
  Any,
21
+ Array,
21
22
  Bool,
22
23
  Column,
23
24
  Date,
@@ -90,5 +91,6 @@ __all__ = [
90
91
  "String",
91
92
  "Struct",
92
93
  "List",
94
+ "Array",
93
95
  "Object",
94
96
  ]
@@ -98,7 +98,7 @@ class SchemaMeta(ABCMeta):
98
98
  )
99
99
 
100
100
  # 2) Check that the columns referenced in the group rules exist.
101
- for name, rule in result.rules.items():
101
+ for rule_name, rule in result.rules.items():
102
102
  if isinstance(rule, GroupRule):
103
103
  missing_columns = set(rule.group_columns) - set(result.columns)
104
104
  if len(missing_columns) > 0:
@@ -106,7 +106,7 @@ class SchemaMeta(ABCMeta):
106
106
  sorted(f"'{col}'" for col in missing_columns)
107
107
  )
108
108
  raise ImplementationError(
109
- f"Group validation rule '{name}' has been implemented "
109
+ f"Group validation rule '{rule_name}' has been implemented "
110
110
  f"incorrectly. It references {len(missing_columns)} columns "
111
111
  f"which are not in the schema: {missing_list}."
112
112
  )
@@ -116,16 +116,16 @@ class SchemaMeta(ABCMeta):
116
116
  # empty data frame and checking for the evaluated dtypes.
117
117
  if len(result.rules) > 0:
118
118
  lf_empty = pl.LazyFrame(
119
- schema={name: col.dtype for name, col in result.columns.items()}
119
+ schema={col_name: col.dtype for col_name, col in result.columns.items()}
120
120
  )
121
121
  # NOTE: For some reason, `polars` does not yield correct dtypes when calling
122
122
  # `collect_schema()`
123
123
  schema = with_evaluation_rules(lf_empty, result.rules).collect().schema
124
- for name, rule in result.rules.items():
125
- dtype = schema[name]
124
+ for rule_name, rule in result.rules.items():
125
+ dtype = schema[rule_name]
126
126
  if not isinstance(dtype, pl.Boolean):
127
127
  raise RuleImplementationError(
128
- name, dtype, isinstance(rule, GroupRule)
128
+ rule_name, dtype, isinstance(rule, GroupRule)
129
129
  )
130
130
 
131
131
  return super().__new__(mcs, name, bases, namespace, *args, **kwargs)
@@ -126,5 +126,7 @@ def _with_group_rules(lf: pl.LazyFrame, rules: dict[str, GroupRule]) -> pl.LazyF
126
126
  # preserves the order of the left data frame.
127
127
  result = lf
128
128
  for group_columns, frame in group_evaluations.items():
129
- result = result.join(frame, on=list(group_columns), how="left")
129
+ result = result.join(
130
+ frame, on=list(group_columns), how="left", nulls_equal=True
131
+ )
130
132
  return result
@@ -3,6 +3,7 @@
3
3
 
4
4
  from ._base import Column
5
5
  from .any import Any
6
+ from .array import Array
6
7
  from .bool import Bool
7
8
  from .datetime import Date, Datetime, Duration, Time
8
9
  from .decimal import Decimal
@@ -17,6 +18,7 @@ from .struct import Struct
17
18
  __all__ = [
18
19
  "Column",
19
20
  "Any",
21
+ "Array",
20
22
  "Bool",
21
23
  "Date",
22
24
  "Datetime",
@@ -0,0 +1,105 @@
1
+ # Copyright (c) QuantCo 2025-2025
2
+ # SPDX-License-Identifier: BSD-3-Clause
3
+
4
+ from __future__ import annotations
5
+
6
+ import math
7
+ from collections.abc import Callable, Sequence
8
+ from typing import Any, Literal
9
+
10
+ import polars as pl
11
+
12
+ from dataframely._compat import pa, sa, sa_TypeEngine
13
+ from dataframely.random import Generator
14
+
15
+ from ._base import Column
16
+ from .struct import Struct
17
+
18
+
19
+ class Array(Column):
20
+ """A fixed-shape array column."""
21
+
22
+ def __init__(
23
+ self,
24
+ inner: Column,
25
+ shape: int | tuple[int, ...],
26
+ *,
27
+ nullable: bool = True,
28
+ # polars doesn't yet support grouping by arrays,
29
+ # see https://github.com/pola-rs/polars/issues/22574
30
+ primary_key: Literal[False] = False,
31
+ check: Callable[[pl.Expr], pl.Expr] | None = None,
32
+ alias: str | None = None,
33
+ metadata: dict[str, Any] | None = None,
34
+ ):
35
+ """
36
+ Args:
37
+ inner: The inner column type. No validation rules on the inner type are supported yet.
38
+ shape: The shape of the array.
39
+ nullable: Whether this column may contain null values.
40
+ primary_key: Whether this column is part of the primary key of the schema.
41
+ Not yet supported for the Array type.
42
+ check: A custom check to run for this column. Must return a non-aggregated
43
+ boolean expression.
44
+ alias: An overwrite for this column's name which allows for using a column
45
+ name that is not a valid Python identifier. Especially note that setting
46
+ this option does _not_ allow to refer to the column with two different
47
+ names, the specified alias is the only valid name.
48
+ metadata: A dictionary of metadata to attach to the column.
49
+ """
50
+ if inner.primary_key or (
51
+ isinstance(inner, Struct)
52
+ and any(col.primary_key for col in inner.inner.values())
53
+ ):
54
+ raise ValueError(
55
+ "`primary_key=True` is not yet supported for inner types of the Array type."
56
+ )
57
+
58
+ # We disallow validation rules on the inner type since Polars arrays currently don't support .eval(). Converting
59
+ # to a list and calling .list.eval() is possible, however, since the shape can have multiple axes, the recursive
60
+ # conversion could have significant performance impact. Hence, we simply disallow inner validation rules.
61
+ # Another option would be to allow validation rules only for sampling, but not enforce them.
62
+ if inner.validation_rules(pl.lit(None)):
63
+ raise ValueError(
64
+ "Validation rules on the inner type of Array are not yet supported."
65
+ )
66
+
67
+ super().__init__(
68
+ nullable=nullable,
69
+ primary_key=False,
70
+ check=check,
71
+ alias=alias,
72
+ metadata=metadata,
73
+ )
74
+ self.inner = inner
75
+ self.shape = shape if isinstance(shape, tuple) else (shape,)
76
+
77
+ @property
78
+ def dtype(self) -> pl.DataType:
79
+ return pl.Array(self.inner.dtype, self.shape)
80
+
81
+ def sqlalchemy_dtype(self, dialect: sa.Dialect) -> sa_TypeEngine:
82
+ # NOTE: We might want to add support for PostgreSQL's ARRAY type or use JSON in the future.
83
+ raise NotImplementedError("SQL column cannot have 'Array' type.")
84
+
85
+ def _pyarrow_dtype_of_shape(self, shape: Sequence[int]) -> pa.DataType:
86
+ if shape:
87
+ size, *rest = shape
88
+ return pa.list_(self._pyarrow_dtype_of_shape(rest), size)
89
+ else:
90
+ return self.inner.pyarrow_dtype
91
+
92
+ @property
93
+ def pyarrow_dtype(self) -> pa.DataType:
94
+ return self._pyarrow_dtype_of_shape(self.shape)
95
+
96
+ def _sample_unchecked(self, generator: Generator, n: int) -> pl.Series:
97
+ # Sample the inner elements in a flat series
98
+ n_elements = n * math.prod(self.shape)
99
+ all_elements = self.inner.sample(generator, n_elements)
100
+
101
+ # Finally, apply a null mask
102
+ return generator._apply_null_mask(
103
+ all_elements.reshape((n, *self.shape)),
104
+ null_probability=self._null_probability,
105
+ )
@@ -49,7 +49,7 @@ class Enum(Column):
49
49
  alias=alias,
50
50
  metadata=metadata,
51
51
  )
52
- self.categories = categories
52
+ self.categories = list(categories)
53
53
 
54
54
  @property
55
55
  def dtype(self) -> pl.DataType:
@@ -147,4 +147,7 @@ class List(Column):
147
147
  chain([0], element_lengths.cum_sum()), element_lengths
148
148
  )
149
149
  ]
150
- return pl.Series(list_elements)
150
+ # Finally, apply a null mask
151
+ return generator._apply_null_mask(
152
+ pl.Series(list_elements), null_probability=self._null_probability
153
+ )
@@ -96,10 +96,14 @@ class Struct(Column):
96
96
  return pa.struct({name: col.pyarrow_dtype for name, col in self.inner.items()})
97
97
 
98
98
  def _sample_unchecked(self, generator: Generator, n: int) -> pl.Series:
99
- return (
99
+ series = (
100
100
  pl.DataFrame(
101
101
  {name: col.sample(generator, n) for name, col in self.inner.items()}
102
102
  )
103
103
  .select(pl.struct(pl.all()))
104
104
  .to_series()
105
105
  )
106
+ # Apply a null mask.
107
+ return generator._apply_null_mask(
108
+ series, null_probability=self._null_probability
109
+ )
@@ -16,6 +16,7 @@ from mypy.nodes import (
16
16
  MemberExpr,
17
17
  NameExpr,
18
18
  StrExpr,
19
+ TupleExpr,
19
20
  TypeInfo,
20
21
  Var,
21
22
  )
@@ -189,7 +190,28 @@ def _convert_dy_column_to_dtype(
189
190
  ],
190
191
  )
191
192
  return api.named_type("builtins.list")
192
- if column_type == "Any":
193
+ if column_type == "Array":
194
+ if isinstance(column_args[0], CallExpr) and isinstance(
195
+ column_args[0].callee, MemberExpr | NameExpr
196
+ ):
197
+ inner_type = _convert_dy_column_to_dtype(
198
+ api,
199
+ column_args[0].callee.name,
200
+ column_args[0].args,
201
+ )
202
+ # If the array has more than one dimension, return a list of lists of the inner type.
203
+ if len(column_args) > 1 and isinstance(column_args[1], TupleExpr):
204
+ for _ in range(len(column_args[1].items) - 1):
205
+ inner_type = api.named_type(
206
+ "builtins.list",
207
+ [inner_type],
208
+ )
209
+ return api.named_type(
210
+ "builtins.list",
211
+ [inner_type],
212
+ )
213
+ return api.named_type("builtins.list")
214
+ if column_type == "Any" or column_type == "Object":
193
215
  return AnyType(TypeOfAny.explicit)
194
216
  # If we can't infer the type, we default to `Any`.
195
217
  # This is, for example, the case for self-defined types, e.g., via `functools.partial`.
@@ -392,8 +392,10 @@ class Generator:
392
392
  def _apply_null_mask(self, series: pl.Series, null_probability: float) -> pl.Series:
393
393
  if null_probability == 0:
394
394
  return series
395
- null_mask = self.numpy_generator.random(series.len()) < null_probability
396
- return series.scatter(np.where(null_mask)[0], None)
395
+ null_mask = (
396
+ pl.Series(self.numpy_generator.random(series.len())) > null_probability
397
+ )
398
+ return pl.select(pl.when(null_mask).then(series)).to_series()
397
399
 
398
400
 
399
401
  # --------------------------------------- UTILS -------------------------------------- #
@@ -6,6 +6,7 @@ from .const import (
6
6
  COLUMN_TYPES,
7
7
  FLOAT_COLUMN_TYPES,
8
8
  INTEGER_COLUMN_TYPES,
9
+ NO_VALIDATION_COLUMN_TYPES,
9
10
  SUPERTYPE_COLUMN_TYPES,
10
11
  )
11
12
  from .factory import create_collection, create_collection_raw, create_schema
@@ -18,6 +19,7 @@ __all__ = [
18
19
  "FLOAT_COLUMN_TYPES",
19
20
  "INTEGER_COLUMN_TYPES",
20
21
  "SUPERTYPE_COLUMN_TYPES",
22
+ "NO_VALIDATION_COLUMN_TYPES",
21
23
  "create_collection",
22
24
  "create_collection_raw",
23
25
  "create_schema",
@@ -47,3 +47,8 @@ SUPERTYPE_COLUMN_TYPES: list[type[dc.Column]] = [
47
47
  ALL_COLUMN_TYPES: list[type[dc.Column]] = (
48
48
  [dc.Any] + COLUMN_TYPES + SUPERTYPE_COLUMN_TYPES
49
49
  )
50
+
51
+ # The following is a list of column types that, when created with default parameter values, add no validation rules.
52
+ NO_VALIDATION_COLUMN_TYPES: list[type[dc.Column]] = [
53
+ t for t in ALL_COLUMN_TYPES if t not in FLOAT_COLUMN_TYPES
54
+ ]
@@ -19,7 +19,7 @@ description = "A declarative, polars-native data frame validation library"
19
19
  name = "dataframely"
20
20
  readme = "README.md"
21
21
  requires-python = ">=3.11"
22
- version = "1.2.0"
22
+ version = "1.3.0"
23
23
 
24
24
  [project.urls]
25
25
  Repository = "https://github.com/quantco/dataframely"
@@ -0,0 +1,159 @@
1
+ # Copyright (c) QuantCo 2025-2025
2
+ # SPDX-License-Identifier: BSD-3-Clause
3
+
4
+ import polars as pl
5
+ import pytest
6
+
7
+ import dataframely as dy
8
+ from dataframely.columns._base import Column
9
+ from dataframely.testing import create_schema
10
+
11
+
12
+ @pytest.mark.parametrize(
13
+ "inner",
14
+ [
15
+ (dy.Int64()),
16
+ (dy.Integer()),
17
+ ],
18
+ )
19
+ def test_integer_array(inner: Column) -> None:
20
+ schema = create_schema("test", {"a": dy.Array(inner, 1)})
21
+ assert schema.is_valid(
22
+ pl.DataFrame(
23
+ {"a": [[1], [2], [3]]},
24
+ schema={
25
+ "a": pl.Array(pl.Int64, 1),
26
+ },
27
+ )
28
+ )
29
+
30
+
31
+ def test_invalid_inner_type() -> None:
32
+ schema = create_schema("test", {"a": dy.Array(dy.Int64(), 1)})
33
+ assert not schema.is_valid(pl.DataFrame({"a": [["1"], ["2"], ["3"]]}))
34
+
35
+
36
+ def test_invalid_shape() -> None:
37
+ schema = create_schema("test", {"a": dy.Array(dy.Int64(), 2)})
38
+ assert not schema.is_valid(
39
+ pl.DataFrame(
40
+ {"a": [[1], [2], [3]]},
41
+ schema={
42
+ "a": pl.Array(pl.Int64, 1),
43
+ },
44
+ )
45
+ )
46
+
47
+
48
+ @pytest.mark.parametrize(
49
+ ("column", "dtype", "is_valid"),
50
+ [
51
+ (
52
+ dy.Array(dy.Int64(), 1),
53
+ pl.Array(pl.Int64(), 1),
54
+ True,
55
+ ),
56
+ (
57
+ dy.Array(dy.String(), 1),
58
+ pl.Array(pl.Int64(), 1),
59
+ False,
60
+ ),
61
+ (
62
+ dy.Array(dy.String(), 1),
63
+ pl.Array(pl.Int64(), 2),
64
+ False,
65
+ ),
66
+ (
67
+ dy.Array(dy.Int64(), (1,)),
68
+ pl.Array(pl.Int64(), (1,)),
69
+ True,
70
+ ),
71
+ (
72
+ dy.Array(dy.Int64(), (1,)),
73
+ pl.Array(pl.Int64(), (2,)),
74
+ False,
75
+ ),
76
+ (
77
+ dy.Array(dy.String(), 1),
78
+ dy.Array(dy.String(), 1),
79
+ False,
80
+ ),
81
+ (
82
+ dy.Array(dy.String(), 1),
83
+ dy.String(),
84
+ False,
85
+ ),
86
+ (
87
+ dy.Array(dy.String(), 1),
88
+ pl.String(),
89
+ False,
90
+ ),
91
+ (
92
+ dy.Array(dy.Array(dy.String(), 1), 1),
93
+ pl.Array(pl.String(), (1, 1)),
94
+ True,
95
+ ),
96
+ (
97
+ dy.Array(dy.String(), (1, 1)),
98
+ pl.Array(pl.Array(pl.String(), 1), 1),
99
+ True,
100
+ ),
101
+ ],
102
+ )
103
+ def test_validate_dtype(column: Column, dtype: pl.DataType, is_valid: bool) -> None:
104
+ assert column.validate_dtype(dtype) == is_valid
105
+
106
+
107
+ def test_nested_arrays() -> None:
108
+ schema = create_schema("test", {"a": dy.Array(dy.Array(dy.Int64(), 1), 1)})
109
+ assert schema.is_valid(
110
+ pl.DataFrame(
111
+ {"a": [[[1]], [[2]], [[3]]]},
112
+ schema={
113
+ "a": pl.Array(pl.Int64, (1, 1)),
114
+ },
115
+ )
116
+ )
117
+
118
+
119
+ def test_nested_array() -> None:
120
+ schema = create_schema("test", {"a": dy.Array(dy.Array(dy.Int64(), 1), 1)})
121
+ assert schema.is_valid(
122
+ pl.DataFrame(
123
+ {"a": [[[1]], [[2]], [[3]]]},
124
+ schema={
125
+ "a": pl.Array(pl.Int64, (1, 1)),
126
+ },
127
+ )
128
+ )
129
+
130
+
131
+ def test_array_with_inner_pk() -> None:
132
+ with pytest.raises(ValueError):
133
+ column = dy.Array(dy.String(primary_key=True), 2)
134
+ create_schema(
135
+ "test",
136
+ {"a": column},
137
+ )
138
+
139
+
140
+ def test_array_with_rules() -> None:
141
+ with pytest.raises(ValueError):
142
+ create_schema(
143
+ "test", {"a": dy.Array(dy.String(min_length=2, nullable=False), 1)}
144
+ )
145
+
146
+
147
+ def test_outer_nullability() -> None:
148
+ schema = create_schema(
149
+ "test",
150
+ {
151
+ "nullable": dy.Array(
152
+ inner=dy.Integer(),
153
+ shape=1,
154
+ nullable=True,
155
+ )
156
+ },
157
+ )
158
+ df = pl.DataFrame({"nullable": [None, None]})
159
+ schema.validate(df, cast=True)
@@ -52,3 +52,12 @@ def test_valid_cast(
52
52
  schema = create_schema("test", {"a": enum})
53
53
  df = df_type(data)
54
54
  assert schema.is_valid(df, cast=True) == valid
55
+
56
+
57
+ @pytest.mark.parametrize("type1", [list, tuple])
58
+ @pytest.mark.parametrize("type2", [list, tuple])
59
+ def test_different_sequences(type1: type, type2: type) -> None:
60
+ allowed = ["a", "b"]
61
+ S = create_schema("test", {"x": dy.Enum(type1(allowed))})
62
+ df = pl.DataFrame({"x": pl.Series(["a", "b"], dtype=pl.Enum(type2(allowed)))})
63
+ S.validate(df)
@@ -36,6 +36,7 @@ from dataframely.testing import create_schema
36
36
  (dy.UInt64(), pl.UInt64()),
37
37
  (dy.String(), pl.String()),
38
38
  (dy.List(dy.String()), pl.List(pl.String())),
39
+ (dy.Array(dy.String(), 1), pl.Array(pl.String(), 1)),
39
40
  (dy.Struct({"a": dy.String()}), pl.Struct({"a": pl.String()})),
40
41
  (dy.Enum(["a", "b"]), pl.Enum(["a", "b"])),
41
42
  ],
@@ -8,6 +8,7 @@ from dataframely.columns import Column
8
8
  from dataframely.testing import (
9
9
  ALL_COLUMN_TYPES,
10
10
  COLUMN_TYPES,
11
+ NO_VALIDATION_COLUMN_TYPES,
11
12
  SUPERTYPE_COLUMN_TYPES,
12
13
  create_schema,
13
14
  )
@@ -32,6 +33,7 @@ def test_equal_polars_schema_enum() -> None:
32
33
  "inner",
33
34
  [c() for c in ALL_COLUMN_TYPES]
34
35
  + [dy.List(t()) for t in ALL_COLUMN_TYPES]
36
+ + [dy.Array(t(), 1) for t in NO_VALIDATION_COLUMN_TYPES]
35
37
  + [dy.Struct({"a": t()}) for t in ALL_COLUMN_TYPES],
36
38
  )
37
39
  def test_equal_polars_schema_list(inner: Column) -> None:
@@ -41,10 +43,33 @@ def test_equal_polars_schema_list(inner: Column) -> None:
41
43
  assert actual == expected
42
44
 
43
45
 
46
+ @pytest.mark.parametrize(
47
+ "inner",
48
+ [c() for c in NO_VALIDATION_COLUMN_TYPES]
49
+ + [dy.List(t()) for t in NO_VALIDATION_COLUMN_TYPES]
50
+ + [dy.Array(t(), 1) for t in NO_VALIDATION_COLUMN_TYPES]
51
+ + [dy.Struct({"a": t()}) for t in NO_VALIDATION_COLUMN_TYPES],
52
+ )
53
+ @pytest.mark.parametrize(
54
+ "shape",
55
+ [
56
+ 1,
57
+ 0,
58
+ (0, 0),
59
+ ],
60
+ )
61
+ def test_equal_polars_schema_array(inner: Column, shape: int | tuple[int, ...]) -> None:
62
+ schema = create_schema("test", {"a": dy.Array(inner, shape)})
63
+ actual = schema.pyarrow_schema()
64
+ expected = schema.create_empty().to_arrow().schema
65
+ assert actual == expected
66
+
67
+
44
68
  @pytest.mark.parametrize(
45
69
  "inner",
46
70
  [c() for c in ALL_COLUMN_TYPES]
47
71
  + [dy.Struct({"a": t()}) for t in ALL_COLUMN_TYPES]
72
+ + [dy.Array(t(), 1) for t in NO_VALIDATION_COLUMN_TYPES]
48
73
  + [dy.List(t()) for t in ALL_COLUMN_TYPES],
49
74
  )
50
75
  def test_equal_polars_schema_struct(inner: Column) -> None:
@@ -71,6 +96,7 @@ def test_nullability_information_enum(nullable: bool) -> None:
71
96
  "inner",
72
97
  [c() for c in ALL_COLUMN_TYPES]
73
98
  + [dy.List(t()) for t in ALL_COLUMN_TYPES]
99
+ + [dy.Array(t(), 1) for t in NO_VALIDATION_COLUMN_TYPES]
74
100
  + [dy.Struct({"a": t()}) for t in ALL_COLUMN_TYPES],
75
101
  )
76
102
  @pytest.mark.parametrize("nullable", [True, False])
@@ -83,6 +109,7 @@ def test_nullability_information_list(inner: Column, nullable: bool) -> None:
83
109
  "inner",
84
110
  [c() for c in ALL_COLUMN_TYPES]
85
111
  + [dy.Struct({"a": t()}) for t in ALL_COLUMN_TYPES]
112
+ + [dy.Array(t(), 1) for t in NO_VALIDATION_COLUMN_TYPES]
86
113
  + [dy.List(t()) for t in ALL_COLUMN_TYPES],
87
114
  )
88
115
  @pytest.mark.parametrize("nullable", [True, False])
@@ -178,12 +178,20 @@ def test_sample_enum(generator: Generator) -> None:
178
178
  def test_sample_list(generator: Generator) -> None:
179
179
  column = dy.List(dy.String(regex="[abc]"), min_length=5, max_length=10)
180
180
  samples = sample_and_validate(column, generator, n=10_000)
181
- assert set(samples.list.len()) == set(range(5, 11))
181
+ assert set(samples.list.len()) == set(range(5, 11)) | {None}
182
+
183
+
184
+ def test_sample_array(generator: Generator) -> None:
185
+ column = dy.Array(dy.Bool(), (2, 3))
186
+ samples = sample_and_validate(column, generator, n=10_000)
187
+ assert samples.is_null().any()
188
+ assert set(samples.arr.len()) == {2, None}
182
189
 
183
190
 
184
191
  def test_sample_struct(generator: Generator) -> None:
185
192
  column = dy.Struct({"a": dy.String(regex="[abc]"), "b": dy.String(regex="[a-z]xx")})
186
193
  samples = sample_and_validate(column, generator, n=10_000)
194
+ assert samples.is_null().any()
187
195
  assert len(samples) == 10_000
188
196
 
189
197
 
@@ -139,6 +139,14 @@ def test_raise_for_list_column(dialect: sa.Dialect) -> None:
139
139
  dy.List(dy.String()).sqlalchemy_dtype(dialect)
140
140
 
141
141
 
142
+ @pytest.mark.parametrize("dialect", [MSDialect_pyodbc(), PGDialect_psycopg2()])
143
+ def test_raise_for_array_column(dialect: sa.Dialect) -> None:
144
+ with pytest.raises(
145
+ NotImplementedError, match="SQL column cannot have 'Array' type."
146
+ ):
147
+ dy.Array(dy.String(), 1).sqlalchemy_dtype(dialect)
148
+
149
+
142
150
  @pytest.mark.parametrize("dialect", [MSDialect_pyodbc(), PGDialect_psycopg2()])
143
151
  def test_raise_for_struct_column(dialect: sa.Dialect) -> None:
144
152
  with pytest.raises(
@@ -24,6 +24,11 @@ def test_string_representation_list() -> None:
24
24
  assert str(column) == dy.List.__name__.lower()
25
25
 
26
26
 
27
+ def test_string_representation_array() -> None:
28
+ column = dy.Array(dy.String(), 1)
29
+ assert str(column) == dy.Array.__name__.lower()
30
+
31
+
27
32
  def test_string_representation_struct() -> None:
28
33
  column = dy.Struct({"a": dy.String()})
29
34
  assert str(column) == dy.Struct.__name__.lower()
@@ -18,6 +18,12 @@ class MySchema(dy.Schema):
18
18
  d = dy.Any(alias="e")
19
19
 
20
20
 
21
+ class MySchemaWithRule(MySchema):
22
+ @dy.rule()
23
+ def a_greater_than_c() -> pl.Expr:
24
+ return pl.col("a") > pl.col("c")
25
+
26
+
21
27
  def test_column_names() -> None:
22
28
  assert MySchema.column_names() == ["a", "b", "c", "e"]
23
29
 
@@ -75,3 +81,11 @@ def test_col_in_polars_expression() -> None:
75
81
  .select(MySchema.a.col)
76
82
  )
77
83
  assert df.row(0) == (1,)
84
+
85
+
86
+ def test_dunder_name() -> None:
87
+ assert MySchema.__name__ == "MySchema"
88
+
89
+
90
+ def test_dunder_name_with_rule() -> None:
91
+ assert MySchemaWithRule.__name__ == "MySchemaWithRule"