ambers 0.4.0__tar.gz → 0.4.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ambers-0.4.0 → ambers-0.4.2}/.github/workflows/ci.yml +1 -1
- {ambers-0.4.0 → ambers-0.4.2}/.gitignore +1 -0
- {ambers-0.4.0 → ambers-0.4.2}/CHANGELOG.md +17 -0
- {ambers-0.4.0 → ambers-0.4.2}/Cargo.lock +1 -1
- {ambers-0.4.0 → ambers-0.4.2}/Cargo.toml +1 -1
- {ambers-0.4.0 → ambers-0.4.2}/PKG-INFO +37 -1
- {ambers-0.4.0 → ambers-0.4.2}/README.md +36 -0
- {ambers-0.4.0 → ambers-0.4.2}/apply_labels.md +11 -1
- {ambers-0.4.0 → ambers-0.4.2}/pyproject.toml +1 -1
- {ambers-0.4.0 → ambers-0.4.2}/python/ambers/__init__.py +6 -0
- {ambers-0.4.0 → ambers-0.4.2}/python/ambers/__init__.pyi +110 -1
- ambers-0.4.2/python/ambers/_missing.py +108 -0
- {ambers-0.4.0 → ambers-0.4.2}/python/ambers/_transforms.py +33 -20
- ambers-0.4.2/python/ambers/_validate.py +384 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/python/diff.rs +3 -6
- {ambers-0.4.0 → ambers-0.4.2}/tests/test_apply_labels.py +36 -0
- ambers-0.4.2/tests/test_apply_missing.py +312 -0
- ambers-0.4.2/tests/test_validate.py +325 -0
- ambers-0.4.2/validate.md +315 -0
- ambers-0.4.0/refactoring.md +0 -28
- {ambers-0.4.0 → ambers-0.4.2}/.cargo/config.toml +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/.github/workflows/release.yml +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/LICENSE +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/images/ambers-banner-v2.svg +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/images/ambers-banner.svg +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/metadata.md +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/python/ambers/_containers.py +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/python/ambers/_io.py +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/arrow_convert.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/columnar.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/compression/bytecode.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/compression/bytecode_encoder.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/compression/mod.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/compression/zlib.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/constants.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/dictionary.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/document.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/encoding.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/error.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/header.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/info_records/encoding_record.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/info_records/float_info.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/info_records/integer_info.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/info_records/long_string_labels.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/info_records/long_string_missing.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/info_records/long_var_names.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/info_records/mod.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/info_records/mr_sets.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/info_records/var_attributes.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/info_records/var_display.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/info_records/very_long_strings.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/io_utils.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/lib.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/metadata.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/python/conversions.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/python/io.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/python/metadata.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/python/mod.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/scanner.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/value_labels.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/variable.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/writer/data.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/writer/layout.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/writer/mod.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/writer/records.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/src/writer/tests.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/tests/bench_read.py +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/tests/bench_write.py +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/tests/conftest.py +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/tests/diag_zsav.py +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/tests/test_long_strings.py +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/tests/test_medium_write.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/tests/test_metadata.py +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/tests/test_metadata_api.py +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/tests/test_paths.example.py +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/tests/test_temporal.rs +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/tests/test_wide_columns.py +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/tests/test_write_roundtrip.py +0 -0
- {ambers-0.4.0 → ambers-0.4.2}/tests/test_writer_issues.py +0 -0
|
@@ -60,4 +60,4 @@ jobs:
|
|
|
60
60
|
- name: Run Python tests (small/synthetic data only)
|
|
61
61
|
run: |
|
|
62
62
|
uv pip install --system pytest
|
|
63
|
-
pytest tests/test_apply_labels.py tests/test_metadata_api.py tests/test_writer_issues.py -v
|
|
63
|
+
pytest tests/test_apply_labels.py tests/test_apply_missing.py tests/test_validate.py tests/test_metadata_api.py tests/test_writer_issues.py -v -k "not RealFile"
|
|
@@ -2,6 +2,23 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to ambers are documented in this file.
|
|
4
4
|
|
|
5
|
+
## [0.4.2] - 2026-04-07
|
|
6
|
+
|
|
7
|
+
- Add `validate(df, meta)` — check value label quality: unlabeled values (error) and duplicate labels (warning)
|
|
8
|
+
- `ValidationReport` with `is_valid`, `errors`, `warnings`, `raise_if_invalid()`, `to_frame()`
|
|
9
|
+
- Repr truncation: max 10 issues shown, long messages shortened, box width capped at 80 chars
|
|
10
|
+
- Shared pure helpers between `validate()` and `apply_labels()` (logic/policy separation)
|
|
11
|
+
- `columns` + `exclude` can be combined in `validate()` (lenient), stay mutually exclusive in `apply_labels`/`apply_missing` (strict)
|
|
12
|
+
- Add `validate.md` documentation with full API reference and examples
|
|
13
|
+
- 29 tests covering all checks, filtering, repr truncation, and shared helper consistency
|
|
14
|
+
|
|
15
|
+
## [0.4.1] - 2026-04-07
|
|
16
|
+
|
|
17
|
+
- Add `apply_missing(df, meta)` — nullify SPSS user-defined missing value codes (discrete, range, range+discrete)
|
|
18
|
+
- Add `exclude=` parameter to both `apply_labels` and `apply_missing` — skip specific columns, mutually exclusive with `columns=`
|
|
19
|
+
- 35 new tests: 28 for apply_missing (all SPSS missing value combinations), 7 for exclude parameter
|
|
20
|
+
- Add `test_apply_missing.py` to CI pytest
|
|
21
|
+
|
|
5
22
|
## [0.4.0] - 2026-04-07
|
|
6
23
|
|
|
7
24
|
- **Modularity refactor:** split `src/python/mod.rs` (2,299 LOC) into 5 focused submodules: `conversions.rs`, `metadata.rs`, `diff.rs`, `io.rs`, and thin `mod.rs`
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ambers
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.2
|
|
4
4
|
Requires-Dist: polars>=1.3
|
|
5
5
|
License-File: LICENSE
|
|
6
6
|
Summary: Pure Rust SPSS .sav/.zsav reader with Polars DataFrame output
|
|
@@ -83,6 +83,19 @@ labeled = am.apply_labels(df, meta) # Enum dtype (order
|
|
|
83
83
|
labeled.write_excel("survey.xlsx") # Enum auto-casts to String
|
|
84
84
|
labeled = am.apply_labels(df, meta, output="string") # String dtype for export
|
|
85
85
|
labeled = am.apply_labels(df, meta, output="enum_null") # Enum, unmapped → null
|
|
86
|
+
labeled = am.apply_labels(df, meta, exclude=["weight", "id"]) # skip specific columns
|
|
87
|
+
|
|
88
|
+
# Apply missing values — nullify SPSS user-defined missing codes
|
|
89
|
+
clean = am.apply_missing(df, meta) # all columns with specs
|
|
90
|
+
clean = am.apply_missing(df, meta, columns=["Q1", "Q2"]) # specific columns only
|
|
91
|
+
clean = am.apply_missing(df, meta, exclude=["age"]) # skip specific columns
|
|
92
|
+
|
|
93
|
+
# Validate — check value label quality before analysis
|
|
94
|
+
report = am.validate(df, meta)
|
|
95
|
+
print(report) # box-drawing summary
|
|
96
|
+
report.is_valid # True if no errors
|
|
97
|
+
report.raise_if_invalid() # raise if errors found
|
|
98
|
+
report.to_frame() # DataFrame for export
|
|
86
99
|
```
|
|
87
100
|
|
|
88
101
|
`.sav` uses bytecode compression by default, `.zsav` uses zlib. Pass `compression=` to override (`"uncompressed"`, `"bytecode"`, `"zlib"`). Pass `meta=` to preserve all metadata from a prior `read_sav()`, or omit it to infer formats from the DataFrame.
|
|
@@ -143,6 +156,29 @@ labeled = am.apply_labels(df, meta, output="enum_null")
|
|
|
143
156
|
|
|
144
157
|
Numeric columns without value labels are skipped. String columns always pass through unmapped text. See [apply_labels.md](apply_labels.md) for full documentation.
|
|
145
158
|
|
|
159
|
+
### validate
|
|
160
|
+
|
|
161
|
+
Check value label quality before analysis — catch unlabeled values and duplicate labels upfront.
|
|
162
|
+
|
|
163
|
+
```python
|
|
164
|
+
sav = am.read_sav("survey.sav")
|
|
165
|
+
df, meta = sav.data, sav.meta
|
|
166
|
+
|
|
167
|
+
report = am.validate(df, meta)
|
|
168
|
+
print(report) # box-drawing summary
|
|
169
|
+
report.is_valid # True if no errors (warnings OK)
|
|
170
|
+
report.raise_if_invalid() # raise ValueError if errors
|
|
171
|
+
|
|
172
|
+
# Programmatic access
|
|
173
|
+
for error in report.errors:
|
|
174
|
+
print(f"{error.column}: {error.details['unlabeled_values']}")
|
|
175
|
+
|
|
176
|
+
# Export as DataFrame
|
|
177
|
+
report.to_frame().write_csv("validation_issues.csv")
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
See [validate.md](validate.md) for full documentation.
|
|
181
|
+
|
|
146
182
|
## Rust
|
|
147
183
|
|
|
148
184
|
```rust
|
|
@@ -73,6 +73,19 @@ labeled = am.apply_labels(df, meta) # Enum dtype (order
|
|
|
73
73
|
labeled.write_excel("survey.xlsx") # Enum auto-casts to String
|
|
74
74
|
labeled = am.apply_labels(df, meta, output="string") # String dtype for export
|
|
75
75
|
labeled = am.apply_labels(df, meta, output="enum_null") # Enum, unmapped → null
|
|
76
|
+
labeled = am.apply_labels(df, meta, exclude=["weight", "id"]) # skip specific columns
|
|
77
|
+
|
|
78
|
+
# Apply missing values — nullify SPSS user-defined missing codes
|
|
79
|
+
clean = am.apply_missing(df, meta) # all columns with specs
|
|
80
|
+
clean = am.apply_missing(df, meta, columns=["Q1", "Q2"]) # specific columns only
|
|
81
|
+
clean = am.apply_missing(df, meta, exclude=["age"]) # skip specific columns
|
|
82
|
+
|
|
83
|
+
# Validate — check value label quality before analysis
|
|
84
|
+
report = am.validate(df, meta)
|
|
85
|
+
print(report) # box-drawing summary
|
|
86
|
+
report.is_valid # True if no errors
|
|
87
|
+
report.raise_if_invalid() # raise if errors found
|
|
88
|
+
report.to_frame() # DataFrame for export
|
|
76
89
|
```
|
|
77
90
|
|
|
78
91
|
`.sav` uses bytecode compression by default, `.zsav` uses zlib. Pass `compression=` to override (`"uncompressed"`, `"bytecode"`, `"zlib"`). Pass `meta=` to preserve all metadata from a prior `read_sav()`, or omit it to infer formats from the DataFrame.
|
|
@@ -133,6 +146,29 @@ labeled = am.apply_labels(df, meta, output="enum_null")
|
|
|
133
146
|
|
|
134
147
|
Numeric columns without value labels are skipped. String columns always pass through unmapped text. See [apply_labels.md](apply_labels.md) for full documentation.
|
|
135
148
|
|
|
149
|
+
### validate
|
|
150
|
+
|
|
151
|
+
Check value label quality before analysis — catch unlabeled values and duplicate labels upfront.
|
|
152
|
+
|
|
153
|
+
```python
|
|
154
|
+
sav = am.read_sav("survey.sav")
|
|
155
|
+
df, meta = sav.data, sav.meta
|
|
156
|
+
|
|
157
|
+
report = am.validate(df, meta)
|
|
158
|
+
print(report) # box-drawing summary
|
|
159
|
+
report.is_valid # True if no errors (warnings OK)
|
|
160
|
+
report.raise_if_invalid() # raise ValueError if errors
|
|
161
|
+
|
|
162
|
+
# Programmatic access
|
|
163
|
+
for error in report.errors:
|
|
164
|
+
print(f"{error.column}: {error.details['unlabeled_values']}")
|
|
165
|
+
|
|
166
|
+
# Export as DataFrame
|
|
167
|
+
report.to_frame().write_csv("validation_issues.csv")
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
See [validate.md](validate.md) for full documentation.
|
|
171
|
+
|
|
136
172
|
## Rust
|
|
137
173
|
|
|
138
174
|
```rust
|
|
@@ -85,6 +85,7 @@ def apply_labels(
|
|
|
85
85
|
meta: SpssMetadata,
|
|
86
86
|
*,
|
|
87
87
|
columns: list[str] | None = None,
|
|
88
|
+
exclude: list[str] | None = None,
|
|
88
89
|
output: Literal["enum", "string", "enum_null"] = "enum",
|
|
89
90
|
) -> DataFrame | LazyFrame
|
|
90
91
|
```
|
|
@@ -108,7 +109,7 @@ An `SpssMetadata` object containing value labels. Get it from `sav.meta`.
|
|
|
108
109
|
|
|
109
110
|
### `columns`
|
|
110
111
|
|
|
111
|
-
Which columns to apply labels to. Default `None` applies to all columns that have value labels in the metadata AND exist in the data.
|
|
112
|
+
Which columns to apply labels to. Default `None` applies to all columns that have value labels in the metadata AND exist in the data. Mutually exclusive with `exclude`.
|
|
112
113
|
|
|
113
114
|
```python
|
|
114
115
|
# All columns with value labels
|
|
@@ -122,6 +123,15 @@ When specified explicitly, raises `ValueError` if:
|
|
|
122
123
|
- A column doesn't exist in the data
|
|
123
124
|
- A column has no value labels (or an empty label dict) in the metadata
|
|
124
125
|
|
|
126
|
+
### `exclude`
|
|
127
|
+
|
|
128
|
+
Columns to skip. When set, all columns with value labels are processed except those listed here. Mutually exclusive with `columns`.
|
|
129
|
+
|
|
130
|
+
```python
|
|
131
|
+
# Apply labels to everything except weight and ID columns
|
|
132
|
+
am.apply_labels(df, meta, exclude=["weight", "respondent_id"])
|
|
133
|
+
```
|
|
134
|
+
|
|
125
135
|
### `output`
|
|
126
136
|
|
|
127
137
|
Controls both the output dtype and how unmapped values are handled. One parameter, three clear modes:
|
|
@@ -3,11 +3,17 @@
|
|
|
3
3
|
from ambers._ambers import MetaDiff, SpssMetadata
|
|
4
4
|
from ambers._containers import SavFile
|
|
5
5
|
from ambers._io import read_sav, read_sav_meta, scan_sav, write_sav
|
|
6
|
+
from ambers._missing import apply_missing
|
|
6
7
|
from ambers._transforms import apply_labels
|
|
8
|
+
from ambers._validate import ValidationIssue, ValidationReport, validate
|
|
7
9
|
|
|
8
10
|
__all__ = [
|
|
9
11
|
"SavFile",
|
|
10
12
|
"apply_labels",
|
|
13
|
+
"apply_missing",
|
|
14
|
+
"validate",
|
|
15
|
+
"ValidationReport",
|
|
16
|
+
"ValidationIssue",
|
|
11
17
|
"read_sav",
|
|
12
18
|
"read_sav_meta",
|
|
13
19
|
"scan_sav",
|
|
@@ -755,6 +755,7 @@ def apply_labels(
|
|
|
755
755
|
meta: SpssMetadata,
|
|
756
756
|
*,
|
|
757
757
|
columns: list[str] | None = ...,
|
|
758
|
+
exclude: list[str] | None = ...,
|
|
758
759
|
output: str = ...,
|
|
759
760
|
) -> polars.DataFrame: ...
|
|
760
761
|
@overload
|
|
@@ -763,6 +764,7 @@ def apply_labels(
|
|
|
763
764
|
meta: SpssMetadata,
|
|
764
765
|
*,
|
|
765
766
|
columns: list[str] | None = ...,
|
|
767
|
+
exclude: list[str] | None = ...,
|
|
766
768
|
output: str = ...,
|
|
767
769
|
) -> polars.LazyFrame: ...
|
|
768
770
|
def apply_labels(
|
|
@@ -770,6 +772,7 @@ def apply_labels(
|
|
|
770
772
|
meta: SpssMetadata,
|
|
771
773
|
*,
|
|
772
774
|
columns: list[str] | None = None,
|
|
775
|
+
exclude: list[str] | None = None,
|
|
773
776
|
output: str = "enum",
|
|
774
777
|
) -> polars.DataFrame | polars.LazyFrame:
|
|
775
778
|
"""Replace numeric/string codes with value labels from SPSS metadata.
|
|
@@ -785,7 +788,10 @@ def apply_labels(
|
|
|
785
788
|
SpssMetadata with value labels.
|
|
786
789
|
columns
|
|
787
790
|
Columns to apply labels to. None applies to all columns with
|
|
788
|
-
value labels in the metadata.
|
|
791
|
+
value labels in the metadata. Mutually exclusive with ``exclude``.
|
|
792
|
+
exclude
|
|
793
|
+
Columns to skip. All columns with value labels are processed
|
|
794
|
+
except those listed here. Mutually exclusive with ``columns``.
|
|
789
795
|
output
|
|
790
796
|
Output mode for labeled numeric columns.
|
|
791
797
|
|
|
@@ -808,3 +814,106 @@ def apply_labels(
|
|
|
808
814
|
>>> labeled.write_excel("survey.xlsx")
|
|
809
815
|
"""
|
|
810
816
|
...
|
|
817
|
+
|
|
818
|
+
def apply_missing(
|
|
819
|
+
df: pl.DataFrame | pl.LazyFrame,
|
|
820
|
+
meta: SpssMetadata,
|
|
821
|
+
*,
|
|
822
|
+
columns: list[str] | None = None,
|
|
823
|
+
exclude: list[str] | None = None,
|
|
824
|
+
) -> pl.DataFrame | pl.LazyFrame:
|
|
825
|
+
"""Replace SPSS user-defined missing value codes with null.
|
|
826
|
+
|
|
827
|
+
Converts values that match the missing value specifications in
|
|
828
|
+
``meta.variable_missing_values`` to null. Handles discrete values,
|
|
829
|
+
ranges, and range-plus-discrete combinations.
|
|
830
|
+
|
|
831
|
+
**Scope:** User-defined missing values only. SPSS system missing
|
|
832
|
+
(SYSMIS) is already read as null by the reader.
|
|
833
|
+
|
|
834
|
+
Parameters
|
|
835
|
+
----------
|
|
836
|
+
df
|
|
837
|
+
A Polars DataFrame or LazyFrame with raw SPSS data.
|
|
838
|
+
meta
|
|
839
|
+
SpssMetadata with missing value definitions.
|
|
840
|
+
columns
|
|
841
|
+
Columns to apply missing values to. None applies to all
|
|
842
|
+
columns with missing value specs in metadata. Columns not
|
|
843
|
+
found in the DataFrame or without specs are silently skipped.
|
|
844
|
+
Mutually exclusive with ``exclude``.
|
|
845
|
+
exclude
|
|
846
|
+
Columns to skip. All columns with missing value specs are
|
|
847
|
+
processed except those listed here. Mutually exclusive with
|
|
848
|
+
``columns``.
|
|
849
|
+
|
|
850
|
+
Returns
|
|
851
|
+
-------
|
|
852
|
+
polars.DataFrame | polars.LazyFrame
|
|
853
|
+
|
|
854
|
+
Examples
|
|
855
|
+
--------
|
|
856
|
+
>>> sav = am.read_sav("survey.sav")
|
|
857
|
+
>>> df, meta = sav.data, sav.meta
|
|
858
|
+
>>> clean = am.apply_missing(df, meta)
|
|
859
|
+
>>> clean.null_count()
|
|
860
|
+
"""
|
|
861
|
+
...
|
|
862
|
+
|
|
863
|
+
class ValidationIssue:
|
|
864
|
+
severity: str
|
|
865
|
+
column: str
|
|
866
|
+
check: str
|
|
867
|
+
message: str
|
|
868
|
+
details: dict
|
|
869
|
+
|
|
870
|
+
class ValidationReport:
|
|
871
|
+
issues: list[ValidationIssue]
|
|
872
|
+
|
|
873
|
+
@property
|
|
874
|
+
def is_valid(self) -> bool: ...
|
|
875
|
+
@property
|
|
876
|
+
def errors(self) -> list[ValidationIssue]: ...
|
|
877
|
+
@property
|
|
878
|
+
def warnings(self) -> list[ValidationIssue]: ...
|
|
879
|
+
def raise_if_invalid(self) -> None: ...
|
|
880
|
+
def to_frame(self) -> pl.DataFrame: ...
|
|
881
|
+
|
|
882
|
+
def validate(
|
|
883
|
+
df: pl.DataFrame | pl.LazyFrame,
|
|
884
|
+
meta: SpssMetadata,
|
|
885
|
+
*,
|
|
886
|
+
columns: list[str] | None = None,
|
|
887
|
+
exclude: list[str] | None = None,
|
|
888
|
+
) -> ValidationReport:
|
|
889
|
+
"""Validate SPSS value label quality.
|
|
890
|
+
|
|
891
|
+
Checks numeric columns with value labels for unlabeled values
|
|
892
|
+
(error) and duplicate labels (warning). Columns without value
|
|
893
|
+
labels are skipped.
|
|
894
|
+
|
|
895
|
+
Parameters
|
|
896
|
+
----------
|
|
897
|
+
df
|
|
898
|
+
A Polars DataFrame or LazyFrame.
|
|
899
|
+
meta
|
|
900
|
+
SpssMetadata with value labels.
|
|
901
|
+
columns
|
|
902
|
+
Columns to check. None checks all numeric columns with value
|
|
903
|
+
labels. Can be combined with ``exclude``.
|
|
904
|
+
exclude
|
|
905
|
+
Columns to skip. Applied after ``columns`` filtering.
|
|
906
|
+
|
|
907
|
+
Returns
|
|
908
|
+
-------
|
|
909
|
+
ValidationReport
|
|
910
|
+
|
|
911
|
+
Examples
|
|
912
|
+
--------
|
|
913
|
+
>>> sav = am.read_sav("survey.sav")
|
|
914
|
+
>>> df, meta = sav.data, sav.meta
|
|
915
|
+
>>> report = am.validate(df, meta)
|
|
916
|
+
>>> report.is_valid
|
|
917
|
+
>>> report.raise_if_invalid()
|
|
918
|
+
"""
|
|
919
|
+
...
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""Apply SPSS user-defined missing value specs to DataFrames."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import polars as pl
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def apply_missing(
|
|
9
|
+
df: pl.DataFrame | pl.LazyFrame,
|
|
10
|
+
meta,
|
|
11
|
+
*,
|
|
12
|
+
columns: list[str] | None = None,
|
|
13
|
+
exclude: list[str] | None = None,
|
|
14
|
+
) -> pl.DataFrame | pl.LazyFrame:
|
|
15
|
+
"""Replace SPSS user-defined missing value codes with null.
|
|
16
|
+
|
|
17
|
+
Converts values that match the missing value specifications in
|
|
18
|
+
``meta.variable_missing_values`` to null. This handles discrete
|
|
19
|
+
values, ranges, and range-plus-discrete combinations as defined
|
|
20
|
+
in the SPSS file.
|
|
21
|
+
|
|
22
|
+
**Scope:** User-defined missing values only (discrete, range,
|
|
23
|
+
range+discrete). SPSS system missing (SYSMIS) is already read
|
|
24
|
+
as null by the reader and is not affected by this function.
|
|
25
|
+
|
|
26
|
+
Args:
|
|
27
|
+
df: A Polars DataFrame or LazyFrame with raw SPSS data.
|
|
28
|
+
meta: An ``SpssMetadata`` object with missing value definitions.
|
|
29
|
+
columns: Columns to apply missing values to. ``None`` applies
|
|
30
|
+
to all columns that have missing value specs in metadata.
|
|
31
|
+
Columns not found in the DataFrame or without specs are
|
|
32
|
+
silently skipped. Mutually exclusive with ``exclude``.
|
|
33
|
+
exclude: Columns to skip. When set, all columns with missing
|
|
34
|
+
value specs are processed except those listed here.
|
|
35
|
+
Mutually exclusive with ``columns``.
|
|
36
|
+
|
|
37
|
+
Returns:
|
|
38
|
+
DataFrame or LazyFrame (same type as input) with missing value
|
|
39
|
+
codes replaced by null. Existing nulls are preserved.
|
|
40
|
+
"""
|
|
41
|
+
if not isinstance(df, (pl.DataFrame, pl.LazyFrame)):
|
|
42
|
+
raise TypeError(
|
|
43
|
+
f"df must be a polars DataFrame or LazyFrame, got {type(df).__name__}"
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
if columns is not None and exclude is not None:
|
|
47
|
+
raise ValueError("columns and exclude are mutually exclusive")
|
|
48
|
+
|
|
49
|
+
missing_specs = meta.variable_missing_values
|
|
50
|
+
if not missing_specs:
|
|
51
|
+
return df
|
|
52
|
+
|
|
53
|
+
# Determine which columns to process
|
|
54
|
+
schema = df.collect_schema() if isinstance(df, pl.LazyFrame) else df.schema
|
|
55
|
+
df_columns = set(schema.names())
|
|
56
|
+
|
|
57
|
+
if columns is not None:
|
|
58
|
+
target_cols = [c for c in columns if c in df_columns and c in missing_specs]
|
|
59
|
+
else:
|
|
60
|
+
target_cols = [c for c in missing_specs if c in df_columns]
|
|
61
|
+
if exclude is not None:
|
|
62
|
+
exclude_set = set(exclude)
|
|
63
|
+
target_cols = [c for c in target_cols if c not in exclude_set]
|
|
64
|
+
|
|
65
|
+
if not target_cols:
|
|
66
|
+
return df
|
|
67
|
+
|
|
68
|
+
# Build one expression per column
|
|
69
|
+
exprs = []
|
|
70
|
+
for name in target_cols:
|
|
71
|
+
spec = missing_specs[name]
|
|
72
|
+
condition = _build_condition(name, spec)
|
|
73
|
+
if condition is not None:
|
|
74
|
+
exprs.append(
|
|
75
|
+
pl.when(condition)
|
|
76
|
+
.then(pl.lit(None))
|
|
77
|
+
.otherwise(pl.col(name))
|
|
78
|
+
.alias(name)
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
if not exprs:
|
|
82
|
+
return df
|
|
83
|
+
|
|
84
|
+
return df.with_columns(exprs)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _build_condition(name: str, spec: dict) -> pl.Expr | None:
|
|
88
|
+
"""Build a Polars boolean expression for a missing value spec."""
|
|
89
|
+
spec_type = spec.get("type")
|
|
90
|
+
col = pl.col(name)
|
|
91
|
+
|
|
92
|
+
if spec_type == "discrete":
|
|
93
|
+
values = spec.get("values", [])
|
|
94
|
+
if not values:
|
|
95
|
+
return None
|
|
96
|
+
return col.is_in(values)
|
|
97
|
+
|
|
98
|
+
elif spec_type == "range":
|
|
99
|
+
lo = spec["low"]
|
|
100
|
+
hi = spec["high"]
|
|
101
|
+
condition = col.is_between(lo, hi, closed="both")
|
|
102
|
+
# Range + discrete combination
|
|
103
|
+
discrete = spec.get("discrete")
|
|
104
|
+
if discrete is not None:
|
|
105
|
+
condition = condition | (col == discrete)
|
|
106
|
+
return condition
|
|
107
|
+
|
|
108
|
+
return None
|
|
@@ -17,6 +17,7 @@ def apply_labels(
|
|
|
17
17
|
meta,
|
|
18
18
|
*,
|
|
19
19
|
columns: list[str] | None = None,
|
|
20
|
+
exclude: list[str] | None = None,
|
|
20
21
|
output: str = "enum",
|
|
21
22
|
) -> pl.DataFrame | pl.LazyFrame:
|
|
22
23
|
"""Replace numeric/string codes with value labels from SPSS metadata.
|
|
@@ -45,7 +46,10 @@ def apply_labels(
|
|
|
45
46
|
columns: Columns to apply labels to. ``None`` applies to all
|
|
46
47
|
columns that have value labels in the metadata. When specified
|
|
47
48
|
explicitly, raises if columns are missing from data or have
|
|
48
|
-
no value labels defined.
|
|
49
|
+
no value labels defined. Mutually exclusive with ``exclude``.
|
|
50
|
+
exclude: Columns to skip. When set, all columns with value
|
|
51
|
+
labels are processed except those listed here. Mutually
|
|
52
|
+
exclusive with ``columns``.
|
|
49
53
|
output: Output mode for labeled numeric columns.
|
|
50
54
|
|
|
51
55
|
- ``"enum"`` (default) — ``pl.Enum`` with ordered categories.
|
|
@@ -89,6 +93,9 @@ def apply_labels(
|
|
|
89
93
|
)
|
|
90
94
|
as_enum, unmapped = _OUTPUT_MODES[output]
|
|
91
95
|
|
|
96
|
+
if columns is not None and exclude is not None:
|
|
97
|
+
raise ValueError("columns and exclude are mutually exclusive")
|
|
98
|
+
|
|
92
99
|
# Get schema + determine target columns
|
|
93
100
|
schema = (
|
|
94
101
|
df.collect_schema() if isinstance(df, pl.LazyFrame) else df.schema
|
|
@@ -106,6 +113,9 @@ def apply_labels(
|
|
|
106
113
|
target = columns
|
|
107
114
|
else:
|
|
108
115
|
target = [c for c in all_labels if c in data_cols]
|
|
116
|
+
if exclude is not None:
|
|
117
|
+
exclude_set = set(exclude)
|
|
118
|
+
target = [c for c in target if c not in exclude_set]
|
|
109
119
|
|
|
110
120
|
if not target:
|
|
111
121
|
return df
|
|
@@ -189,9 +199,12 @@ def _validate_labels(
|
|
|
189
199
|
) -> None:
|
|
190
200
|
"""Pre-check for unmapped values and duplicate labels on numeric columns.
|
|
191
201
|
|
|
202
|
+
Uses shared helpers from _validate.py for the actual checks.
|
|
192
203
|
Raises ValueError with a structured diagnostic if issues are found.
|
|
193
204
|
Only called when unmapped="error" or as_enum=True.
|
|
194
205
|
"""
|
|
206
|
+
from ambers._validate import check_duplicate_labels, check_unlabeled_values
|
|
207
|
+
|
|
195
208
|
unmapped_issues: list[str] = []
|
|
196
209
|
duplicate_issues: list[str] = []
|
|
197
210
|
|
|
@@ -199,19 +212,21 @@ def _validate_labels(
|
|
|
199
212
|
labels = all_labels[col_name]
|
|
200
213
|
label_keys = set(labels.keys())
|
|
201
214
|
|
|
202
|
-
# Check unmapped values
|
|
215
|
+
# Check unmapped values (using shared helper)
|
|
203
216
|
if unmapped == "error":
|
|
204
|
-
col_expr = pl.col(col_name).drop_nulls().unique()
|
|
205
217
|
if isinstance(df, pl.LazyFrame):
|
|
206
218
|
unique_vals = set(
|
|
207
|
-
df.select(
|
|
219
|
+
df.select(pl.col(col_name).drop_nulls().unique())
|
|
220
|
+
.collect()[col_name].to_list()
|
|
208
221
|
)
|
|
209
222
|
else:
|
|
210
|
-
unique_vals = set(
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
223
|
+
unique_vals = set(
|
|
224
|
+
df.select(pl.col(col_name).drop_nulls().unique())
|
|
225
|
+
[col_name].to_list()
|
|
226
|
+
)
|
|
227
|
+
result = check_unlabeled_values(col_name, unique_vals, label_keys)
|
|
228
|
+
if result is not None:
|
|
229
|
+
missing = result["unlabeled_values"]
|
|
215
230
|
vals_str = ", ".join(str(v) for v in missing[:10])
|
|
216
231
|
if len(missing) > 10:
|
|
217
232
|
vals_str += f", ... ({len(missing)} total)"
|
|
@@ -219,30 +234,28 @@ def _validate_labels(
|
|
|
219
234
|
f" {col_name}: {len(missing)} unmapped "
|
|
220
235
|
f"{'value' if len(missing) == 1 else 'values'}: "
|
|
221
236
|
f"[{vals_str}] "
|
|
222
|
-
f"({
|
|
237
|
+
f"({result['unique_in_data']} unique, "
|
|
238
|
+
f"{len(label_keys)} labeled)"
|
|
223
239
|
)
|
|
224
240
|
|
|
225
|
-
# Check duplicate label values
|
|
241
|
+
# Check duplicate label values (using shared helper)
|
|
226
242
|
if as_enum:
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
seen.setdefault(v, []).append(k)
|
|
230
|
-
dupes = {v: keys for v, keys in seen.items() if len(keys) > 1}
|
|
231
|
-
if dupes:
|
|
243
|
+
result = check_duplicate_labels(col_name, labels)
|
|
244
|
+
if result is not None:
|
|
232
245
|
parts = []
|
|
233
|
-
for label_val, keys in
|
|
246
|
+
for label_val, keys in result["duplicates"].items():
|
|
234
247
|
keys_str = ", ".join(str(k) for k in keys)
|
|
235
248
|
parts.append(f"[{keys_str}] -> {label_val!r}")
|
|
236
249
|
duplicate_issues.append(
|
|
237
|
-
f" {col_name}: {len(
|
|
238
|
-
f"{'label' if len(
|
|
250
|
+
f" {col_name}: {len(result['duplicates'])} duplicate "
|
|
251
|
+
f"{'label' if len(result['duplicates']) == 1 else 'labels'}: "
|
|
239
252
|
+ "; ".join(parts)
|
|
240
253
|
)
|
|
241
254
|
|
|
242
255
|
if not unmapped_issues and not duplicate_issues:
|
|
243
256
|
return
|
|
244
257
|
|
|
245
|
-
# Build structured error message
|
|
258
|
+
# Build structured error message (preserve existing format)
|
|
246
259
|
msg_parts: list[str] = []
|
|
247
260
|
|
|
248
261
|
if unmapped_issues:
|