ambers 0.4.0__tar.gz → 0.4.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. {ambers-0.4.0 → ambers-0.4.2}/.github/workflows/ci.yml +1 -1
  2. {ambers-0.4.0 → ambers-0.4.2}/.gitignore +1 -0
  3. {ambers-0.4.0 → ambers-0.4.2}/CHANGELOG.md +17 -0
  4. {ambers-0.4.0 → ambers-0.4.2}/Cargo.lock +1 -1
  5. {ambers-0.4.0 → ambers-0.4.2}/Cargo.toml +1 -1
  6. {ambers-0.4.0 → ambers-0.4.2}/PKG-INFO +37 -1
  7. {ambers-0.4.0 → ambers-0.4.2}/README.md +36 -0
  8. {ambers-0.4.0 → ambers-0.4.2}/apply_labels.md +11 -1
  9. {ambers-0.4.0 → ambers-0.4.2}/pyproject.toml +1 -1
  10. {ambers-0.4.0 → ambers-0.4.2}/python/ambers/__init__.py +6 -0
  11. {ambers-0.4.0 → ambers-0.4.2}/python/ambers/__init__.pyi +110 -1
  12. ambers-0.4.2/python/ambers/_missing.py +108 -0
  13. {ambers-0.4.0 → ambers-0.4.2}/python/ambers/_transforms.py +33 -20
  14. ambers-0.4.2/python/ambers/_validate.py +384 -0
  15. {ambers-0.4.0 → ambers-0.4.2}/src/python/diff.rs +3 -6
  16. {ambers-0.4.0 → ambers-0.4.2}/tests/test_apply_labels.py +36 -0
  17. ambers-0.4.2/tests/test_apply_missing.py +312 -0
  18. ambers-0.4.2/tests/test_validate.py +325 -0
  19. ambers-0.4.2/validate.md +315 -0
  20. ambers-0.4.0/refactoring.md +0 -28
  21. {ambers-0.4.0 → ambers-0.4.2}/.cargo/config.toml +0 -0
  22. {ambers-0.4.0 → ambers-0.4.2}/.github/workflows/release.yml +0 -0
  23. {ambers-0.4.0 → ambers-0.4.2}/LICENSE +0 -0
  24. {ambers-0.4.0 → ambers-0.4.2}/images/ambers-banner-v2.svg +0 -0
  25. {ambers-0.4.0 → ambers-0.4.2}/images/ambers-banner.svg +0 -0
  26. {ambers-0.4.0 → ambers-0.4.2}/metadata.md +0 -0
  27. {ambers-0.4.0 → ambers-0.4.2}/python/ambers/_containers.py +0 -0
  28. {ambers-0.4.0 → ambers-0.4.2}/python/ambers/_io.py +0 -0
  29. {ambers-0.4.0 → ambers-0.4.2}/src/arrow_convert.rs +0 -0
  30. {ambers-0.4.0 → ambers-0.4.2}/src/columnar.rs +0 -0
  31. {ambers-0.4.0 → ambers-0.4.2}/src/compression/bytecode.rs +0 -0
  32. {ambers-0.4.0 → ambers-0.4.2}/src/compression/bytecode_encoder.rs +0 -0
  33. {ambers-0.4.0 → ambers-0.4.2}/src/compression/mod.rs +0 -0
  34. {ambers-0.4.0 → ambers-0.4.2}/src/compression/zlib.rs +0 -0
  35. {ambers-0.4.0 → ambers-0.4.2}/src/constants.rs +0 -0
  36. {ambers-0.4.0 → ambers-0.4.2}/src/dictionary.rs +0 -0
  37. {ambers-0.4.0 → ambers-0.4.2}/src/document.rs +0 -0
  38. {ambers-0.4.0 → ambers-0.4.2}/src/encoding.rs +0 -0
  39. {ambers-0.4.0 → ambers-0.4.2}/src/error.rs +0 -0
  40. {ambers-0.4.0 → ambers-0.4.2}/src/header.rs +0 -0
  41. {ambers-0.4.0 → ambers-0.4.2}/src/info_records/encoding_record.rs +0 -0
  42. {ambers-0.4.0 → ambers-0.4.2}/src/info_records/float_info.rs +0 -0
  43. {ambers-0.4.0 → ambers-0.4.2}/src/info_records/integer_info.rs +0 -0
  44. {ambers-0.4.0 → ambers-0.4.2}/src/info_records/long_string_labels.rs +0 -0
  45. {ambers-0.4.0 → ambers-0.4.2}/src/info_records/long_string_missing.rs +0 -0
  46. {ambers-0.4.0 → ambers-0.4.2}/src/info_records/long_var_names.rs +0 -0
  47. {ambers-0.4.0 → ambers-0.4.2}/src/info_records/mod.rs +0 -0
  48. {ambers-0.4.0 → ambers-0.4.2}/src/info_records/mr_sets.rs +0 -0
  49. {ambers-0.4.0 → ambers-0.4.2}/src/info_records/var_attributes.rs +0 -0
  50. {ambers-0.4.0 → ambers-0.4.2}/src/info_records/var_display.rs +0 -0
  51. {ambers-0.4.0 → ambers-0.4.2}/src/info_records/very_long_strings.rs +0 -0
  52. {ambers-0.4.0 → ambers-0.4.2}/src/io_utils.rs +0 -0
  53. {ambers-0.4.0 → ambers-0.4.2}/src/lib.rs +0 -0
  54. {ambers-0.4.0 → ambers-0.4.2}/src/metadata.rs +0 -0
  55. {ambers-0.4.0 → ambers-0.4.2}/src/python/conversions.rs +0 -0
  56. {ambers-0.4.0 → ambers-0.4.2}/src/python/io.rs +0 -0
  57. {ambers-0.4.0 → ambers-0.4.2}/src/python/metadata.rs +0 -0
  58. {ambers-0.4.0 → ambers-0.4.2}/src/python/mod.rs +0 -0
  59. {ambers-0.4.0 → ambers-0.4.2}/src/scanner.rs +0 -0
  60. {ambers-0.4.0 → ambers-0.4.2}/src/value_labels.rs +0 -0
  61. {ambers-0.4.0 → ambers-0.4.2}/src/variable.rs +0 -0
  62. {ambers-0.4.0 → ambers-0.4.2}/src/writer/data.rs +0 -0
  63. {ambers-0.4.0 → ambers-0.4.2}/src/writer/layout.rs +0 -0
  64. {ambers-0.4.0 → ambers-0.4.2}/src/writer/mod.rs +0 -0
  65. {ambers-0.4.0 → ambers-0.4.2}/src/writer/records.rs +0 -0
  66. {ambers-0.4.0 → ambers-0.4.2}/src/writer/tests.rs +0 -0
  67. {ambers-0.4.0 → ambers-0.4.2}/tests/bench_read.py +0 -0
  68. {ambers-0.4.0 → ambers-0.4.2}/tests/bench_write.py +0 -0
  69. {ambers-0.4.0 → ambers-0.4.2}/tests/conftest.py +0 -0
  70. {ambers-0.4.0 → ambers-0.4.2}/tests/diag_zsav.py +0 -0
  71. {ambers-0.4.0 → ambers-0.4.2}/tests/test_long_strings.py +0 -0
  72. {ambers-0.4.0 → ambers-0.4.2}/tests/test_medium_write.rs +0 -0
  73. {ambers-0.4.0 → ambers-0.4.2}/tests/test_metadata.py +0 -0
  74. {ambers-0.4.0 → ambers-0.4.2}/tests/test_metadata_api.py +0 -0
  75. {ambers-0.4.0 → ambers-0.4.2}/tests/test_paths.example.py +0 -0
  76. {ambers-0.4.0 → ambers-0.4.2}/tests/test_temporal.rs +0 -0
  77. {ambers-0.4.0 → ambers-0.4.2}/tests/test_wide_columns.py +0 -0
  78. {ambers-0.4.0 → ambers-0.4.2}/tests/test_write_roundtrip.py +0 -0
  79. {ambers-0.4.0 → ambers-0.4.2}/tests/test_writer_issues.py +0 -0
@@ -60,4 +60,4 @@ jobs:
60
60
  - name: Run Python tests (small/synthetic data only)
61
61
  run: |
62
62
  uv pip install --system pytest
63
- pytest tests/test_apply_labels.py tests/test_metadata_api.py tests/test_writer_issues.py -v
63
+ pytest tests/test_apply_labels.py tests/test_apply_missing.py tests/test_validate.py tests/test_metadata_api.py tests/test_writer_issues.py -v -k "not RealFile"
@@ -13,3 +13,4 @@ tests/test_paths.py
13
13
  CLAUDE.md
14
14
  milestone4.md
15
15
  notebook_test/
16
+ refactoring.md
@@ -2,6 +2,23 @@
2
2
 
3
3
  All notable changes to ambers are documented in this file.
4
4
 
5
+ ## [0.4.2] - 2026-04-07
6
+
7
+ - Add `validate(df, meta)` — check value label quality: unlabeled values (error) and duplicate labels (warning)
8
+ - `ValidationReport` with `is_valid`, `errors`, `warnings`, `raise_if_invalid()`, `to_frame()`
9
+ - Repr truncation: max 10 issues shown, long messages shortened, box width capped at 80 chars
10
+ - Shared pure helpers between `validate()` and `apply_labels()` (logic/policy separation)
11
+ - `columns` + `exclude` can be combined in `validate()` (lenient), stay mutually exclusive in `apply_labels`/`apply_missing` (strict)
12
+ - Add `validate.md` documentation with full API reference and examples
13
+ - 29 tests covering all checks, filtering, repr truncation, and shared helper consistency
14
+
15
+ ## [0.4.1] - 2026-04-07
16
+
17
+ - Add `apply_missing(df, meta)` — nullify SPSS user-defined missing value codes (discrete, range, range+discrete)
18
+ - Add `exclude=` parameter to both `apply_labels` and `apply_missing` — skip specific columns, mutually exclusive with `columns=`
19
+ - 35 new tests: 28 for apply_missing (all SPSS missing value combinations), 7 for exclude parameter
20
+ - Add `test_apply_missing.py` to CI pytest
21
+
5
22
  ## [0.4.0] - 2026-04-07
6
23
 
7
24
  - **Modularity refactor:** split `src/python/mod.rs` (2,299 LOC) into 5 focused submodules: `conversions.rs`, `metadata.rs`, `diff.rs`, `io.rs`, and thin `mod.rs`
@@ -33,7 +33,7 @@ dependencies = [
33
33
 
34
34
  [[package]]
35
35
  name = "ambers"
36
- version = "0.4.0"
36
+ version = "0.4.2"
37
37
  dependencies = [
38
38
  "arrow",
39
39
  "encoding_rs",
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "ambers"
3
- version = "0.4.0"
3
+ version = "0.4.2"
4
4
  edition = "2024"
5
5
  license = "MIT"
6
6
  description = "Pure Rust reader for SPSS .sav and .zsav files"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ambers
3
- Version: 0.4.0
3
+ Version: 0.4.2
4
4
  Requires-Dist: polars>=1.3
5
5
  License-File: LICENSE
6
6
  Summary: Pure Rust SPSS .sav/.zsav reader with Polars DataFrame output
@@ -83,6 +83,19 @@ labeled = am.apply_labels(df, meta) # Enum dtype (order
83
83
  labeled.write_excel("survey.xlsx") # Enum auto-casts to String
84
84
  labeled = am.apply_labels(df, meta, output="string") # String dtype for export
85
85
  labeled = am.apply_labels(df, meta, output="enum_null") # Enum, unmapped → null
86
+ labeled = am.apply_labels(df, meta, exclude=["weight", "id"]) # skip specific columns
87
+
88
+ # Apply missing values — nullify SPSS user-defined missing codes
89
+ clean = am.apply_missing(df, meta) # all columns with specs
90
+ clean = am.apply_missing(df, meta, columns=["Q1", "Q2"]) # specific columns only
91
+ clean = am.apply_missing(df, meta, exclude=["age"]) # skip specific columns
92
+
93
+ # Validate — check value label quality before analysis
94
+ report = am.validate(df, meta)
95
+ print(report) # box-drawing summary
96
+ report.is_valid # True if no errors
97
+ report.raise_if_invalid() # raise if errors found
98
+ report.to_frame() # DataFrame for export
86
99
  ```
87
100
 
88
101
  `.sav` uses bytecode compression by default, `.zsav` uses zlib. Pass `compression=` to override (`"uncompressed"`, `"bytecode"`, `"zlib"`). Pass `meta=` to preserve all metadata from a prior `read_sav()`, or omit it to infer formats from the DataFrame.
@@ -143,6 +156,29 @@ labeled = am.apply_labels(df, meta, output="enum_null")
143
156
 
144
157
  Numeric columns without value labels are skipped. String columns always pass through unmapped text. See [apply_labels.md](apply_labels.md) for full documentation.
145
158
 
159
+ ### validate
160
+
161
+ Check value label quality before analysis — catch unlabeled values and duplicate labels upfront.
162
+
163
+ ```python
164
+ sav = am.read_sav("survey.sav")
165
+ df, meta = sav.data, sav.meta
166
+
167
+ report = am.validate(df, meta)
168
+ print(report) # box-drawing summary
169
+ report.is_valid # True if no errors (warnings OK)
170
+ report.raise_if_invalid() # raise ValueError if errors
171
+
172
+ # Programmatic access
173
+ for error in report.errors:
174
+ print(f"{error.column}: {error.details['unlabeled_values']}")
175
+
176
+ # Export as DataFrame
177
+ report.to_frame().write_csv("validation_issues.csv")
178
+ ```
179
+
180
+ See [validate.md](validate.md) for full documentation.
181
+
146
182
  ## Rust
147
183
 
148
184
  ```rust
@@ -73,6 +73,19 @@ labeled = am.apply_labels(df, meta) # Enum dtype (order
73
73
  labeled.write_excel("survey.xlsx") # Enum auto-casts to String
74
74
  labeled = am.apply_labels(df, meta, output="string") # String dtype for export
75
75
  labeled = am.apply_labels(df, meta, output="enum_null") # Enum, unmapped → null
76
+ labeled = am.apply_labels(df, meta, exclude=["weight", "id"]) # skip specific columns
77
+
78
+ # Apply missing values — nullify SPSS user-defined missing codes
79
+ clean = am.apply_missing(df, meta) # all columns with specs
80
+ clean = am.apply_missing(df, meta, columns=["Q1", "Q2"]) # specific columns only
81
+ clean = am.apply_missing(df, meta, exclude=["age"]) # skip specific columns
82
+
83
+ # Validate — check value label quality before analysis
84
+ report = am.validate(df, meta)
85
+ print(report) # box-drawing summary
86
+ report.is_valid # True if no errors
87
+ report.raise_if_invalid() # raise if errors found
88
+ report.to_frame() # DataFrame for export
76
89
  ```
77
90
 
78
91
  `.sav` uses bytecode compression by default, `.zsav` uses zlib. Pass `compression=` to override (`"uncompressed"`, `"bytecode"`, `"zlib"`). Pass `meta=` to preserve all metadata from a prior `read_sav()`, or omit it to infer formats from the DataFrame.
@@ -133,6 +146,29 @@ labeled = am.apply_labels(df, meta, output="enum_null")
133
146
 
134
147
  Numeric columns without value labels are skipped. String columns always pass through unmapped text. See [apply_labels.md](apply_labels.md) for full documentation.
135
148
 
149
+ ### validate
150
+
151
+ Check value label quality before analysis — catch unlabeled values and duplicate labels upfront.
152
+
153
+ ```python
154
+ sav = am.read_sav("survey.sav")
155
+ df, meta = sav.data, sav.meta
156
+
157
+ report = am.validate(df, meta)
158
+ print(report) # box-drawing summary
159
+ report.is_valid # True if no errors (warnings OK)
160
+ report.raise_if_invalid() # raise ValueError if errors
161
+
162
+ # Programmatic access
163
+ for error in report.errors:
164
+ print(f"{error.column}: {error.details['unlabeled_values']}")
165
+
166
+ # Export as DataFrame
167
+ report.to_frame().write_csv("validation_issues.csv")
168
+ ```
169
+
170
+ See [validate.md](validate.md) for full documentation.
171
+
136
172
  ## Rust
137
173
 
138
174
  ```rust
@@ -85,6 +85,7 @@ def apply_labels(
85
85
  meta: SpssMetadata,
86
86
  *,
87
87
  columns: list[str] | None = None,
88
+ exclude: list[str] | None = None,
88
89
  output: Literal["enum", "string", "enum_null"] = "enum",
89
90
  ) -> DataFrame | LazyFrame
90
91
  ```
@@ -108,7 +109,7 @@ An `SpssMetadata` object containing value labels. Get it from `sav.meta`.
108
109
 
109
110
  ### `columns`
110
111
 
111
- Which columns to apply labels to. Default `None` applies to all columns that have value labels in the metadata AND exist in the data.
112
+ Which columns to apply labels to. Default `None` applies to all columns that have value labels in the metadata AND exist in the data. Mutually exclusive with `exclude`.
112
113
 
113
114
  ```python
114
115
  # All columns with value labels
@@ -122,6 +123,15 @@ When specified explicitly, raises `ValueError` if:
122
123
  - A column doesn't exist in the data
123
124
  - A column has no value labels (or an empty label dict) in the metadata
124
125
 
126
+ ### `exclude`
127
+
128
+ Columns to skip. When set, all columns with value labels are processed except those listed here. Mutually exclusive with `columns`.
129
+
130
+ ```python
131
+ # Apply labels to everything except weight and ID columns
132
+ am.apply_labels(df, meta, exclude=["weight", "respondent_id"])
133
+ ```
134
+
125
135
  ### `output`
126
136
 
127
137
  Controls both the output dtype and how unmapped values are handled. One parameter, three clear modes:
@@ -4,7 +4,7 @@ build-backend = "maturin"
4
4
 
5
5
  [project]
6
6
  name = "ambers"
7
- version = "0.4.0"
7
+ version = "0.4.2"
8
8
  description = "Pure Rust SPSS .sav/.zsav reader with Polars DataFrame output"
9
9
  license = { text = "MIT" }
10
10
  readme = "README.md"
@@ -3,11 +3,17 @@
3
3
  from ambers._ambers import MetaDiff, SpssMetadata
4
4
  from ambers._containers import SavFile
5
5
  from ambers._io import read_sav, read_sav_meta, scan_sav, write_sav
6
+ from ambers._missing import apply_missing
6
7
  from ambers._transforms import apply_labels
8
+ from ambers._validate import ValidationIssue, ValidationReport, validate
7
9
 
8
10
  __all__ = [
9
11
  "SavFile",
10
12
  "apply_labels",
13
+ "apply_missing",
14
+ "validate",
15
+ "ValidationReport",
16
+ "ValidationIssue",
11
17
  "read_sav",
12
18
  "read_sav_meta",
13
19
  "scan_sav",
@@ -755,6 +755,7 @@ def apply_labels(
755
755
  meta: SpssMetadata,
756
756
  *,
757
757
  columns: list[str] | None = ...,
758
+ exclude: list[str] | None = ...,
758
759
  output: str = ...,
759
760
  ) -> polars.DataFrame: ...
760
761
  @overload
@@ -763,6 +764,7 @@ def apply_labels(
763
764
  meta: SpssMetadata,
764
765
  *,
765
766
  columns: list[str] | None = ...,
767
+ exclude: list[str] | None = ...,
766
768
  output: str = ...,
767
769
  ) -> polars.LazyFrame: ...
768
770
  def apply_labels(
@@ -770,6 +772,7 @@ def apply_labels(
770
772
  meta: SpssMetadata,
771
773
  *,
772
774
  columns: list[str] | None = None,
775
+ exclude: list[str] | None = None,
773
776
  output: str = "enum",
774
777
  ) -> polars.DataFrame | polars.LazyFrame:
775
778
  """Replace numeric/string codes with value labels from SPSS metadata.
@@ -785,7 +788,10 @@ def apply_labels(
785
788
  SpssMetadata with value labels.
786
789
  columns
787
790
  Columns to apply labels to. None applies to all columns with
788
- value labels in the metadata.
791
+ value labels in the metadata. Mutually exclusive with ``exclude``.
792
+ exclude
793
+ Columns to skip. All columns with value labels are processed
794
+ except those listed here. Mutually exclusive with ``columns``.
789
795
  output
790
796
  Output mode for labeled numeric columns.
791
797
 
@@ -808,3 +814,106 @@ def apply_labels(
808
814
  >>> labeled.write_excel("survey.xlsx")
809
815
  """
810
816
  ...
817
+
818
+ def apply_missing(
819
+ df: pl.DataFrame | pl.LazyFrame,
820
+ meta: SpssMetadata,
821
+ *,
822
+ columns: list[str] | None = None,
823
+ exclude: list[str] | None = None,
824
+ ) -> pl.DataFrame | pl.LazyFrame:
825
+ """Replace SPSS user-defined missing value codes with null.
826
+
827
+ Converts values that match the missing value specifications in
828
+ ``meta.variable_missing_values`` to null. Handles discrete values,
829
+ ranges, and range-plus-discrete combinations.
830
+
831
+ **Scope:** User-defined missing values only. SPSS system missing
832
+ (SYSMIS) is already read as null by the reader.
833
+
834
+ Parameters
835
+ ----------
836
+ df
837
+ A Polars DataFrame or LazyFrame with raw SPSS data.
838
+ meta
839
+ SpssMetadata with missing value definitions.
840
+ columns
841
+ Columns to apply missing values to. None applies to all
842
+ columns with missing value specs in metadata. Columns not
843
+ found in the DataFrame or without specs are silently skipped.
844
+ Mutually exclusive with ``exclude``.
845
+ exclude
846
+ Columns to skip. All columns with missing value specs are
847
+ processed except those listed here. Mutually exclusive with
848
+ ``columns``.
849
+
850
+ Returns
851
+ -------
852
+ polars.DataFrame | polars.LazyFrame
853
+
854
+ Examples
855
+ --------
856
+ >>> sav = am.read_sav("survey.sav")
857
+ >>> df, meta = sav.data, sav.meta
858
+ >>> clean = am.apply_missing(df, meta)
859
+ >>> clean.null_count()
860
+ """
861
+ ...
862
+
863
+ class ValidationIssue:
864
+ severity: str
865
+ column: str
866
+ check: str
867
+ message: str
868
+ details: dict
869
+
870
+ class ValidationReport:
871
+ issues: list[ValidationIssue]
872
+
873
+ @property
874
+ def is_valid(self) -> bool: ...
875
+ @property
876
+ def errors(self) -> list[ValidationIssue]: ...
877
+ @property
878
+ def warnings(self) -> list[ValidationIssue]: ...
879
+ def raise_if_invalid(self) -> None: ...
880
+ def to_frame(self) -> pl.DataFrame: ...
881
+
882
+ def validate(
883
+ df: pl.DataFrame | pl.LazyFrame,
884
+ meta: SpssMetadata,
885
+ *,
886
+ columns: list[str] | None = None,
887
+ exclude: list[str] | None = None,
888
+ ) -> ValidationReport:
889
+ """Validate SPSS value label quality.
890
+
891
+ Checks numeric columns with value labels for unlabeled values
892
+ (error) and duplicate labels (warning). Columns without value
893
+ labels are skipped.
894
+
895
+ Parameters
896
+ ----------
897
+ df
898
+ A Polars DataFrame or LazyFrame.
899
+ meta
900
+ SpssMetadata with value labels.
901
+ columns
902
+ Columns to check. None checks all numeric columns with value
903
+ labels. Can be combined with ``exclude``.
904
+ exclude
905
+ Columns to skip. Applied after ``columns`` filtering.
906
+
907
+ Returns
908
+ -------
909
+ ValidationReport
910
+
911
+ Examples
912
+ --------
913
+ >>> sav = am.read_sav("survey.sav")
914
+ >>> df, meta = sav.data, sav.meta
915
+ >>> report = am.validate(df, meta)
916
+ >>> report.is_valid
917
+ >>> report.raise_if_invalid()
918
+ """
919
+ ...
@@ -0,0 +1,108 @@
1
+ """Apply SPSS user-defined missing value specs to DataFrames."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import polars as pl
6
+
7
+
8
+ def apply_missing(
9
+ df: pl.DataFrame | pl.LazyFrame,
10
+ meta,
11
+ *,
12
+ columns: list[str] | None = None,
13
+ exclude: list[str] | None = None,
14
+ ) -> pl.DataFrame | pl.LazyFrame:
15
+ """Replace SPSS user-defined missing value codes with null.
16
+
17
+ Converts values that match the missing value specifications in
18
+ ``meta.variable_missing_values`` to null. This handles discrete
19
+ values, ranges, and range-plus-discrete combinations as defined
20
+ in the SPSS file.
21
+
22
+ **Scope:** User-defined missing values only (discrete, range,
23
+ range+discrete). SPSS system missing (SYSMIS) is already read
24
+ as null by the reader and is not affected by this function.
25
+
26
+ Args:
27
+ df: A Polars DataFrame or LazyFrame with raw SPSS data.
28
+ meta: An ``SpssMetadata`` object with missing value definitions.
29
+ columns: Columns to apply missing values to. ``None`` applies
30
+ to all columns that have missing value specs in metadata.
31
+ Columns not found in the DataFrame or without specs are
32
+ silently skipped. Mutually exclusive with ``exclude``.
33
+ exclude: Columns to skip. When set, all columns with missing
34
+ value specs are processed except those listed here.
35
+ Mutually exclusive with ``columns``.
36
+
37
+ Returns:
38
+ DataFrame or LazyFrame (same type as input) with missing value
39
+ codes replaced by null. Existing nulls are preserved.
40
+ """
41
+ if not isinstance(df, (pl.DataFrame, pl.LazyFrame)):
42
+ raise TypeError(
43
+ f"df must be a polars DataFrame or LazyFrame, got {type(df).__name__}"
44
+ )
45
+
46
+ if columns is not None and exclude is not None:
47
+ raise ValueError("columns and exclude are mutually exclusive")
48
+
49
+ missing_specs = meta.variable_missing_values
50
+ if not missing_specs:
51
+ return df
52
+
53
+ # Determine which columns to process
54
+ schema = df.collect_schema() if isinstance(df, pl.LazyFrame) else df.schema
55
+ df_columns = set(schema.names())
56
+
57
+ if columns is not None:
58
+ target_cols = [c for c in columns if c in df_columns and c in missing_specs]
59
+ else:
60
+ target_cols = [c for c in missing_specs if c in df_columns]
61
+ if exclude is not None:
62
+ exclude_set = set(exclude)
63
+ target_cols = [c for c in target_cols if c not in exclude_set]
64
+
65
+ if not target_cols:
66
+ return df
67
+
68
+ # Build one expression per column
69
+ exprs = []
70
+ for name in target_cols:
71
+ spec = missing_specs[name]
72
+ condition = _build_condition(name, spec)
73
+ if condition is not None:
74
+ exprs.append(
75
+ pl.when(condition)
76
+ .then(pl.lit(None))
77
+ .otherwise(pl.col(name))
78
+ .alias(name)
79
+ )
80
+
81
+ if not exprs:
82
+ return df
83
+
84
+ return df.with_columns(exprs)
85
+
86
+
87
+ def _build_condition(name: str, spec: dict) -> pl.Expr | None:
88
+ """Build a Polars boolean expression for a missing value spec."""
89
+ spec_type = spec.get("type")
90
+ col = pl.col(name)
91
+
92
+ if spec_type == "discrete":
93
+ values = spec.get("values", [])
94
+ if not values:
95
+ return None
96
+ return col.is_in(values)
97
+
98
+ elif spec_type == "range":
99
+ lo = spec["low"]
100
+ hi = spec["high"]
101
+ condition = col.is_between(lo, hi, closed="both")
102
+ # Range + discrete combination
103
+ discrete = spec.get("discrete")
104
+ if discrete is not None:
105
+ condition = condition | (col == discrete)
106
+ return condition
107
+
108
+ return None
@@ -17,6 +17,7 @@ def apply_labels(
17
17
  meta,
18
18
  *,
19
19
  columns: list[str] | None = None,
20
+ exclude: list[str] | None = None,
20
21
  output: str = "enum",
21
22
  ) -> pl.DataFrame | pl.LazyFrame:
22
23
  """Replace numeric/string codes with value labels from SPSS metadata.
@@ -45,7 +46,10 @@ def apply_labels(
45
46
  columns: Columns to apply labels to. ``None`` applies to all
46
47
  columns that have value labels in the metadata. When specified
47
48
  explicitly, raises if columns are missing from data or have
48
- no value labels defined.
49
+ no value labels defined. Mutually exclusive with ``exclude``.
50
+ exclude: Columns to skip. When set, all columns with value
51
+ labels are processed except those listed here. Mutually
52
+ exclusive with ``columns``.
49
53
  output: Output mode for labeled numeric columns.
50
54
 
51
55
  - ``"enum"`` (default) — ``pl.Enum`` with ordered categories.
@@ -89,6 +93,9 @@ def apply_labels(
89
93
  )
90
94
  as_enum, unmapped = _OUTPUT_MODES[output]
91
95
 
96
+ if columns is not None and exclude is not None:
97
+ raise ValueError("columns and exclude are mutually exclusive")
98
+
92
99
  # Get schema + determine target columns
93
100
  schema = (
94
101
  df.collect_schema() if isinstance(df, pl.LazyFrame) else df.schema
@@ -106,6 +113,9 @@ def apply_labels(
106
113
  target = columns
107
114
  else:
108
115
  target = [c for c in all_labels if c in data_cols]
116
+ if exclude is not None:
117
+ exclude_set = set(exclude)
118
+ target = [c for c in target if c not in exclude_set]
109
119
 
110
120
  if not target:
111
121
  return df
@@ -189,9 +199,12 @@ def _validate_labels(
189
199
  ) -> None:
190
200
  """Pre-check for unmapped values and duplicate labels on numeric columns.
191
201
 
202
+ Uses shared helpers from _validate.py for the actual checks.
192
203
  Raises ValueError with a structured diagnostic if issues are found.
193
204
  Only called when unmapped="error" or as_enum=True.
194
205
  """
206
+ from ambers._validate import check_duplicate_labels, check_unlabeled_values
207
+
195
208
  unmapped_issues: list[str] = []
196
209
  duplicate_issues: list[str] = []
197
210
 
@@ -199,19 +212,21 @@ def _validate_labels(
199
212
  labels = all_labels[col_name]
200
213
  label_keys = set(labels.keys())
201
214
 
202
- # Check unmapped values
215
+ # Check unmapped values (using shared helper)
203
216
  if unmapped == "error":
204
- col_expr = pl.col(col_name).drop_nulls().unique()
205
217
  if isinstance(df, pl.LazyFrame):
206
218
  unique_vals = set(
207
- df.select(col_expr).collect()[col_name].to_list()
219
+ df.select(pl.col(col_name).drop_nulls().unique())
220
+ .collect()[col_name].to_list()
208
221
  )
209
222
  else:
210
- unique_vals = set(df.select(col_expr)[col_name].to_list())
211
- n_unique = len(unique_vals)
212
- n_labeled = len(label_keys)
213
- missing = sorted(unique_vals - label_keys)
214
- if missing:
223
+ unique_vals = set(
224
+ df.select(pl.col(col_name).drop_nulls().unique())
225
+ [col_name].to_list()
226
+ )
227
+ result = check_unlabeled_values(col_name, unique_vals, label_keys)
228
+ if result is not None:
229
+ missing = result["unlabeled_values"]
215
230
  vals_str = ", ".join(str(v) for v in missing[:10])
216
231
  if len(missing) > 10:
217
232
  vals_str += f", ... ({len(missing)} total)"
@@ -219,30 +234,28 @@ def _validate_labels(
219
234
  f" {col_name}: {len(missing)} unmapped "
220
235
  f"{'value' if len(missing) == 1 else 'values'}: "
221
236
  f"[{vals_str}] "
222
- f"({n_unique} unique, {n_labeled} labeled)"
237
+ f"({result['unique_in_data']} unique, "
238
+ f"{len(label_keys)} labeled)"
223
239
  )
224
240
 
225
- # Check duplicate label values
241
+ # Check duplicate label values (using shared helper)
226
242
  if as_enum:
227
- seen: dict[str, list] = {}
228
- for k, v in labels.items():
229
- seen.setdefault(v, []).append(k)
230
- dupes = {v: keys for v, keys in seen.items() if len(keys) > 1}
231
- if dupes:
243
+ result = check_duplicate_labels(col_name, labels)
244
+ if result is not None:
232
245
  parts = []
233
- for label_val, keys in dupes.items():
246
+ for label_val, keys in result["duplicates"].items():
234
247
  keys_str = ", ".join(str(k) for k in keys)
235
248
  parts.append(f"[{keys_str}] -> {label_val!r}")
236
249
  duplicate_issues.append(
237
- f" {col_name}: {len(dupes)} duplicate "
238
- f"{'label' if len(dupes) == 1 else 'labels'}: "
250
+ f" {col_name}: {len(result['duplicates'])} duplicate "
251
+ f"{'label' if len(result['duplicates']) == 1 else 'labels'}: "
239
252
  + "; ".join(parts)
240
253
  )
241
254
 
242
255
  if not unmapped_issues and not duplicate_issues:
243
256
  return
244
257
 
245
- # Build structured error message
258
+ # Build structured error message (preserve existing format)
246
259
  msg_parts: list[str] = []
247
260
 
248
261
  if unmapped_issues: