ambers 0.4.2__tar.gz → 0.4.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ambers-0.4.2 → ambers-0.4.4}/.github/workflows/ci.yml +1 -1
- {ambers-0.4.2 → ambers-0.4.4}/.gitignore +1 -0
- {ambers-0.4.2 → ambers-0.4.4}/CHANGELOG.md +17 -0
- {ambers-0.4.2 → ambers-0.4.4}/Cargo.lock +1 -1
- {ambers-0.4.2 → ambers-0.4.4}/Cargo.toml +1 -1
- {ambers-0.4.2 → ambers-0.4.4}/PKG-INFO +1 -1
- {ambers-0.4.2 → ambers-0.4.4}/pyproject.toml +1 -1
- {ambers-0.4.2 → ambers-0.4.4}/python/ambers/__init__.py +2 -0
- {ambers-0.4.2 → ambers-0.4.4}/python/ambers/_io.py +2 -2
- ambers-0.4.4/python/ambers/codebook/__init__.py +330 -0
- ambers-0.4.4/python/ambers/codebook/_engine.py +304 -0
- ambers-0.4.4/python/ambers/codebook/_types.py +382 -0
- ambers-0.4.4/tests/test_codebook.py +385 -0
- ambers-0.4.4/tests/test_codebook_engine.py +238 -0
- ambers-0.4.4/tests/test_codebook_types.py +358 -0
- ambers-0.4.4/tests/test_polars_compat.py +162 -0
- {ambers-0.4.2 → ambers-0.4.4}/tests/test_writer_issues.py +2 -1
- {ambers-0.4.2 → ambers-0.4.4}/.cargo/config.toml +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/.github/workflows/release.yml +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/LICENSE +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/README.md +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/apply_labels.md +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/images/ambers-banner-v2.svg +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/images/ambers-banner.svg +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/metadata.md +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/python/ambers/__init__.pyi +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/python/ambers/_containers.py +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/python/ambers/_missing.py +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/python/ambers/_transforms.py +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/python/ambers/_validate.py +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/arrow_convert.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/columnar.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/compression/bytecode.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/compression/bytecode_encoder.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/compression/mod.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/compression/zlib.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/constants.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/dictionary.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/document.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/encoding.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/error.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/header.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/info_records/encoding_record.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/info_records/float_info.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/info_records/integer_info.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/info_records/long_string_labels.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/info_records/long_string_missing.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/info_records/long_var_names.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/info_records/mod.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/info_records/mr_sets.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/info_records/var_attributes.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/info_records/var_display.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/info_records/very_long_strings.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/io_utils.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/lib.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/metadata.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/python/conversions.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/python/diff.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/python/io.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/python/metadata.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/python/mod.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/scanner.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/value_labels.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/variable.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/writer/data.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/writer/layout.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/writer/mod.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/writer/records.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/src/writer/tests.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/tests/bench_read.py +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/tests/bench_write.py +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/tests/conftest.py +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/tests/diag_zsav.py +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/tests/test_apply_labels.py +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/tests/test_apply_missing.py +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/tests/test_long_strings.py +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/tests/test_medium_write.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/tests/test_metadata.py +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/tests/test_metadata_api.py +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/tests/test_paths.example.py +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/tests/test_temporal.rs +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/tests/test_validate.py +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/tests/test_wide_columns.py +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/tests/test_write_roundtrip.py +0 -0
- {ambers-0.4.2 → ambers-0.4.4}/validate.md +0 -0
|
@@ -60,4 +60,4 @@ jobs:
|
|
|
60
60
|
- name: Run Python tests (small/synthetic data only)
|
|
61
61
|
run: |
|
|
62
62
|
uv pip install --system pytest
|
|
63
|
-
pytest tests/test_apply_labels.py tests/test_apply_missing.py tests/test_validate.py tests/test_metadata_api.py tests/test_writer_issues.py -v -k "not RealFile"
|
|
63
|
+
pytest tests/test_apply_labels.py tests/test_apply_missing.py tests/test_validate.py tests/test_metadata_api.py tests/test_writer_issues.py tests/test_polars_compat.py -v -k "not RealFile"
|
|
@@ -2,6 +2,23 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to ambers are documented in this file.
|
|
4
4
|
|
|
5
|
+
## [0.4.4] - 2026-09-18
|
|
6
|
+
|
|
7
|
+
- Fix `FutureWarning: from_arrow(<ArrowStreamExportable>) will return a Series` emitted by `read_sav()` and `scan_sav()` on Polars >= 1.44 — Arrow streams are now converted with `pl.DataFrame()` (available since Polars 1.3, which remains the minimum)
|
|
8
|
+
- Polars 2.0 compatibility: `read_sav()`/`scan_sav()` no longer break when `pl.from_arrow()` returns a Series; `codebook()` type detection casts before `is_in()` to satisfy 2.0's strict coercion. Full Python test suite passes on Polars 1.40, 1.44.2 and 2.0.0rc1
|
|
9
|
+
- Add `tests/test_polars_compat.py` (eager, lazy, and multi-batch conversion with FutureWarning treated as error); added to CI
|
|
10
|
+
|
|
11
|
+
## [0.4.3] - 2026-04-30
|
|
12
|
+
|
|
13
|
+
- Add `codebook(df, meta)` — generate a Polars DataFrame data dictionary documenting every variable and its values
|
|
14
|
+
- Two views: `view="variables"` (default, one row per variable) and `view="values"` (one row per value)
|
|
15
|
+
- 5-way variable type detection: single-select, multi-select, numeric, text, date — with full multi-select tiers (MR sets, binary patterns, sibling series, generic binary)
|
|
16
|
+
- `values_format=` controls the variables-view `values` column: `"string"` (default, newline-joined `"1=Low\n2=Medium"`) for clean marimo HTML and Excel rendering; `"struct"` for `List[Struct{value_code, value_label}]` with `.explode().unnest()` workflows
|
|
17
|
+
- `include_meta=True` adds `variable_measure` and `variable_format` columns to the values view
|
|
18
|
+
- `columns=` and `exclude=` filtering combinable
|
|
19
|
+
- Strict validation: rejects unknown `view` values; rejects `values_format` with `view="values"`
|
|
20
|
+
- 40 tests; integration verified on real SPSS files
|
|
21
|
+
|
|
5
22
|
## [0.4.2] - 2026-04-07
|
|
6
23
|
|
|
7
24
|
- Add `validate(df, meta)` — check value label quality: unlabeled values (error) and duplicate labels (warning)
|
|
@@ -6,11 +6,13 @@ from ambers._io import read_sav, read_sav_meta, scan_sav, write_sav
|
|
|
6
6
|
from ambers._missing import apply_missing
|
|
7
7
|
from ambers._transforms import apply_labels
|
|
8
8
|
from ambers._validate import ValidationIssue, ValidationReport, validate
|
|
9
|
+
from ambers.codebook import codebook
|
|
9
10
|
|
|
10
11
|
__all__ = [
|
|
11
12
|
"SavFile",
|
|
12
13
|
"apply_labels",
|
|
13
14
|
"apply_missing",
|
|
15
|
+
"codebook",
|
|
14
16
|
"validate",
|
|
15
17
|
"ValidationReport",
|
|
16
18
|
"ValidationIssue",
|
|
@@ -65,7 +65,7 @@ def read_sav(
|
|
|
65
65
|
resolved = None
|
|
66
66
|
|
|
67
67
|
stream, meta = _read_sav(str(path), columns=resolved, n_rows=n_rows)
|
|
68
|
-
df = pl.
|
|
68
|
+
df = pl.DataFrame(stream)
|
|
69
69
|
if row_index_name is not None:
|
|
70
70
|
df = df.with_row_index(row_index_name, offset=row_index_offset)
|
|
71
71
|
|
|
@@ -186,7 +186,7 @@ def scan_sav(
|
|
|
186
186
|
scanner.limit(min(limits))
|
|
187
187
|
|
|
188
188
|
while (batch := scanner.next_batch()) is not None:
|
|
189
|
-
df = pl.
|
|
189
|
+
df = pl.DataFrame(batch)
|
|
190
190
|
if predicate is not None:
|
|
191
191
|
df = df.filter(predicate)
|
|
192
192
|
yield df
|
|
@@ -0,0 +1,330 @@
|
|
|
1
|
+
"""Codebook generation for SPSS data.
|
|
2
|
+
|
|
3
|
+
Produces a Polars DataFrame documenting every variable and its values.
|
|
4
|
+
|
|
5
|
+
Two views, named after the granularity of each row:
|
|
6
|
+
|
|
7
|
+
- **variables** (default): one row per variable. Columns:
|
|
8
|
+
``variable``, ``variable_label``, ``variable_type``, ``values``,
|
|
9
|
+
``n_valid``, ``n_missing``, ``n_total``, ``n_unique``,
|
|
10
|
+
``n_labeled``, ``n_unlabeled``.
|
|
11
|
+
|
|
12
|
+
The ``values`` column has two formats controlled by ``values_format=``:
|
|
13
|
+
|
|
14
|
+
- ``"string"`` (default): newline-separated ``"1=Low\\n2=Medium\\n5=High"``,
|
|
15
|
+
renders cleanly in marimo HTML and Excel.
|
|
16
|
+
- ``"struct"``: ``List[Struct{value_code, value_label}]``, native dtypes,
|
|
17
|
+
explodable + unnestable for programmatic work.
|
|
18
|
+
|
|
19
|
+
- **values**: one row per (variable, value) for categorical variables,
|
|
20
|
+
one summary row for numeric/text/date. Columns:
|
|
21
|
+
``variable``, ``variable_label``, ``variable_type``,
|
|
22
|
+
``value_code``, ``value_label``, ``value_n``,
|
|
23
|
+
``n_valid``, ``pct_valid``, ``n_total``, ``pct_total``.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import polars as pl
|
|
29
|
+
|
|
30
|
+
MISSING_LABEL = "MISSING"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def codebook(
|
|
34
|
+
df: pl.DataFrame | pl.LazyFrame,
|
|
35
|
+
meta,
|
|
36
|
+
*,
|
|
37
|
+
view: str = "variables",
|
|
38
|
+
columns: list[str] | None = None,
|
|
39
|
+
exclude: list[str] | None = None,
|
|
40
|
+
include_meta: bool = False,
|
|
41
|
+
values_format: str = "string",
|
|
42
|
+
) -> pl.DataFrame:
|
|
43
|
+
"""Generate a codebook from data and metadata.
|
|
44
|
+
|
|
45
|
+
Args:
|
|
46
|
+
df: A Polars DataFrame or LazyFrame.
|
|
47
|
+
meta: An ``SpssMetadata`` object.
|
|
48
|
+
view: ``"variables"`` (default, one row per variable) or
|
|
49
|
+
``"values"`` (one row per value).
|
|
50
|
+
columns: Columns to include. ``None`` includes all.
|
|
51
|
+
Can be combined with ``exclude``.
|
|
52
|
+
exclude: Columns to skip. Applied after ``columns``.
|
|
53
|
+
include_meta: If True, add ``variable_measure`` and
|
|
54
|
+
``variable_format`` columns to the values view.
|
|
55
|
+
values_format: ``view="variables"`` only — ignored in the values
|
|
56
|
+
view (and passing a non-default value with ``view="values"``
|
|
57
|
+
raises ``ValueError`` to flag the misuse). ``"string"``
|
|
58
|
+
(default) emits ``values`` as newline-separated
|
|
59
|
+
``"1=Low\\n2=Medium"``. ``"struct"`` emits
|
|
60
|
+
``List[Struct{value_code, value_label}]`` for
|
|
61
|
+
``.explode().unnest()`` workflows.
|
|
62
|
+
|
|
63
|
+
Returns:
|
|
64
|
+
A Polars DataFrame with the codebook.
|
|
65
|
+
|
|
66
|
+
**Variables view columns:**
|
|
67
|
+
``variable``, ``variable_label``, ``variable_type``, ``values``,
|
|
68
|
+
``n_valid``, ``n_missing``, ``n_total``, ``n_unique``,
|
|
69
|
+
``n_labeled``, ``n_unlabeled``.
|
|
70
|
+
|
|
71
|
+
**Values view columns:**
|
|
72
|
+
``variable``, ``variable_label``, ``variable_type``,
|
|
73
|
+
``value_code``, ``value_label``, ``value_n``,
|
|
74
|
+
``n_valid``, ``pct_valid``, ``n_total``, ``pct_total``.
|
|
75
|
+
"""
|
|
76
|
+
from ._engine import build_rows
|
|
77
|
+
from ._types import detect_types
|
|
78
|
+
|
|
79
|
+
if not isinstance(df, (pl.DataFrame, pl.LazyFrame)):
|
|
80
|
+
raise TypeError(
|
|
81
|
+
f"df must be a polars DataFrame or LazyFrame, got {type(df).__name__}"
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
if view not in ("variables", "values"):
|
|
85
|
+
raise ValueError(
|
|
86
|
+
f"view must be 'variables' or 'values', got {view!r}"
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
if values_format not in ("string", "struct"):
|
|
90
|
+
raise ValueError(
|
|
91
|
+
f"values_format must be 'string' or 'struct', got {values_format!r}"
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
if view == "values" and values_format != "string":
|
|
95
|
+
raise ValueError(
|
|
96
|
+
"values_format only applies to view='variables'; "
|
|
97
|
+
"omit values_format when using view='values'"
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
# Collect LazyFrame
|
|
101
|
+
if isinstance(df, pl.LazyFrame):
|
|
102
|
+
df = df.collect()
|
|
103
|
+
|
|
104
|
+
# Determine target columns
|
|
105
|
+
target = list(df.columns) if columns is None else [c for c in columns if c in df.columns]
|
|
106
|
+
if exclude is not None:
|
|
107
|
+
exclude_set = set(exclude)
|
|
108
|
+
target = [c for c in target if c not in exclude_set]
|
|
109
|
+
|
|
110
|
+
if not target:
|
|
111
|
+
if view == "values":
|
|
112
|
+
return _empty_detail(include_meta)
|
|
113
|
+
return _empty_summary(values_format)
|
|
114
|
+
|
|
115
|
+
# Phase 1: type detection
|
|
116
|
+
type_map = detect_types(df.select(target), meta)
|
|
117
|
+
|
|
118
|
+
# Phase 2: build raw rows
|
|
119
|
+
raw = build_rows(df, meta, type_map, target)
|
|
120
|
+
|
|
121
|
+
# Phase 3: computed columns
|
|
122
|
+
detail = _compute_detail(raw)
|
|
123
|
+
|
|
124
|
+
if view == "variables":
|
|
125
|
+
return _build_summary(detail, values_format)
|
|
126
|
+
|
|
127
|
+
# Select final columns for detail view
|
|
128
|
+
cols = [
|
|
129
|
+
"variable", "variable_label", "variable_type",
|
|
130
|
+
]
|
|
131
|
+
if include_meta:
|
|
132
|
+
cols += ["variable_measure", "variable_format"]
|
|
133
|
+
cols += [
|
|
134
|
+
"value_code", "value_label", "value_n",
|
|
135
|
+
"n_valid", "pct_valid", "n_total", "pct_total",
|
|
136
|
+
]
|
|
137
|
+
|
|
138
|
+
return detail.select([c for c in cols if c in detail.columns])
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _compute_detail(raw: pl.DataFrame) -> pl.DataFrame:
|
|
142
|
+
"""Add computed columns: n_valid, pct_valid, n_total, pct_total."""
|
|
143
|
+
# Merge single-select/multi-select → categorical in output
|
|
144
|
+
result = raw.with_columns(
|
|
145
|
+
pl.when(pl.col("variable_type").is_in(["single-select", "multi-select"]))
|
|
146
|
+
.then(pl.lit("categorical"))
|
|
147
|
+
.otherwise(pl.col("variable_type"))
|
|
148
|
+
.alias("variable_type")
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
# Internal flag: is this a MISSING row? (used for computation, not in output)
|
|
152
|
+
is_missing = pl.col("value_label").eq(MISSING_LABEL).fill_null(False)
|
|
153
|
+
|
|
154
|
+
# n_valid: sum of value_n for non-missing rows per variable
|
|
155
|
+
# For MISSING rows: n_valid = its own value_n (ultrasav behavior)
|
|
156
|
+
result = result.with_columns(
|
|
157
|
+
pl.when(is_missing)
|
|
158
|
+
.then(pl.col("value_n"))
|
|
159
|
+
.otherwise(
|
|
160
|
+
pl.col("value_n")
|
|
161
|
+
.filter(~is_missing)
|
|
162
|
+
.sum()
|
|
163
|
+
.over("variable")
|
|
164
|
+
)
|
|
165
|
+
.cast(pl.UInt32)
|
|
166
|
+
.alias("n_valid")
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
# n_total: sum of ALL value_n per variable
|
|
170
|
+
result = result.with_columns(
|
|
171
|
+
pl.col("value_n").sum().over("variable").cast(pl.UInt32).alias("n_total")
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
# pct_valid: value_n / n_valid (null for missing rows, null if n_valid=0)
|
|
175
|
+
result = result.with_columns(
|
|
176
|
+
pl.when(is_missing)
|
|
177
|
+
.then(pl.lit(None, dtype=pl.Float64))
|
|
178
|
+
.otherwise(pl.col("value_n").cast(pl.Float64) / pl.col("n_valid").cast(pl.Float64))
|
|
179
|
+
.alias("pct_valid")
|
|
180
|
+
).with_columns(
|
|
181
|
+
pl.when(pl.col("pct_valid").is_nan())
|
|
182
|
+
.then(pl.lit(None, dtype=pl.Float64))
|
|
183
|
+
.otherwise(pl.col("pct_valid"))
|
|
184
|
+
.alias("pct_valid")
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
# pct_total: value_n / n_total (null if n_total=0)
|
|
188
|
+
result = result.with_columns(
|
|
189
|
+
(pl.col("value_n").cast(pl.Float64) / pl.col("n_total").cast(pl.Float64))
|
|
190
|
+
.alias("pct_total")
|
|
191
|
+
).with_columns(
|
|
192
|
+
pl.when(pl.col("pct_total").is_nan())
|
|
193
|
+
.then(pl.lit(None, dtype=pl.Float64))
|
|
194
|
+
.otherwise(pl.col("pct_total"))
|
|
195
|
+
.alias("pct_total")
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
return result
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _build_summary(detail: pl.DataFrame, values_format: str = "string") -> pl.DataFrame:
|
|
202
|
+
"""Aggregate detail view into one row per variable.
|
|
203
|
+
|
|
204
|
+
Uses single group_by().agg() instead of per-variable Python loop.
|
|
205
|
+
"""
|
|
206
|
+
if detail.height == 0:
|
|
207
|
+
return _empty_summary(values_format)
|
|
208
|
+
|
|
209
|
+
is_missing = pl.col("value_label").eq(MISSING_LABEL).fill_null(False)
|
|
210
|
+
is_unlabeled = pl.col("value_code").is_not_null() & pl.col("value_label").is_null()
|
|
211
|
+
has_code_and_label = (
|
|
212
|
+
~is_missing
|
|
213
|
+
& pl.col("value_code").is_not_null()
|
|
214
|
+
& pl.col("value_label").is_not_null()
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
summary = detail.group_by("variable", maintain_order=True).agg(
|
|
218
|
+
pl.col("variable_label").first(),
|
|
219
|
+
pl.col("variable_type").first(),
|
|
220
|
+
|
|
221
|
+
# n_valid: first n_valid from non-missing rows (or 0)
|
|
222
|
+
pl.col("n_valid").filter(~is_missing).first().fill_null(0).alias("n_valid"),
|
|
223
|
+
|
|
224
|
+
# n_missing: value_n of the MISSING row (or 0)
|
|
225
|
+
pl.col("value_n").filter(is_missing).first().fill_null(0).alias("n_missing"),
|
|
226
|
+
|
|
227
|
+
# n_unique: count of rows with non-null value_code
|
|
228
|
+
# (for categorical: includes meta-only labels with count=0)
|
|
229
|
+
# (for non-categorical: equals value_n — preserved quirk)
|
|
230
|
+
pl.col("value_code").drop_nulls().len().alias("n_unique"),
|
|
231
|
+
|
|
232
|
+
# n_labeled: rows with value_code AND value_label (non-missing)
|
|
233
|
+
pl.col("value_label").filter(has_code_and_label).len().alias("n_labeled"),
|
|
234
|
+
|
|
235
|
+
# n_unlabeled: rows where unlabeled (value_code not null, value_label null)
|
|
236
|
+
is_unlabeled.sum().cast(pl.UInt32).alias("n_unlabeled"),
|
|
237
|
+
|
|
238
|
+
# values: List[Struct{value_code, value_label}] — native dtypes preserved
|
|
239
|
+
pl.struct([
|
|
240
|
+
pl.col("value_code"),
|
|
241
|
+
pl.col("value_label"),
|
|
242
|
+
])
|
|
243
|
+
.filter(has_code_and_label)
|
|
244
|
+
.alias("values"),
|
|
245
|
+
)
|
|
246
|
+
|
|
247
|
+
# Add n_total = n_valid + n_missing
|
|
248
|
+
summary = summary.with_columns(
|
|
249
|
+
(pl.col("n_valid") + pl.col("n_missing")).cast(pl.UInt32).alias("n_total")
|
|
250
|
+
)
|
|
251
|
+
|
|
252
|
+
# Format the values column for human-readable rendering when requested.
|
|
253
|
+
# The struct list was filtered to has_code_and_label upstream, so empty
|
|
254
|
+
# lists here mean "no labeled values" (e.g. non-categorical) — the
|
|
255
|
+
# ~is_cat masking below converts those to nulls.
|
|
256
|
+
if values_format == "string":
|
|
257
|
+
# Cast value_code to string and strip a trailing ".0" so Float64
|
|
258
|
+
# codes (1.0) render as "1" while genuine fractional codes (1.5) are
|
|
259
|
+
# preserved.
|
|
260
|
+
formatted_pair = pl.format(
|
|
261
|
+
"{}={}",
|
|
262
|
+
pl.element().struct.field("value_code")
|
|
263
|
+
.cast(pl.String)
|
|
264
|
+
.str.replace(r"\.0$", ""),
|
|
265
|
+
pl.element().struct.field("value_label"),
|
|
266
|
+
)
|
|
267
|
+
summary = summary.with_columns(
|
|
268
|
+
pl.col("values")
|
|
269
|
+
.list.eval(formatted_pair)
|
|
270
|
+
.list.join("\n")
|
|
271
|
+
.alias("values")
|
|
272
|
+
)
|
|
273
|
+
|
|
274
|
+
# For non-categorical: set n_labeled, n_unlabeled, values to null
|
|
275
|
+
# Also fix n_unique for non-categorical (should be value_n from summary row, not 0)
|
|
276
|
+
is_cat = pl.col("variable_type") == "categorical"
|
|
277
|
+
values_null = pl.lit(None, dtype=pl.String) if values_format == "string" else pl.lit(None)
|
|
278
|
+
summary = summary.with_columns(
|
|
279
|
+
pl.when(~is_cat).then(pl.lit(None, dtype=pl.UInt32)).otherwise(pl.col("n_labeled")).alias("n_labeled"),
|
|
280
|
+
pl.when(~is_cat).then(pl.lit(None, dtype=pl.UInt32)).otherwise(pl.col("n_unlabeled")).alias("n_unlabeled"),
|
|
281
|
+
pl.when(~is_cat).then(values_null).otherwise(pl.col("values")).alias("values"),
|
|
282
|
+
# Non-cat n_unique: should equal n_valid (preserved quirk from original)
|
|
283
|
+
pl.when(~is_cat).then(pl.col("n_valid")).otherwise(pl.col("n_unique")).alias("n_unique"),
|
|
284
|
+
)
|
|
285
|
+
|
|
286
|
+
# Cast to proper dtypes
|
|
287
|
+
summary = summary.with_columns(
|
|
288
|
+
pl.col("n_valid").cast(pl.UInt32),
|
|
289
|
+
pl.col("n_missing").cast(pl.UInt32),
|
|
290
|
+
pl.col("n_unique").cast(pl.UInt32),
|
|
291
|
+
)
|
|
292
|
+
|
|
293
|
+
# Reorder columns
|
|
294
|
+
return summary.select([
|
|
295
|
+
"variable", "variable_label", "variable_type", "values",
|
|
296
|
+
"n_valid", "n_missing", "n_total",
|
|
297
|
+
"n_unique", "n_labeled", "n_unlabeled",
|
|
298
|
+
])
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def _empty_detail(include_meta: bool = False) -> pl.DataFrame:
|
|
302
|
+
schema = {
|
|
303
|
+
"variable": pl.String, "variable_label": pl.String,
|
|
304
|
+
"variable_type": pl.String,
|
|
305
|
+
}
|
|
306
|
+
if include_meta:
|
|
307
|
+
schema["variable_measure"] = pl.String
|
|
308
|
+
schema["variable_format"] = pl.String
|
|
309
|
+
schema.update({
|
|
310
|
+
"value_code": pl.Int64, "value_label": pl.String,
|
|
311
|
+
"value_n": pl.UInt32, "n_valid": pl.UInt32,
|
|
312
|
+
"pct_valid": pl.Float64, "n_total": pl.UInt32,
|
|
313
|
+
"pct_total": pl.Float64,
|
|
314
|
+
})
|
|
315
|
+
return pl.DataFrame(schema=schema)
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def _empty_summary(values_format: str = "string") -> pl.DataFrame:
|
|
319
|
+
if values_format == "string":
|
|
320
|
+
values_dtype = pl.String
|
|
321
|
+
else:
|
|
322
|
+
values_dtype = pl.List(pl.Struct({"value_code": pl.Int64, "value_label": pl.String}))
|
|
323
|
+
return pl.DataFrame(schema={
|
|
324
|
+
"variable": pl.String, "variable_label": pl.String,
|
|
325
|
+
"variable_type": pl.String,
|
|
326
|
+
"values": values_dtype,
|
|
327
|
+
"n_valid": pl.UInt32, "n_missing": pl.UInt32,
|
|
328
|
+
"n_total": pl.UInt32, "n_unique": pl.UInt32,
|
|
329
|
+
"n_labeled": pl.UInt32, "n_unlabeled": pl.UInt32,
|
|
330
|
+
})
|