yamaa 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- yamaa-0.1.0/.gitignore +9 -0
- yamaa-0.1.0/PKG-INFO +344 -0
- yamaa-0.1.0/README.md +326 -0
- yamaa-0.1.0/agents.md +8 -0
- yamaa-0.1.0/pyproject.toml +35 -0
- yamaa-0.1.0/src/yamaa/__init__.py +9 -0
- yamaa-0.1.0/src/yamaa/domain.py +252 -0
- yamaa-0.1.0/src/yamaa/expressions/__init__.py +96 -0
- yamaa-0.1.0/src/yamaa/expressions/core.py +350 -0
- yamaa-0.1.0/src/yamaa/expressions/dispatch.py +92 -0
- yamaa-0.1.0/src/yamaa/expressions/numeric.py +650 -0
- yamaa-0.1.0/src/yamaa/expressions/predicates.py +644 -0
- yamaa-0.1.0/src/yamaa/expressions/scalar.py +279 -0
- yamaa-0.1.0/src/yamaa/expressions/strings.py +354 -0
- yamaa-0.1.0/src/yamaa/expressions/text.py +25 -0
- yamaa-0.1.0/src/yamaa/io/README.md +84 -0
- yamaa-0.1.0/src/yamaa/io/__init__.py +71 -0
- yamaa-0.1.0/src/yamaa/io/artifact.py +436 -0
- yamaa-0.1.0/src/yamaa/io/csv.py +244 -0
- yamaa-0.1.0/src/yamaa/io/parquet.py +65 -0
- yamaa-0.1.0/src/yamaa/io/polars.py +114 -0
- yamaa-0.1.0/src/yamaa/io/project.py +657 -0
- yamaa-0.1.0/src/yamaa/io/publish.py +97 -0
- yamaa-0.1.0/src/yamaa/io/source.py +216 -0
- yamaa-0.1.0/src/yamaa/models/__init__.py +51 -0
- yamaa-0.1.0/src/yamaa/models/values.py +464 -0
- yamaa-0.1.0/src/yamaa/odm/README.md +59 -0
- yamaa-0.1.0/src/yamaa/odm/__init__.py +40 -0
- yamaa-0.1.0/src/yamaa/odm/bindings.py +177 -0
- yamaa-0.1.0/src/yamaa/odm/context.py +384 -0
- yamaa-0.1.0/src/yamaa/odm/errors.py +5 -0
- yamaa-0.1.0/src/yamaa/odm/parquet.py +119 -0
- yamaa-0.1.0/src/yamaa/odm/readers.py +602 -0
- yamaa-0.1.0/src/yamaa/odm/schema.py +120 -0
- yamaa-0.1.0/src/yamaa/planning/__init__.py +25 -0
- yamaa-0.1.0/src/yamaa/planning/execution.py +1081 -0
- yamaa-0.1.0/src/yamaa/regex.py +146 -0
- yamaa-0.1.0/src/yamaa/runtime/__init__.py +25 -0
- yamaa-0.1.0/src/yamaa/runtime/executor.py +549 -0
- yamaa-0.1.0/src/yamaa/runtime/lifecycle.py +267 -0
- yamaa-0.1.0/src/yamaa/specification/__init__.py +16 -0
- yamaa-0.1.0/src/yamaa/specification/_yaml.py +234 -0
- yamaa-0.1.0/src/yamaa/specification/diagnostics.py +29 -0
- yamaa-0.1.0/src/yamaa/specification/loader.py +59 -0
- yamaa-0.1.0/src/yamaa/specification/models.py +128 -0
- yamaa-0.1.0/src/yamaa/specification/schema.py +994 -0
- yamaa-0.1.0/src/yamaa/verification/__init__.py +29 -0
- yamaa-0.1.0/src/yamaa/verification/checks.py +830 -0
- yamaa-0.1.0/src/yamaa/verification/diagnostics.py +61 -0
- yamaa-0.1.0/tests/expressions/test_core.py +189 -0
- yamaa-0.1.0/tests/expressions/test_numeric.py +335 -0
- yamaa-0.1.0/tests/expressions/test_predicates.py +195 -0
- yamaa-0.1.0/tests/expressions/test_scalar.py +289 -0
- yamaa-0.1.0/tests/expressions/test_strings.py +454 -0
- yamaa-0.1.0/tests/io/test_artifact.py +213 -0
- yamaa-0.1.0/tests/io/test_artifact_csv.py +260 -0
- yamaa-0.1.0/tests/io/test_artifact_parquet.py +163 -0
- yamaa-0.1.0/tests/io/test_csv.py +72 -0
- yamaa-0.1.0/tests/io/test_polars.py +65 -0
- yamaa-0.1.0/tests/io/test_project.py +586 -0
- yamaa-0.1.0/tests/io/test_publish.py +109 -0
- yamaa-0.1.0/tests/io/test_source.py +408 -0
- yamaa-0.1.0/tests/models/test_values.py +230 -0
- yamaa-0.1.0/tests/odm/conftest.py +79 -0
- yamaa-0.1.0/tests/odm/test_bindings.py +86 -0
- yamaa-0.1.0/tests/odm/test_context.py +497 -0
- yamaa-0.1.0/tests/odm/test_parquet.py +143 -0
- yamaa-0.1.0/tests/odm/test_readers.py +213 -0
- yamaa-0.1.0/tests/odm/test_schema.py +66 -0
- yamaa-0.1.0/tests/planning/test_execution.py +233 -0
- yamaa-0.1.0/tests/runtime/test_executor.py +367 -0
- yamaa-0.1.0/tests/runtime/test_expression_examples.py +201 -0
- yamaa-0.1.0/tests/runtime/test_lifecycle.py +77 -0
- yamaa-0.1.0/tests/specification/test_loader.py +519 -0
- yamaa-0.1.0/tests/test_domain.py +90 -0
- yamaa-0.1.0/tests/test_examples.py +48 -0
- yamaa-0.1.0/tests/test_package.py +14 -0
- yamaa-0.1.0/tests/verification/test_checks.py +705 -0
- yamaa-0.1.0/uv.lock +627 -0
yamaa-0.1.0/.gitignore
ADDED
yamaa-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,344 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: yamaa
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Python helpers for the YAMAA clinical data specification
|
|
5
|
+
Author: YAMAA contributors
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Requires-Python: >=3.11
|
|
8
|
+
Requires-Dist: lxml<7,>=6.1.3
|
|
9
|
+
Requires-Dist: polars<2,>=1.35.2
|
|
10
|
+
Requires-Dist: pyarrow<26,>=25.0.1
|
|
11
|
+
Requires-Dist: pydantic<3,>=2.13.5
|
|
12
|
+
Requires-Dist: pyyaml<7,>=6.0.3
|
|
13
|
+
Requires-Dist: regress==2025.10.1
|
|
14
|
+
Provides-Extra: test
|
|
15
|
+
Requires-Dist: pytest<10,>=9.1.1; extra == 'test'
|
|
16
|
+
Requires-Dist: ruff<0.17,>=0.16.7; extra == 'test'
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
|
|
19
|
+
# YAMAA Python
|
|
20
|
+
|
|
21
|
+
The Python package currently provides general CDISC ODM helpers. It uses
|
|
22
|
+
Pydantic for public data contracts and Polars for tabular data operations.
|
|
23
|
+
The package supports Python 3.11 and newer; CI exercises Python 3.11 and 3.14.
|
|
24
|
+
|
|
25
|
+
## Install and test
|
|
26
|
+
|
|
27
|
+
From the repository root, create the locked test environment and run all Python
|
|
28
|
+
checks:
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
uv sync --project python --extra test --locked --no-editable
|
|
32
|
+
uv run --project python --no-sync ruff format --check python/src/yamaa python/tests
|
|
33
|
+
uv run --project python --no-sync ruff check python/src/yamaa python/tests
|
|
34
|
+
uv run --project python --no-sync pytest python/tests
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
For installation with `pip`:
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
python -m pip install './python[test]'
|
|
41
|
+
python -m pytest python/tests
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Tests import the installed package. They do not require `PYTHONPATH` or depend
|
|
45
|
+
on the repository's current working directory.
|
|
46
|
+
|
|
47
|
+
## ODM helpers
|
|
48
|
+
|
|
49
|
+
```python
|
|
50
|
+
from yamaa.odm import iter_odm_records, read_odm, write_odm_parquet
|
|
51
|
+
|
|
52
|
+
frame = read_odm("input.xml")
|
|
53
|
+
result = write_odm_parquet("input.xml", "clinical-items.parquet")
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
See the [ODM helper documentation](src/yamaa/odm/README.md) for the fixed schema
|
|
57
|
+
and supported XML layouts.
|
|
58
|
+
|
|
59
|
+
## Specification loader
|
|
60
|
+
|
|
61
|
+
Load one specification against the repository schema bundle:
|
|
62
|
+
|
|
63
|
+
```python
|
|
64
|
+
from yamaa.specification import load_specification
|
|
65
|
+
|
|
66
|
+
loaded = load_specification("study/spec.yaml", "yaml")
|
|
67
|
+
print(loaded.specification.domain)
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
The loader applies YAML 1.2 core scalar rules, rejects YAML features outside
|
|
71
|
+
the authored-source contract, reads safe schema includes, validates the document
|
|
72
|
+
against the bundle, materializes R006 shorthands and defaults, and returns strict
|
|
73
|
+
Pydantic models. It does not execute the specification or resolve inheritance.
|
|
74
|
+
|
|
75
|
+
## Typed values and scalar expressions
|
|
76
|
+
|
|
77
|
+
The runtime value kernel exposes strict Pydantic result models, explicit
|
|
78
|
+
missingness, R011 conversions, R016 date and datetime values, an ordered Polars
|
|
79
|
+
table contract, and R004 predicate evaluation.
|
|
80
|
+
|
|
81
|
+
### Registered operations
|
|
82
|
+
|
|
83
|
+
Scalar dispatch supports exactly these fourteen operations. Every other
|
|
84
|
+
registered keyword returns an explicit `UnsupportedResult` until its owning
|
|
85
|
+
runtime component is implemented, and an unregistered keyword fails schema
|
|
86
|
+
validation before it reaches dispatch.
|
|
87
|
+
|
|
88
|
+
| Operation | Rule | What it returns |
|
|
89
|
+
|---|---|---|
|
|
90
|
+
| `source` | R002, R008 | the named source or derived variable |
|
|
91
|
+
| `literal` | R007 | the declared scalar, unchanged |
|
|
92
|
+
| `mapping` | R007, R019 | an inline dictionary lookup on a string source |
|
|
93
|
+
| `compute` | R010 | one scalar numeric formula in the closed grammar |
|
|
94
|
+
| `coalesce` | R007 | the first non-missing source, else `default` |
|
|
95
|
+
| `greatest`, `least` | R007, R019 | the row-wise extreme of comparable sources |
|
|
96
|
+
| `case` | R004, R007 | the first true branch, then `otherwise`, else missing |
|
|
97
|
+
| `cut` | R007 | the label of the break interval a numeric source lands in |
|
|
98
|
+
| `str_extract` | R022 | one capture group of the leftmost match |
|
|
99
|
+
| `str_concat` | R007 | its nested expression results, in order |
|
|
100
|
+
| `str_template` | R012 | literal text with its placeholders interpolated |
|
|
101
|
+
| `str_upper`, `str_lower` | R019 | the exact ASCII casing substitution |
|
|
102
|
+
|
|
103
|
+
`compute` reads the closed R010 grammar: the operators `+ - * /` with unary
|
|
104
|
+
sign, and exactly `ABS`, `CEIL`, `FLOOR`, `TRUNC`, `SQRT`, `POWER`, `EXP`,
|
|
105
|
+
`LN`, `MOD`, `GREATEST`, `LEAST`, `NULLIF`, and `COALESCE`. There is no host
|
|
106
|
+
`eval`: a formula is tokenized, parsed, and evaluated in the association it
|
|
107
|
+
was written in, and a division by zero, a negative `SQRT`, a non-positive
|
|
108
|
+
`LN`, an invalid `POWER`, or an integer overflow fails the run rather than
|
|
109
|
+
becoming missing. The relational, temporal, window, aggregate, and project
|
|
110
|
+
function families remain unsupported.
|
|
111
|
+
|
|
112
|
+
Every regular expression in the package -- the R006 `pattern` descriptor, the
|
|
113
|
+
R009 `matches` verification, and `str_extract` -- is read by `yamaa.regex`,
|
|
114
|
+
the single binding of the `regress` distribution R022 pins. Python `re` reads
|
|
115
|
+
no pattern of the language. The shared vectors in `yaml/conformance/regex.yaml`
|
|
116
|
+
are replayed through all three consumers.
|
|
117
|
+
|
|
118
|
+
`yamaa.expressions` also exposes the two closed parsers directly -- the R010
|
|
119
|
+
`parse_numeric` and the R012 `parse_template` -- each checked against the
|
|
120
|
+
vectors in `yaml/grammar/`, which is the single source those grammars are
|
|
121
|
+
written in.
|
|
122
|
+
|
|
123
|
+
```python
|
|
124
|
+
from yamaa.expressions import MappingResolver, evaluate_expression
|
|
125
|
+
from yamaa.models import ValueResult, convert_value
|
|
126
|
+
from yamaa.specification.models import Expression
|
|
127
|
+
|
|
128
|
+
resolver = MappingResolver({"RAW.AGE": "42"})
|
|
129
|
+
expression = Expression(root={"source": {"variable": "RAW.AGE"}})
|
|
130
|
+
source = evaluate_expression(expression, resolver)
|
|
131
|
+
|
|
132
|
+
assert isinstance(source, ValueResult)
|
|
133
|
+
age = convert_value(source.value, "int")
|
|
134
|
+
assert age == ValueResult(value=42)
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
Run this component's focused tests from the repository root:
|
|
138
|
+
|
|
139
|
+
```bash
|
|
140
|
+
uv run --project python --isolated --extra test pytest \
|
|
141
|
+
python/tests/models python/tests/expressions \
|
|
142
|
+
python/tests/runtime/test_expression_examples.py
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
## ODM source binding and contextual resolution
|
|
146
|
+
|
|
147
|
+
Build one binding plan from a normalized specification and its loaded source
|
|
148
|
+
tables, then share one index across row-local resolvers:
|
|
149
|
+
|
|
150
|
+
```python
|
|
151
|
+
from yamaa.odm import BindingIndex, build_binding_plan
|
|
152
|
+
|
|
153
|
+
plan = build_binding_plan(loaded_spec.specification, loaded_sources)
|
|
154
|
+
index = BindingIndex(plan, loaded_sources)
|
|
155
|
+
resolver = index.context({"ODM": current_odm_row}, {"STUDYID": "STUDY01"})
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
The resolver implements the scalar expression protocol. Direct qualified
|
|
159
|
+
fields read the supplied source record, unqualified names read completed output
|
|
160
|
+
values, and a long-form ODM item uses every context column present in the ODM
|
|
161
|
+
projection. Duplicate ODM items require a structured R008 `multiple_matches`
|
|
162
|
+
policy; successful duplicate selection is returned with
|
|
163
|
+
`handled_by="multiple_matches"` so the executor can count that path.
|
|
164
|
+
Implicit cross-dataset row selection remains the keyed-join component's
|
|
165
|
+
responsibility; this context resolves only source records its caller has
|
|
166
|
+
explicitly bound.
|
|
167
|
+
|
|
168
|
+
Run this component's focused tests from the repository root:
|
|
169
|
+
|
|
170
|
+
```bash
|
|
171
|
+
uv run --project python --isolated --extra test pytest \
|
|
172
|
+
python/tests/odm
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
## CSV source ingestion
|
|
176
|
+
|
|
177
|
+
Create one resource manager for the approved project and use normalized
|
|
178
|
+
`DatasetSource` declarations to load ordered, typed Polars tables:
|
|
179
|
+
|
|
180
|
+
```python
|
|
181
|
+
from yamaa.io import ProjectResources, load_source_tables
|
|
182
|
+
from yamaa.specification.models import DatasetSource
|
|
183
|
+
|
|
184
|
+
resources = ProjectResources("study")
|
|
185
|
+
datasets = {
|
|
186
|
+
"DM": DatasetSource(
|
|
187
|
+
path="input/dm.csv",
|
|
188
|
+
types={"AGE": "int", "RFSTDTC": "date"},
|
|
189
|
+
)
|
|
190
|
+
}
|
|
191
|
+
loaded = load_source_tables(datasets, resources)
|
|
192
|
+
dm = loaded["DM"].table
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
### Keeping code and data in different places
|
|
196
|
+
|
|
197
|
+
A study that keeps its data outside its specifications says so in its own
|
|
198
|
+
`yamaa-project.yaml`, at the top of the study:
|
|
199
|
+
|
|
200
|
+
```yaml
|
|
201
|
+
version: "1.0"
|
|
202
|
+
data_roots:
|
|
203
|
+
- /data/pilot7
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
A run finds that file by walking up from the entry specification, and the
|
|
207
|
+
directory holding it is the project root. Data may then be declared by a
|
|
208
|
+
rooted path:
|
|
209
|
+
|
|
210
|
+
```python
|
|
211
|
+
from yamaa.io import ProjectResources, approve_roots
|
|
212
|
+
|
|
213
|
+
approved = approve_roots("study/adam/adsl/spec.yaml")
|
|
214
|
+
resources = ProjectResources(approved.project_root, data_roots=approved.data_roots)
|
|
215
|
+
datasets = {"LBREF": DatasetSource(path="/data/pilot7/reference/lbref.csv")}
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
A rooted path naming no approved root fails as `resource_path_not_relative`,
|
|
219
|
+
and the failure names the written path alone. A study that ships no
|
|
220
|
+
configuration behaves exactly as before: the entry file's directory is the
|
|
221
|
+
project root and nothing outside it can be read.
|
|
222
|
+
|
|
223
|
+
The roots are fixed before any specification is read, and only the entry
|
|
224
|
+
study's own configuration contributes to them. A layer inherited under R017
|
|
225
|
+
never widens them, so an organization template cannot redirect where a study
|
|
226
|
+
reads from. A runner keeps the last word over a study it did not write:
|
|
227
|
+
|
|
228
|
+
```python
|
|
229
|
+
# Cap what the configuration may approve; a root outside these fails the run.
|
|
230
|
+
approve_roots(entry, data_roots=["/data"])
|
|
231
|
+
|
|
232
|
+
# Decline the configuration's roots entirely, as a packaging run does.
|
|
233
|
+
approve_roots(entry, read_project_configuration=False)
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
The reader accepts the fixed `.csv` profile only. It parses the retained byte
|
|
237
|
+
snapshot in memory, preserves source row and header order, and reads a field
|
|
238
|
+
with no characters as missing whether it was written bare or quoted, before
|
|
239
|
+
applying declared types. A `str`, `int`, or `float` column lands in the
|
|
240
|
+
matching native Polars type, a `date` column in `pl.Date`, and a `datetime`
|
|
241
|
+
column in `pl.Datetime("us")`, so an ingested table answers ordinary Polars
|
|
242
|
+
expressions. The reader does not create CSV or other intermediate files.
|
|
243
|
+
Producer-linked `schema` workflow resolution and header-contract comparison
|
|
244
|
+
remain part of the workflow component; this ingestion API rejects a declaration
|
|
245
|
+
carrying `schema` until that component supplies its resolved producer contract.
|
|
246
|
+
|
|
247
|
+
The R023 syntax scanner in `yamaa.io.csv` imports the standard library alone.
|
|
248
|
+
The repository validator loads that module by path rather than keeping a second
|
|
249
|
+
reader, so one implementation decides how every fixture reads.
|
|
250
|
+
|
|
251
|
+
Run this component's focused tests from the repository root:
|
|
252
|
+
|
|
253
|
+
```bash
|
|
254
|
+
uv run --project python --isolated --extra test pytest python/tests/io
|
|
255
|
+
```
|
|
256
|
+
|
|
257
|
+
## Minimal YAML execution
|
|
258
|
+
|
|
259
|
+
Load and execute one domain specification with the user-facing facade:
|
|
260
|
+
|
|
261
|
+
```python
|
|
262
|
+
from yamaa import yamaa_domain
|
|
263
|
+
|
|
264
|
+
pilot = yamaa_domain("spec.yaml")
|
|
265
|
+
|
|
266
|
+
print(pilot.spec) # normalized specification
|
|
267
|
+
print(pilot.inputs) # dataset name -> Polars DataFrame
|
|
268
|
+
print(pilot.output) # ordered output Polars DataFrame, or None
|
|
269
|
+
print(pilot.issues) # stable Polars issue table
|
|
270
|
+
|
|
271
|
+
pilot.save("expected.parquet")
|
|
272
|
+
```
|
|
273
|
+
|
|
274
|
+
`yamaa_domain` searches upward from the specification for `schema.yaml`; a
|
|
275
|
+
specification kept elsewhere supplies `schema_root=`. Project and data roots
|
|
276
|
+
follow the same R021 configuration rules as the lower-level resource API.
|
|
277
|
+
Loading, source capture, and execution happen once. The facade writes nothing
|
|
278
|
+
until `save` is called: without an argument it uses the specification's
|
|
279
|
+
`output.path`, while an explicit `.csv` or `.parquet` path selects that output
|
|
280
|
+
profile. An unsuccessful run exposes no output and `save` raises
|
|
281
|
+
`DomainRunError` with the same issue table available from `pilot.issues`.
|
|
282
|
+
|
|
283
|
+
The lower-level executor remains available for adapters and injected test
|
|
284
|
+
hooks. It plans dependencies before evaluation, constructs record-driven rows
|
|
285
|
+
in specification and source order, then enriches those rows without changing
|
|
286
|
+
their count. The provider entry point completes all source-independent
|
|
287
|
+
validation before it asks for source bytes. Each scalar completes expression
|
|
288
|
+
evaluation, declared type conversion, conversion handling, and first-match
|
|
289
|
+
override before a dependent reads it. Handler counts include declared paths
|
|
290
|
+
that fired zero times. Column, key, dataset-verification, and ordered-output
|
|
291
|
+
work is delegated to the pure hooks exposed by the verification and I/O
|
|
292
|
+
components.
|
|
293
|
+
|
|
294
|
+
Execution supports every operation in the registered table above, along with
|
|
295
|
+
ungrouped row filters, explicit absent-source defaults, and earlier
|
|
296
|
+
output-column references. Grouped rows, record lookups, inheritance, and the
|
|
297
|
+
relational, temporal, window, aggregate, and project function families return
|
|
298
|
+
an explicit unsupported result rather than a fabricated output. Execution
|
|
299
|
+
never reads an `expected/` artifact.
|
|
300
|
+
|
|
301
|
+
Run the focused tests from the repository root:
|
|
302
|
+
|
|
303
|
+
```bash
|
|
304
|
+
uv run --project python --isolated --extra test pytest \
|
|
305
|
+
python/tests/test_domain.py python/tests/planning python/tests/runtime
|
|
306
|
+
```
|
|
307
|
+
|
|
308
|
+
## Verified tables and published artifacts
|
|
309
|
+
|
|
310
|
+
Assert over a completed table, then write and publish what it produces:
|
|
311
|
+
|
|
312
|
+
```python
|
|
313
|
+
from yamaa.io import ArtifactTarget, build_artifact, publish_artifact
|
|
314
|
+
from yamaa.verification import verify_completed_table
|
|
315
|
+
|
|
316
|
+
verify_completed_table(table, spec.columns, spec.keys, spec.verifications or [])
|
|
317
|
+
artifact = build_artifact(table, spec.output, spec.keys)
|
|
318
|
+
publish_artifact(ArtifactTarget(run_directory / "adsl.csv"), artifact)
|
|
319
|
+
```
|
|
320
|
+
|
|
321
|
+
`yamaa.verification` exposes one hook per R005 stage -- `check_column`,
|
|
322
|
+
`check_keys`, and `check_dataset` -- so an executor runs each assertion when
|
|
323
|
+
R005 says it runs rather than sweeping every check to the end. Each reports
|
|
324
|
+
failures in the committed error shape and leaves the run's fate to its
|
|
325
|
+
caller; `verify_completed_table` runs the three in order and raises. Dataset
|
|
326
|
+
verification accepts typed, per-row record-lookup bindings for the qualified
|
|
327
|
+
fields R004-26 makes visible to predicates.
|
|
328
|
+
|
|
329
|
+
`yamaa.io` writes the other way for the same reason it reads: the artifact
|
|
330
|
+
selects its profile from `output.path`, takes R005's column selection and
|
|
331
|
+
row order, and becomes `.csv` byte for byte or `.parquet` under the R020
|
|
332
|
+
type mapping. Publication replaces one target the caller explicitly
|
|
333
|
+
permits, through a temporary file beside it, so a failure leaves the
|
|
334
|
+
previous artifact in place.
|
|
335
|
+
|
|
336
|
+
See the [input and output documentation](src/yamaa/io/README.md) for what
|
|
337
|
+
each step owns.
|
|
338
|
+
|
|
339
|
+
Run this component's focused tests from the repository root:
|
|
340
|
+
|
|
341
|
+
```bash
|
|
342
|
+
uv run --project python --isolated --extra test pytest \
|
|
343
|
+
python/tests/verification python/tests/io
|
|
344
|
+
```
|
yamaa-0.1.0/README.md
ADDED
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
# YAMAA Python
|
|
2
|
+
|
|
3
|
+
The Python package currently provides general CDISC ODM helpers. It uses
|
|
4
|
+
Pydantic for public data contracts and Polars for tabular data operations.
|
|
5
|
+
The package supports Python 3.11 and newer; CI exercises Python 3.11 and 3.14.
|
|
6
|
+
|
|
7
|
+
## Install and test
|
|
8
|
+
|
|
9
|
+
From the repository root, create the locked test environment and run all Python
|
|
10
|
+
checks:
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
uv sync --project python --extra test --locked --no-editable
|
|
14
|
+
uv run --project python --no-sync ruff format --check python/src/yamaa python/tests
|
|
15
|
+
uv run --project python --no-sync ruff check python/src/yamaa python/tests
|
|
16
|
+
uv run --project python --no-sync pytest python/tests
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
For installation with `pip`:
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
python -m pip install './python[test]'
|
|
23
|
+
python -m pytest python/tests
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
Tests import the installed package. They do not require `PYTHONPATH` or depend
|
|
27
|
+
on the repository's current working directory.
|
|
28
|
+
|
|
29
|
+
## ODM helpers
|
|
30
|
+
|
|
31
|
+
```python
|
|
32
|
+
from yamaa.odm import iter_odm_records, read_odm, write_odm_parquet
|
|
33
|
+
|
|
34
|
+
frame = read_odm("input.xml")
|
|
35
|
+
result = write_odm_parquet("input.xml", "clinical-items.parquet")
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
See the [ODM helper documentation](src/yamaa/odm/README.md) for the fixed schema
|
|
39
|
+
and supported XML layouts.
|
|
40
|
+
|
|
41
|
+
## Specification loader
|
|
42
|
+
|
|
43
|
+
Load one specification against the repository schema bundle:
|
|
44
|
+
|
|
45
|
+
```python
|
|
46
|
+
from yamaa.specification import load_specification
|
|
47
|
+
|
|
48
|
+
loaded = load_specification("study/spec.yaml", "yaml")
|
|
49
|
+
print(loaded.specification.domain)
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
The loader applies YAML 1.2 core scalar rules, rejects YAML features outside
|
|
53
|
+
the authored-source contract, reads safe schema includes, validates the document
|
|
54
|
+
against the bundle, materializes R006 shorthands and defaults, and returns strict
|
|
55
|
+
Pydantic models. It does not execute the specification or resolve inheritance.
|
|
56
|
+
|
|
57
|
+
## Typed values and scalar expressions
|
|
58
|
+
|
|
59
|
+
The runtime value kernel exposes strict Pydantic result models, explicit
|
|
60
|
+
missingness, R011 conversions, R016 date and datetime values, an ordered Polars
|
|
61
|
+
table contract, and R004 predicate evaluation.
|
|
62
|
+
|
|
63
|
+
### Registered operations
|
|
64
|
+
|
|
65
|
+
Scalar dispatch supports exactly these fourteen operations. Every other
|
|
66
|
+
registered keyword returns an explicit `UnsupportedResult` until its owning
|
|
67
|
+
runtime component is implemented, and an unregistered keyword fails schema
|
|
68
|
+
validation before it reaches dispatch.
|
|
69
|
+
|
|
70
|
+
| Operation | Rule | What it returns |
|
|
71
|
+
|---|---|---|
|
|
72
|
+
| `source` | R002, R008 | the named source or derived variable |
|
|
73
|
+
| `literal` | R007 | the declared scalar, unchanged |
|
|
74
|
+
| `mapping` | R007, R019 | an inline dictionary lookup on a string source |
|
|
75
|
+
| `compute` | R010 | one scalar numeric formula in the closed grammar |
|
|
76
|
+
| `coalesce` | R007 | the first non-missing source, else `default` |
|
|
77
|
+
| `greatest`, `least` | R007, R019 | the row-wise extreme of comparable sources |
|
|
78
|
+
| `case` | R004, R007 | the first true branch, then `otherwise`, else missing |
|
|
79
|
+
| `cut` | R007 | the label of the break interval a numeric source lands in |
|
|
80
|
+
| `str_extract` | R022 | one capture group of the leftmost match |
|
|
81
|
+
| `str_concat` | R007 | its nested expression results, in order |
|
|
82
|
+
| `str_template` | R012 | literal text with its placeholders interpolated |
|
|
83
|
+
| `str_upper`, `str_lower` | R019 | the exact ASCII casing substitution |
|
|
84
|
+
|
|
85
|
+
`compute` reads the closed R010 grammar: the operators `+ - * /` with unary
|
|
86
|
+
sign, and exactly `ABS`, `CEIL`, `FLOOR`, `TRUNC`, `SQRT`, `POWER`, `EXP`,
|
|
87
|
+
`LN`, `MOD`, `GREATEST`, `LEAST`, `NULLIF`, and `COALESCE`. There is no host
|
|
88
|
+
`eval`: a formula is tokenized, parsed, and evaluated in the association it
|
|
89
|
+
was written in, and a division by zero, a negative `SQRT`, a non-positive
|
|
90
|
+
`LN`, an invalid `POWER`, or an integer overflow fails the run rather than
|
|
91
|
+
becoming missing. The relational, temporal, window, aggregate, and project
|
|
92
|
+
function families remain unsupported.
|
|
93
|
+
|
|
94
|
+
Every regular expression in the package -- the R006 `pattern` descriptor, the
|
|
95
|
+
R009 `matches` verification, and `str_extract` -- is read by `yamaa.regex`,
|
|
96
|
+
the single binding of the `regress` distribution R022 pins. Python `re` reads
|
|
97
|
+
no pattern of the language. The shared vectors in `yaml/conformance/regex.yaml`
|
|
98
|
+
are replayed through all three consumers.
|
|
99
|
+
|
|
100
|
+
`yamaa.expressions` also exposes the two closed parsers directly -- the R010
|
|
101
|
+
`parse_numeric` and the R012 `parse_template` -- each checked against the
|
|
102
|
+
vectors in `yaml/grammar/`, which is the single source those grammars are
|
|
103
|
+
written in.
|
|
104
|
+
|
|
105
|
+
```python
|
|
106
|
+
from yamaa.expressions import MappingResolver, evaluate_expression
|
|
107
|
+
from yamaa.models import ValueResult, convert_value
|
|
108
|
+
from yamaa.specification.models import Expression
|
|
109
|
+
|
|
110
|
+
resolver = MappingResolver({"RAW.AGE": "42"})
|
|
111
|
+
expression = Expression(root={"source": {"variable": "RAW.AGE"}})
|
|
112
|
+
source = evaluate_expression(expression, resolver)
|
|
113
|
+
|
|
114
|
+
assert isinstance(source, ValueResult)
|
|
115
|
+
age = convert_value(source.value, "int")
|
|
116
|
+
assert age == ValueResult(value=42)
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
Run this component's focused tests from the repository root:
|
|
120
|
+
|
|
121
|
+
```bash
|
|
122
|
+
uv run --project python --isolated --extra test pytest \
|
|
123
|
+
python/tests/models python/tests/expressions \
|
|
124
|
+
python/tests/runtime/test_expression_examples.py
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
## ODM source binding and contextual resolution
|
|
128
|
+
|
|
129
|
+
Build one binding plan from a normalized specification and its loaded source
|
|
130
|
+
tables, then share one index across row-local resolvers:
|
|
131
|
+
|
|
132
|
+
```python
|
|
133
|
+
from yamaa.odm import BindingIndex, build_binding_plan
|
|
134
|
+
|
|
135
|
+
plan = build_binding_plan(loaded_spec.specification, loaded_sources)
|
|
136
|
+
index = BindingIndex(plan, loaded_sources)
|
|
137
|
+
resolver = index.context({"ODM": current_odm_row}, {"STUDYID": "STUDY01"})
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
The resolver implements the scalar expression protocol. Direct qualified
|
|
141
|
+
fields read the supplied source record, unqualified names read completed output
|
|
142
|
+
values, and a long-form ODM item uses every context column present in the ODM
|
|
143
|
+
projection. Duplicate ODM items require a structured R008 `multiple_matches`
|
|
144
|
+
policy; successful duplicate selection is returned with
|
|
145
|
+
`handled_by="multiple_matches"` so the executor can count that path.
|
|
146
|
+
Implicit cross-dataset row selection remains the keyed-join component's
|
|
147
|
+
responsibility; this context resolves only source records its caller has
|
|
148
|
+
explicitly bound.
|
|
149
|
+
|
|
150
|
+
Run this component's focused tests from the repository root:
|
|
151
|
+
|
|
152
|
+
```bash
|
|
153
|
+
uv run --project python --isolated --extra test pytest \
|
|
154
|
+
python/tests/odm
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
## CSV source ingestion
|
|
158
|
+
|
|
159
|
+
Create one resource manager for the approved project and use normalized
|
|
160
|
+
`DatasetSource` declarations to load ordered, typed Polars tables:
|
|
161
|
+
|
|
162
|
+
```python
|
|
163
|
+
from yamaa.io import ProjectResources, load_source_tables
|
|
164
|
+
from yamaa.specification.models import DatasetSource
|
|
165
|
+
|
|
166
|
+
resources = ProjectResources("study")
|
|
167
|
+
datasets = {
|
|
168
|
+
"DM": DatasetSource(
|
|
169
|
+
path="input/dm.csv",
|
|
170
|
+
types={"AGE": "int", "RFSTDTC": "date"},
|
|
171
|
+
)
|
|
172
|
+
}
|
|
173
|
+
loaded = load_source_tables(datasets, resources)
|
|
174
|
+
dm = loaded["DM"].table
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
### Keeping code and data in different places
|
|
178
|
+
|
|
179
|
+
A study that keeps its data outside its specifications says so in its own
|
|
180
|
+
`yamaa-project.yaml`, at the top of the study:
|
|
181
|
+
|
|
182
|
+
```yaml
|
|
183
|
+
version: "1.0"
|
|
184
|
+
data_roots:
|
|
185
|
+
- /data/pilot7
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
A run finds that file by walking up from the entry specification, and the
|
|
189
|
+
directory holding it is the project root. Data may then be declared by a
|
|
190
|
+
rooted path:
|
|
191
|
+
|
|
192
|
+
```python
|
|
193
|
+
from yamaa.io import ProjectResources, approve_roots
|
|
194
|
+
|
|
195
|
+
approved = approve_roots("study/adam/adsl/spec.yaml")
|
|
196
|
+
resources = ProjectResources(approved.project_root, data_roots=approved.data_roots)
|
|
197
|
+
datasets = {"LBREF": DatasetSource(path="/data/pilot7/reference/lbref.csv")}
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
A rooted path naming no approved root fails as `resource_path_not_relative`,
|
|
201
|
+
and the failure names the written path alone. A study that ships no
|
|
202
|
+
configuration behaves exactly as before: the entry file's directory is the
|
|
203
|
+
project root and nothing outside it can be read.
|
|
204
|
+
|
|
205
|
+
The roots are fixed before any specification is read, and only the entry
|
|
206
|
+
study's own configuration contributes to them. A layer inherited under R017
|
|
207
|
+
never widens them, so an organization template cannot redirect where a study
|
|
208
|
+
reads from. A runner keeps the last word over a study it did not write:
|
|
209
|
+
|
|
210
|
+
```python
|
|
211
|
+
# Cap what the configuration may approve; a root outside these fails the run.
|
|
212
|
+
approve_roots(entry, data_roots=["/data"])
|
|
213
|
+
|
|
214
|
+
# Decline the configuration's roots entirely, as a packaging run does.
|
|
215
|
+
approve_roots(entry, read_project_configuration=False)
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
The reader accepts the fixed `.csv` profile only. It parses the retained byte
|
|
219
|
+
snapshot in memory, preserves source row and header order, and reads a field
|
|
220
|
+
with no characters as missing whether it was written bare or quoted, before
|
|
221
|
+
applying declared types. A `str`, `int`, or `float` column lands in the
|
|
222
|
+
matching native Polars type, a `date` column in `pl.Date`, and a `datetime`
|
|
223
|
+
column in `pl.Datetime("us")`, so an ingested table answers ordinary Polars
|
|
224
|
+
expressions. The reader does not create CSV or other intermediate files.
|
|
225
|
+
Producer-linked `schema` workflow resolution and header-contract comparison
|
|
226
|
+
remain part of the workflow component; this ingestion API rejects a declaration
|
|
227
|
+
carrying `schema` until that component supplies its resolved producer contract.
|
|
228
|
+
|
|
229
|
+
The R023 syntax scanner in `yamaa.io.csv` imports the standard library alone.
|
|
230
|
+
The repository validator loads that module by path rather than keeping a second
|
|
231
|
+
reader, so one implementation decides how every fixture reads.
|
|
232
|
+
|
|
233
|
+
Run this component's focused tests from the repository root:
|
|
234
|
+
|
|
235
|
+
```bash
|
|
236
|
+
uv run --project python --isolated --extra test pytest python/tests/io
|
|
237
|
+
```
|
|
238
|
+
|
|
239
|
+
## Minimal YAML execution
|
|
240
|
+
|
|
241
|
+
Load and execute one domain specification with the user-facing facade:
|
|
242
|
+
|
|
243
|
+
```python
|
|
244
|
+
from yamaa import yamaa_domain
|
|
245
|
+
|
|
246
|
+
pilot = yamaa_domain("spec.yaml")
|
|
247
|
+
|
|
248
|
+
print(pilot.spec) # normalized specification
|
|
249
|
+
print(pilot.inputs) # dataset name -> Polars DataFrame
|
|
250
|
+
print(pilot.output) # ordered output Polars DataFrame, or None
|
|
251
|
+
print(pilot.issues) # stable Polars issue table
|
|
252
|
+
|
|
253
|
+
pilot.save("expected.parquet")
|
|
254
|
+
```
|
|
255
|
+
|
|
256
|
+
`yamaa_domain` searches upward from the specification for `schema.yaml`; a
|
|
257
|
+
specification kept elsewhere supplies `schema_root=`. Project and data roots
|
|
258
|
+
follow the same R021 configuration rules as the lower-level resource API.
|
|
259
|
+
Loading, source capture, and execution happen once. The facade writes nothing
|
|
260
|
+
until `save` is called: without an argument it uses the specification's
|
|
261
|
+
`output.path`, while an explicit `.csv` or `.parquet` path selects that output
|
|
262
|
+
profile. An unsuccessful run exposes no output and `save` raises
|
|
263
|
+
`DomainRunError` with the same issue table available from `pilot.issues`.
|
|
264
|
+
|
|
265
|
+
The lower-level executor remains available for adapters and injected test
|
|
266
|
+
hooks. It plans dependencies before evaluation, constructs record-driven rows
|
|
267
|
+
in specification and source order, then enriches those rows without changing
|
|
268
|
+
their count. The provider entry point completes all source-independent
|
|
269
|
+
validation before it asks for source bytes. Each scalar completes expression
|
|
270
|
+
evaluation, declared type conversion, conversion handling, and first-match
|
|
271
|
+
override before a dependent reads it. Handler counts include declared paths
|
|
272
|
+
that fired zero times. Column, key, dataset-verification, and ordered-output
|
|
273
|
+
work is delegated to the pure hooks exposed by the verification and I/O
|
|
274
|
+
components.
|
|
275
|
+
|
|
276
|
+
Execution supports every operation in the registered table above, along with
|
|
277
|
+
ungrouped row filters, explicit absent-source defaults, and earlier
|
|
278
|
+
output-column references. Grouped rows, record lookups, inheritance, and the
|
|
279
|
+
relational, temporal, window, aggregate, and project function families return
|
|
280
|
+
an explicit unsupported result rather than a fabricated output. Execution
|
|
281
|
+
never reads an `expected/` artifact.
|
|
282
|
+
|
|
283
|
+
Run the focused tests from the repository root:
|
|
284
|
+
|
|
285
|
+
```bash
|
|
286
|
+
uv run --project python --isolated --extra test pytest \
|
|
287
|
+
python/tests/test_domain.py python/tests/planning python/tests/runtime
|
|
288
|
+
```
|
|
289
|
+
|
|
290
|
+
## Verified tables and published artifacts
|
|
291
|
+
|
|
292
|
+
Assert over a completed table, then write and publish what it produces:
|
|
293
|
+
|
|
294
|
+
```python
|
|
295
|
+
from yamaa.io import ArtifactTarget, build_artifact, publish_artifact
|
|
296
|
+
from yamaa.verification import verify_completed_table
|
|
297
|
+
|
|
298
|
+
verify_completed_table(table, spec.columns, spec.keys, spec.verifications or [])
|
|
299
|
+
artifact = build_artifact(table, spec.output, spec.keys)
|
|
300
|
+
publish_artifact(ArtifactTarget(run_directory / "adsl.csv"), artifact)
|
|
301
|
+
```
|
|
302
|
+
|
|
303
|
+
`yamaa.verification` exposes one hook per R005 stage -- `check_column`,
|
|
304
|
+
`check_keys`, and `check_dataset` -- so an executor runs each assertion when
|
|
305
|
+
R005 says it runs rather than sweeping every check to the end. Each reports
|
|
306
|
+
failures in the committed error shape and leaves the run's fate to its
|
|
307
|
+
caller; `verify_completed_table` runs the three in order and raises. Dataset
|
|
308
|
+
verification accepts typed, per-row record-lookup bindings for the qualified
|
|
309
|
+
fields R004-26 makes visible to predicates.
|
|
310
|
+
|
|
311
|
+
`yamaa.io` writes the other way for the same reason it reads: the artifact
|
|
312
|
+
selects its profile from `output.path`, takes R005's column selection and
|
|
313
|
+
row order, and becomes `.csv` byte for byte or `.parquet` under the R020
|
|
314
|
+
type mapping. Publication replaces one target the caller explicitly
|
|
315
|
+
permits, through a temporary file beside it, so a failure leaves the
|
|
316
|
+
previous artifact in place.
|
|
317
|
+
|
|
318
|
+
See the [input and output documentation](src/yamaa/io/README.md) for what
|
|
319
|
+
each step owns.
|
|
320
|
+
|
|
321
|
+
Run this component's focused tests from the repository root:
|
|
322
|
+
|
|
323
|
+
```bash
|
|
324
|
+
uv run --project python --isolated --extra test pytest \
|
|
325
|
+
python/tests/verification python/tests/io
|
|
326
|
+
```
|
yamaa-0.1.0/agents.md
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# Python Packages Layer
|
|
2
|
+
|
|
3
|
+
This directory is dedicated to the development of Python packages.
|
|
4
|
+
|
|
5
|
+
**Agent Guidelines:**
|
|
6
|
+
- Follow PEP 8 style guidelines for all Python code.
|
|
7
|
+
- Ensure packages are appropriately structured with clear dependency management.
|
|
8
|
+
- Write unit tests for all implemented logic (e.g., using pytest).
|