openstatspec 0.4.0__tar.gz → 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {openstatspec-0.4.0 → openstatspec-0.6.0}/.github/workflows/ci.yml +3 -3
- {openstatspec-0.4.0 → openstatspec-0.6.0}/.github/workflows/release.yml +1 -1
- {openstatspec-0.4.0 → openstatspec-0.6.0}/CHANGELOG.md +85 -0
- {openstatspec-0.4.0 → openstatspec-0.6.0}/PKG-INFO +18 -11
- {openstatspec-0.4.0 → openstatspec-0.6.0}/README.md +17 -10
- {openstatspec-0.4.0 → openstatspec-0.6.0}/docs/release-readiness.md +23 -6
- {openstatspec-0.4.0 → openstatspec-0.6.0}/docs/transformations.md +33 -19
- {openstatspec-0.4.0 → openstatspec-0.6.0}/pyproject.toml +5 -2
- openstatspec-0.6.0/src/openstatspec/__init__.py +48 -0
- {openstatspec-0.4.0 → openstatspec-0.6.0}/src/openstatspec/api.py +110 -18
- {openstatspec-0.4.0 → openstatspec-0.6.0}/src/openstatspec/cli.py +11 -3
- {openstatspec-0.4.0 → openstatspec-0.6.0}/src/openstatspec/core/__init__.py +14 -0
- {openstatspec-0.4.0 → openstatspec-0.6.0}/src/openstatspec/frontends/spss/__init__.py +7 -3
- openstatspec-0.6.0/src/openstatspec/frontends/spss/binding.py +543 -0
- {openstatspec-0.4.0 → openstatspec-0.6.0}/src/openstatspec/frontends/spss/compiler.py +14 -1
- {openstatspec-0.4.0 → openstatspec-0.6.0}/src/openstatspec/frontends/spss/execution.py +4 -2
- {openstatspec-0.4.0 → openstatspec-0.6.0}/src/openstatspec/frontends/spss/syntax.py +323 -5
- openstatspec-0.6.0/src/openstatspec/spss/__init__.py +36 -0
- openstatspec-0.6.0/src/openstatspec/spss/sav.py +1244 -0
- openstatspec-0.6.0/src/openstatspec/sql/__init__.py +66 -0
- openstatspec-0.6.0/src/openstatspec/sql/capabilities.py +624 -0
- openstatspec-0.6.0/src/openstatspec/sql/catalog_verification.py +283 -0
- openstatspec-0.6.0/src/openstatspec/sql/database_urls.py +22 -0
- openstatspec-0.6.0/src/openstatspec/sql/dolt_conformance.py +455 -0
- openstatspec-0.6.0/src/openstatspec/sql/inplace_transform.py +1093 -0
- {openstatspec-0.4.0 → openstatspec-0.6.0}/src/openstatspec/sql/profiles.py +155 -38
- openstatspec-0.6.0/src/openstatspec/sql/wide.py +1753 -0
- {openstatspec-0.4.0 → openstatspec-0.6.0}/src/openstatspec/sql/workflow.py +3 -2
- {openstatspec-0.4.0 → openstatspec-0.6.0}/src/openstatspec/transform/__init__.py +14 -2
- openstatspec-0.6.0/src/openstatspec/transform/plan.py +854 -0
- openstatspec-0.6.0/src/openstatspec/transform/schema.py +88 -0
- openstatspec-0.6.0/src/openstatspec/transform/validation.py +420 -0
- {openstatspec-0.4.0 → openstatspec-0.6.0}/tests/conformance.py +1 -1
- {openstatspec-0.4.0 → openstatspec-0.6.0}/tests/test_atomic_import.py +212 -82
- {openstatspec-0.4.0 → openstatspec-0.6.0}/tests/test_attribute_catalog.py +21 -52
- openstatspec-0.6.0/tests/test_catalog_lifecycle.py +208 -0
- openstatspec-0.6.0/tests/test_catalog_persistence_review.py +253 -0
- {openstatspec-0.4.0 → openstatspec-0.6.0}/tests/test_cli.py +9 -5
- openstatspec-0.6.0/tests/test_conditional_inplace_transform.py +374 -0
- {openstatspec-0.4.0 → openstatspec-0.6.0}/tests/test_document_round_trip.py +10 -6
- openstatspec-0.6.0/tests/test_dolt_conformance.py +382 -0
- openstatspec-0.6.0/tests/test_inplace_transform.py +1093 -0
- {openstatspec-0.4.0 → openstatspec-0.6.0}/tests/test_loss_reports.py +39 -36
- {openstatspec-0.4.0 → openstatspec-0.6.0}/tests/test_normative_catalog.py +2 -0
- {openstatspec-0.4.0 → openstatspec-0.6.0}/tests/test_official_conformance.py +9 -4
- {openstatspec-0.4.0 → openstatspec-0.6.0}/tests/test_pyspssio_catalog_authority.py +25 -15
- openstatspec-0.6.0/tests/test_review_transaction_boundary.py +209 -0
- {openstatspec-0.4.0 → openstatspec-0.6.0}/tests/test_sav_sqlite.py +237 -22
- {openstatspec-0.4.0 → openstatspec-0.6.0}/tests/test_spss_catalog_preflight.py +24 -16
- {openstatspec-0.4.0 → openstatspec-0.6.0}/tests/test_sql_profiles.py +341 -108
- {openstatspec-0.4.0 → openstatspec-0.6.0}/tests/test_sql_services.py +127 -154
- {openstatspec-0.4.0 → openstatspec-0.6.0}/tests/test_sql_workflow.py +4 -0
- openstatspec-0.6.0/tests/test_strict_catalog_review.py +188 -0
- {openstatspec-0.4.0 → openstatspec-0.6.0}/tests/test_transform_frontend.py +331 -0
- {openstatspec-0.4.0 → openstatspec-0.6.0}/tests/test_vls_compatible_names.py +10 -44
- openstatspec-0.4.0/src/openstatspec/__init__.py +0 -40
- openstatspec-0.4.0/src/openstatspec/frontends/spss/binding.py +0 -265
- openstatspec-0.4.0/src/openstatspec/spss/__init__.py +0 -22
- openstatspec-0.4.0/src/openstatspec/spss/sav.py +0 -575
- openstatspec-0.4.0/src/openstatspec/sql/__init__.py +0 -14
- openstatspec-0.4.0/src/openstatspec/sql/capabilities.py +0 -452
- openstatspec-0.4.0/src/openstatspec/sql/inplace_transform.py +0 -682
- openstatspec-0.4.0/src/openstatspec/sql/wide.py +0 -1241
- openstatspec-0.4.0/src/openstatspec/transform/plan.py +0 -415
- openstatspec-0.4.0/src/openstatspec/transform/schema.py +0 -46
- openstatspec-0.4.0/src/openstatspec/transform/validation.py +0 -178
- openstatspec-0.4.0/tests/test_inplace_transform.py +0 -508
- {openstatspec-0.4.0 → openstatspec-0.6.0}/.github/verify_release_ref.py +0 -0
- {openstatspec-0.4.0 → openstatspec-0.6.0}/.gitignore +0 -0
- {openstatspec-0.4.0 → openstatspec-0.6.0}/THIRD_PARTY_NOTICES.md +0 -0
- {openstatspec-0.4.0 → openstatspec-0.6.0}/docs/architecture.md +0 -0
- {openstatspec-0.4.0 → openstatspec-0.6.0}/docs/in-place-transformation.md +0 -0
- {openstatspec-0.4.0 → openstatspec-0.6.0}/docs/pyspssio-spike.md +0 -0
- {openstatspec-0.4.0 → openstatspec-0.6.0}/docs/sav-profile.md +0 -0
- {openstatspec-0.4.0 → openstatspec-0.6.0}/docs/sql-transformation-workflow.md +0 -0
- {openstatspec-0.4.0 → openstatspec-0.6.0}/src/openstatspec/core/results.py +0 -0
- {openstatspec-0.4.0 → openstatspec-0.6.0}/src/openstatspec/frontends/__init__.py +0 -0
- {openstatspec-0.4.0 → openstatspec-0.6.0}/src/openstatspec/frontends/sas/.gitkeep +0 -0
- {openstatspec-0.4.0 → openstatspec-0.6.0}/src/openstatspec/frontends/stata/.gitkeep +0 -0
- {openstatspec-0.4.0 → openstatspec-0.6.0}/src/openstatspec/spss/dictionary.py +0 -0
- {openstatspec-0.4.0 → openstatspec-0.6.0}/src/openstatspec/spss/raw_dictionary.py +0 -0
- {openstatspec-0.4.0 → openstatspec-0.6.0}/src/openstatspec/sql/catalog_api.py +0 -0
- {openstatspec-0.4.0 → openstatspec-0.6.0}/src/openstatspec/sql/normative.py +0 -0
- {openstatspec-0.4.0 → openstatspec-0.6.0}/src/openstatspec/transform/errors.py +0 -0
- {openstatspec-0.4.0 → openstatspec-0.6.0}/tests/test_release_ref.py +0 -0
|
@@ -20,7 +20,7 @@ jobs:
|
|
|
20
20
|
uses: actions/checkout@v7
|
|
21
21
|
with: &specification-checkout
|
|
22
22
|
repository: OpenStatSpec/specification
|
|
23
|
-
ref:
|
|
23
|
+
ref: f2fdf687d8cb32b944ca55a3e9e7215ffc603019
|
|
24
24
|
path: openstatspec-specification
|
|
25
25
|
- name: Checkout required SPSS engine
|
|
26
26
|
uses: actions/checkout@v7
|
|
@@ -143,7 +143,7 @@ jobs:
|
|
|
143
143
|
steps: *sql-test-steps
|
|
144
144
|
|
|
145
145
|
dolt-integration:
|
|
146
|
-
name: Dolt ${{ matrix.dolt }} integration
|
|
146
|
+
name: Dolt ${{ matrix.dolt }} read-only / fail-closed integration
|
|
147
147
|
runs-on: ubuntu-latest
|
|
148
148
|
strategy:
|
|
149
149
|
fail-fast: false
|
|
@@ -217,7 +217,7 @@ jobs:
|
|
|
217
217
|
connection.close()
|
|
218
218
|
break
|
|
219
219
|
PY
|
|
220
|
-
- run: python -m pytest -m services
|
|
220
|
+
- run: python -m pytest -m "services and not candidate_evidence"
|
|
221
221
|
- name: Show Dolt logs on failure and stop server
|
|
222
222
|
if: always()
|
|
223
223
|
run: |
|
|
@@ -47,7 +47,7 @@ jobs:
|
|
|
47
47
|
uses: actions/checkout@v7
|
|
48
48
|
with:
|
|
49
49
|
repository: OpenStatSpec/specification
|
|
50
|
-
ref:
|
|
50
|
+
ref: f2fdf687d8cb32b944ca55a3e9e7215ffc603019
|
|
51
51
|
path: openstatspec-specification
|
|
52
52
|
- name: Checkout required SPSS engine
|
|
53
53
|
uses: actions/checkout@v7
|
|
@@ -2,6 +2,72 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to this reference implementation are documented here.
|
|
4
4
|
|
|
5
|
+
## Unreleased
|
|
6
|
+
|
|
7
|
+
_No unreleased changes._
|
|
8
|
+
|
|
9
|
+
## 0.6.0 - 2026-08-05
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- Completed Python parity for bounded in-place SPSS schema transformations:
|
|
14
|
+
STRING, DELETE VARIABLES, string RECODE width validation, and durable
|
|
15
|
+
delete/recreate physical-column bindings.
|
|
16
|
+
- Added PostgreSQL physical attribute-slot preflight and real-service
|
|
17
|
+
regression coverage for delete/recreate at the column limit.
|
|
18
|
+
|
|
19
|
+
### Changed
|
|
20
|
+
|
|
21
|
+
- Schema-changing SPSS plans use the v0.3 contract and fail closed for
|
|
22
|
+
unsupported STRING/DELETE variable ranges.
|
|
23
|
+
|
|
24
|
+
## 0.5.0 - 2026-08-05
|
|
25
|
+
### Fixed
|
|
26
|
+
|
|
27
|
+
- Removed the temporary `*_catalog` compatibility schema and made SAV/ZSAV
|
|
28
|
+
import, validation, export, fidelity reporting, and in-place metadata edits
|
|
29
|
+
use the normative UUID-keyed OpenStatSpec catalog exclusively.
|
|
30
|
+
- Existing databases that contain former compatibility relations must be
|
|
31
|
+
remediated manually before further operations.
|
|
32
|
+
- Imports now reject dataset names that collide with normative UUID identifiers,
|
|
33
|
+
cleanup drops only a physical table actually created by the failing import,
|
|
34
|
+
and non-preserved SPSS compatible variable names are reported as an explicit
|
|
35
|
+
loss requiring export consent.
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
### Added
|
|
39
|
+
|
|
40
|
+
- Added bounded typed expressions with variable references, numeric literals,
|
|
41
|
+
parentheses, `=`, `<`, `<=`, `>`, `>=`, `AND`, and `OR`. String comparison
|
|
42
|
+
and v0.2 string assignment fail closed pending exact portable semantics.
|
|
43
|
+
- Added sequential SPSS-like `COMPUTE` and `IF` assignment operations plus
|
|
44
|
+
`FORMATS`, `VARIABLE LEVEL`, and `EXECUTE`.
|
|
45
|
+
- Added atomic numeric target creation on SQLite and PostgreSQL. MySQL,
|
|
46
|
+
MariaDB, and Dolt fail closed on `target_mode=create`; their targets must be
|
|
47
|
+
provisioned physically and in the catalog by a separate versioned stage
|
|
48
|
+
before this executor applies assignment and metadata operations.
|
|
49
|
+
- Added synthetic exact-program, catalog, failure-boundary, and pre-existing
|
|
50
|
+
target regression coverage.
|
|
51
|
+
|
|
52
|
+
### Changed
|
|
53
|
+
|
|
54
|
+
- Bumped the canonical transformation-plan and SPSS frontend contracts to
|
|
55
|
+
`v0.2`; canonical JSON and hashes include every sequential operation.
|
|
56
|
+
- In-place apply records variable label, value labels, `F` print/write
|
|
57
|
+
format, and measurement level only in the normative catalog.
|
|
58
|
+
- Dolt still requires an exact branch, exact HEAD, and clean working set;
|
|
59
|
+
successful apply leaves an inspectable diff and never calls `DOLT_COMMIT`.
|
|
60
|
+
- Dolt declaration validation is implemented by the Python adapter. The
|
|
61
|
+
specification remains a language-neutral source of JSON contracts and
|
|
62
|
+
fixtures and is not installed as a Python distribution.
|
|
63
|
+
|
|
64
|
+
### Specification basis
|
|
65
|
+
|
|
66
|
+
- Release validation is pinned to the untagged OpenStatSpec specification
|
|
67
|
+
release candidate at immutable commit
|
|
68
|
+
`f2fdf687d8cb32b944ca55a3e9e7215ffc603019`. Its
|
|
69
|
+
`specification_release` remains null until that commit receives a stable tag.
|
|
70
|
+
|
|
5
71
|
## 0.4.0 — 2026-07-31
|
|
6
72
|
|
|
7
73
|
### Added
|
|
@@ -109,6 +175,17 @@ SPSS profile.
|
|
|
109
175
|
- Export of supported dataset semantics to SAV and ZSAV.
|
|
110
176
|
- SQLite, PostgreSQL, MySQL, and MariaDB profiles, including service-backed
|
|
111
177
|
PostgreSQL 17/18, MySQL 8.4/9.7, and MariaDB 11.4/11.8/12.3 CI coverage.
|
|
178
|
+
- Read-only positive Dolt identity and working-set inspection. Operational
|
|
179
|
+
Dolt writes use adapter-owned declaration validation and remain fail-closed
|
|
180
|
+
because the adapter ships no concrete declaration claim. The specification
|
|
181
|
+
remains language-neutral and is consumed only as JSON contracts and fixtures.
|
|
182
|
+
- Added explicit packaged/directory conformance-source injection and exact
|
|
183
|
+
single-match binding across active Dolt product version, adapter
|
|
184
|
+
implementation and version, and specification commit before mutation.
|
|
185
|
+
- Explicit catalog initialization/migration through `initialize_catalog` and
|
|
186
|
+
`openstatspec init`; data operations never auto-create catalog relations.
|
|
187
|
+
- Read-only Dolt working-set evidence through `dolt_state_snapshot` and
|
|
188
|
+
`openstatspec dolt-state`; core performs no Dolt version-control mutations.
|
|
112
189
|
- Preflight checks for target profile limits, atomic imports, validation, a
|
|
113
190
|
command-line interface, and machine-readable capability and loss reports.
|
|
114
191
|
|
|
@@ -121,3 +198,11 @@ SPSS profile.
|
|
|
121
198
|
- Multiple-response sets, variable alignment, variable sets, and custom
|
|
122
199
|
attributes have explicit capability diagnostics; see the SAV profile.
|
|
123
200
|
- Encrypted SPSS files are not supported.
|
|
201
|
+
- Dolt adapter-envelope values are not claims about Dolt server limits. Failure
|
|
202
|
+
recovery is compensating and reports deterministic residual inventory; a
|
|
203
|
+
verified catalog receives a best-effort failed-operation audit when cleanup
|
|
204
|
+
itself fails.
|
|
205
|
+
- Export publication uses a durable, audit-recorded prior-file backup through
|
|
206
|
+
SQL finalization. Post-success backup retention is reported separately and
|
|
207
|
+
does not rewrite a successful export as failed; pre-success failures restore
|
|
208
|
+
the prior destination and close the running operation as failed.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: openstatspec
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Summary: Reference adapter for the OpenStatSpec relational contract
|
|
5
5
|
Author: OpenStatSpec
|
|
6
6
|
License: Apache-2.0
|
|
@@ -30,10 +30,10 @@ For each supported import, one source dataset becomes one dedicated wide SQL
|
|
|
30
30
|
table. Cases are rows and source variables are physical SQL columns. The
|
|
31
31
|
singular UUID-keyed tables from the specification (`dataset`, `variable`,
|
|
32
32
|
`operation`, `fidelity_event`, and related metadata tables) are the public
|
|
33
|
-
catalog contract
|
|
34
|
-
|
|
35
|
-
The adapter does not
|
|
36
|
-
or long-form tables, or harmonize studies or waves.
|
|
33
|
+
catalog contract and the only catalog representation created or read by the
|
|
34
|
+
adapter. Databases created with the former `*_catalog` compatibility schema
|
|
35
|
+
must be remediated to the normative schema before use. The adapter does not
|
|
36
|
+
reshape data, create EAV or long-form tables, or harmonize studies or waves.
|
|
37
37
|
|
|
38
38
|
Unsupported source features, SQL targets, or export paths fail explicitly.
|
|
39
39
|
There is no silent truncation, type conversion, metadata loss, or partial
|
|
@@ -86,12 +86,19 @@ implemented capability boundary.
|
|
|
86
86
|
|
|
87
87
|
## SPSS-like transformation frontend
|
|
88
88
|
|
|
89
|
-
The SPSS-like frontend lowers
|
|
90
|
-
`
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
89
|
+
The bounded SPSS-like frontend lowers `RECODE`, sequential `COMPUTE` and `IF`,
|
|
90
|
+
`VARIABLE LABELS`, `VALUE LABELS`, numeric `FORMATS`, `VARIABLE LEVEL`, and
|
|
91
|
+
`EXECUTE` into a language-neutral typed canonical plan. Conditions support
|
|
92
|
+
parentheses, variable and numeric literal operands, comparisons, `AND`, and
|
|
93
|
+
`OR`. String comparison and v0.2 string assignment currently fail closed.
|
|
94
|
+
The in-place path applies the plan to the same logical dataset, physical wide
|
|
95
|
+
table, and normative/compatibility metadata without a derived dataset, copied
|
|
96
|
+
table, snapshot, or hidden history layer. Numeric targets may be created
|
|
97
|
+
atomically on SQLite and PostgreSQL. MySQL, MariaDB, and Dolt fail closed on
|
|
98
|
+
`target_mode=create` and require a separately provisioned physical and catalog
|
|
99
|
+
target before assignment. Dolt requires the caller's exact clean branch/HEAD,
|
|
100
|
+
leaves success as an inspectable working-set diff, and never calls
|
|
101
|
+
`DOLT_COMMIT`.
|
|
95
102
|
|
|
96
103
|
See the [dataset transformations manual](docs/transformations.md) for schema
|
|
97
104
|
installation, Python and CLI surfaces, database invariants, audit provenance,
|
|
@@ -11,10 +11,10 @@ For each supported import, one source dataset becomes one dedicated wide SQL
|
|
|
11
11
|
table. Cases are rows and source variables are physical SQL columns. The
|
|
12
12
|
singular UUID-keyed tables from the specification (`dataset`, `variable`,
|
|
13
13
|
`operation`, `fidelity_event`, and related metadata tables) are the public
|
|
14
|
-
catalog contract
|
|
15
|
-
|
|
16
|
-
The adapter does not
|
|
17
|
-
or long-form tables, or harmonize studies or waves.
|
|
14
|
+
catalog contract and the only catalog representation created or read by the
|
|
15
|
+
adapter. Databases created with the former `*_catalog` compatibility schema
|
|
16
|
+
must be remediated to the normative schema before use. The adapter does not
|
|
17
|
+
reshape data, create EAV or long-form tables, or harmonize studies or waves.
|
|
18
18
|
|
|
19
19
|
Unsupported source features, SQL targets, or export paths fail explicitly.
|
|
20
20
|
There is no silent truncation, type conversion, metadata loss, or partial
|
|
@@ -67,12 +67,19 @@ implemented capability boundary.
|
|
|
67
67
|
|
|
68
68
|
## SPSS-like transformation frontend
|
|
69
69
|
|
|
70
|
-
The SPSS-like frontend lowers
|
|
71
|
-
`
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
70
|
+
The bounded SPSS-like frontend lowers `RECODE`, sequential `COMPUTE` and `IF`,
|
|
71
|
+
`VARIABLE LABELS`, `VALUE LABELS`, numeric `FORMATS`, `VARIABLE LEVEL`, and
|
|
72
|
+
`EXECUTE` into a language-neutral typed canonical plan. Conditions support
|
|
73
|
+
parentheses, variable and numeric literal operands, comparisons, `AND`, and
|
|
74
|
+
`OR`. String comparison and v0.2 string assignment currently fail closed.
|
|
75
|
+
The in-place path applies the plan to the same logical dataset, physical wide
|
|
76
|
+
table, and normative/compatibility metadata without a derived dataset, copied
|
|
77
|
+
table, snapshot, or hidden history layer. Numeric targets may be created
|
|
78
|
+
atomically on SQLite and PostgreSQL. MySQL, MariaDB, and Dolt fail closed on
|
|
79
|
+
`target_mode=create` and require a separately provisioned physical and catalog
|
|
80
|
+
target before assignment. Dolt requires the caller's exact clean branch/HEAD,
|
|
81
|
+
leaves success as an inspectable working-set diff, and never calls
|
|
82
|
+
`DOLT_COMMIT`.
|
|
76
83
|
|
|
77
84
|
See the [dataset transformations manual](docs/transformations.md) for schema
|
|
78
85
|
installation, Python and CLI surfaces, database invariants, audit provenance,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# 0.
|
|
1
|
+
# 0.5.0 release readiness
|
|
2
2
|
|
|
3
3
|
This page records the expected release contract, not a publication event.
|
|
4
4
|
Creating a version tag remains a separate maintainer action.
|
|
@@ -67,6 +67,17 @@ The gate must prove that:
|
|
|
67
67
|
|
|
68
68
|
- a TransformationPlan object and its strict JSON mapping produce the same
|
|
69
69
|
plan hash and in-place result;
|
|
70
|
+
- the exact bounded `COMPUTE`/`IF` program compiles to all seven ordered
|
|
71
|
+
operations without dropping `FORMATS`, `VARIABLE LEVEL`, or `EXECUTE`;
|
|
72
|
+
- boolean data results match the equivalent expression and the target's label,
|
|
73
|
+
0/1 value labels, `F1.0` print/write format, and nominal level exist in the
|
|
74
|
+
normative catalog;
|
|
75
|
+
- injected schema, data, catalog, and audit failures leave no partial apply;
|
|
76
|
+
- compensation tracks only newly created targets and never drops or rewrites a
|
|
77
|
+
pre-existing target;
|
|
78
|
+
- MySQL, MariaDB, and Dolt reject create-target plans before mutation; their
|
|
79
|
+
service evidence covers assignment to a separately provisioned physical and
|
|
80
|
+
catalog target without schema DDL;
|
|
70
81
|
- top-level SPSS compiler imports and legacy openstatspec.transform re-exports
|
|
71
82
|
still load from an installed wheel;
|
|
72
83
|
- install-in-place-schema, apply-plan, and apply-spss execute their documented
|
|
@@ -78,8 +89,11 @@ The gate must prove that:
|
|
|
78
89
|
source/plan hashes and frontend contract, and contain no copied data;
|
|
79
90
|
- no OpenStatSpec rollback, snapshot, staging, copy, derived-dataset, or
|
|
80
91
|
parallel history artifacts are created; and
|
|
81
|
-
- Dolt checks expected branch, HEAD, and a clean working set
|
|
82
|
-
|
|
92
|
+
- Dolt checks expected branch, HEAD, and a clean working set; success changes
|
|
93
|
+
neither HEAD nor branch and never commits or resets; state is rechecked after
|
|
94
|
+
the dataset lock and success must leave an inspectable working-set diff;
|
|
95
|
+
other supported SQL connections remain allowed; and
|
|
96
|
+
- string comparisons and v0.2 string assignments fail closed until exact
|
|
83
97
|
|
|
84
98
|
The built wheel must contain the generic openstatspec.transform modules and the
|
|
85
99
|
implemented openstatspec.frontends.spss package. Stata and SAS remain empty
|
|
@@ -98,9 +112,12 @@ capability claim, or implied support.
|
|
|
98
112
|
4. Build with `python -m build` and install the generated wheel in a clean
|
|
99
113
|
environment.
|
|
100
114
|
5. Confirm `openstatspec capabilities` reflects the intended support boundary.
|
|
101
|
-
6. Confirm
|
|
102
|
-
|
|
103
|
-
|
|
115
|
+
6. Confirm CI, release fixtures, and capabilities use the untagged OpenStatSpec
|
|
116
|
+
specification release candidate at exact commit
|
|
117
|
+
`f2fdf687d8cb32b944ca55a3e9e7215ffc603019`, publish
|
|
118
|
+
`specification_status=release_candidate`, and leave
|
|
119
|
+
`specification_release` null. If that exact commit receives a stable tag
|
|
120
|
+
before this package is tagged, update the identity and re-run every gate.
|
|
104
121
|
7. Review this document, the README, and CHANGELOG for accurate scope.
|
|
105
122
|
|
|
106
123
|
The tag-triggered release workflow repeats the non-service test suite, builds
|
|
@@ -4,8 +4,13 @@ OpenStatSpec separates transformation syntax, canonical meaning, and database
|
|
|
4
4
|
mutation. This lets multiple language frontends produce the same plan without
|
|
5
5
|
coupling the executor to any one language.
|
|
6
6
|
|
|
7
|
-
The implemented
|
|
8
|
-
`VARIABLE LABELS`,
|
|
7
|
+
The implemented bounded SPSS-like frontend accepts `RECODE`, sequential
|
|
8
|
+
`COMPUTE` and `IF`, `VARIABLE LABELS`, `VALUE LABELS`, numeric `FORMATS`,
|
|
9
|
+
`VARIABLE LEVEL`, and `EXECUTE`. Predicates support typed variable/literal
|
|
10
|
+
operands, parentheses, numeric comparisons, `AND`, and `OR`. String comparison
|
|
11
|
+
and v0.2 string assignment fail closed until exact profile-independent
|
|
12
|
+
collation and explicit-width semantics are available; arbitrary SPSS, Python,
|
|
13
|
+
and SQL expressions are rejected. Stata and SAS are not implemented.
|
|
9
14
|
|
|
10
15
|
## Architecture
|
|
11
16
|
|
|
@@ -106,9 +111,13 @@ result = openstatspec.apply_spss_in_place(
|
|
|
106
111
|
dataset_id="responses",
|
|
107
112
|
actor="agent@example.org",
|
|
108
113
|
source_text="""
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
114
|
+
COMPUTE target = 0.
|
|
115
|
+
IF (source_a = 1 AND source_b = 1) target = 1.
|
|
116
|
+
VARIABLE LABELS target 'Example label'.
|
|
117
|
+
VALUE LABELS target 0 'No' 1 'Yes'.
|
|
118
|
+
FORMATS target (F1.0).
|
|
119
|
+
VARIABLE LEVEL target (NOMINAL).
|
|
120
|
+
EXECUTE.
|
|
112
121
|
""",
|
|
113
122
|
)
|
|
114
123
|
```
|
|
@@ -130,20 +139,25 @@ On Dolt, also pass `--expected-branch` and `--expected-head`.
|
|
|
130
139
|
Every successful apply preserves the logical `dataset_id` and physical
|
|
131
140
|
schema/table identity. It creates no derived dataset, output table, full-table
|
|
132
141
|
copy, staging table, snapshot, rollback artifact, or recovery/history layer.
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
142
|
+
Assignments and recodes use ordered `UPDATE` statements; later operations see
|
|
143
|
+
earlier results. Label, value-label, format, and measurement-level operations
|
|
144
|
+
update the normative catalog.
|
|
145
|
+
|
|
146
|
+
A numeric create target is supported atomically on SQLite and PostgreSQL.
|
|
147
|
+
MySQL, MariaDB, and Dolt reject `target_mode=create` before mutation. On those
|
|
148
|
+
profiles a separate versioned stage must first provision the nullable numeric
|
|
149
|
+
physical column and normative variable row; the transformation executor then
|
|
150
|
+
sees a pre-existing target and performs no schema DDL.
|
|
151
|
+
The public operation reports success only after physical data, normative
|
|
152
|
+
metadata, and the compact audit row are mutually complete.
|
|
153
|
+
|
|
154
|
+
Before Dolt mutation, the executor verifies the expected branch and `HEAD` and
|
|
155
|
+
requires clean `dolt_status`. Success changes the same working set without
|
|
156
|
+
changing `HEAD`; OpenStatSpec does not call `DOLT_COMMIT`, `DOLT_RESET`, switch
|
|
157
|
+
branches, merge, tag, or create a hidden recovery commit. It rechecks branch,
|
|
158
|
+
HEAD, and a clean working set after locking the dataset and immediately before
|
|
159
|
+
mutation. The caller reviews a successful `dolt diff` and
|
|
160
|
+
separately decides whether to commit or restore.
|
|
147
161
|
|
|
148
162
|
## Audit and provenance
|
|
149
163
|
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "openstatspec"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.6.0"
|
|
8
8
|
description = "Reference adapter for the OpenStatSpec relational contract"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
@@ -22,7 +22,10 @@ openstatspec = "openstatspec.cli:main"
|
|
|
22
22
|
[tool.pytest.ini_options]
|
|
23
23
|
testpaths = ["tests"]
|
|
24
24
|
addopts = "-q"
|
|
25
|
-
markers = [
|
|
25
|
+
markers = [
|
|
26
|
+
"services: requires a configured PostgreSQL, MySQL, MariaDB, or Dolt service",
|
|
27
|
+
"candidate_evidence: non-claiming manual probe excluded from release gates",
|
|
28
|
+
]
|
|
26
29
|
|
|
27
30
|
[tool.hatch.build.targets.wheel]
|
|
28
31
|
packages = ["src/openstatspec"]
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Public API for the OpenStatSpec Python reference implementation."""
|
|
2
|
+
|
|
3
|
+
from .api import (
|
|
4
|
+
apply_spss_in_place, apply_transformation_plan_in_place,
|
|
5
|
+
capabilities, capability_matrix, derive_sql_dataset, dolt_state_snapshot, execute_sql_transformation,
|
|
6
|
+
export_sav, get_dataset, import_sav, initialize_catalog, inspect, list_datasets,
|
|
7
|
+
install_in_place_transformation_schema,
|
|
8
|
+
register_sql_transformation, reconcile_derived_removals,
|
|
9
|
+
reconcile_sql_transformation_runs,
|
|
10
|
+
remove_derived_physical_relation, retire_derived, validate, validate_derived,
|
|
11
|
+
)
|
|
12
|
+
from .core import CapabilityDeclaration, LossReport, UnsupportedOperationError
|
|
13
|
+
from .frontends.spss import SpssFrontendCompilation, compile_spss_syntax
|
|
14
|
+
from .sql import DoltConformanceSource
|
|
15
|
+
from .sql.workflow import TransformationError
|
|
16
|
+
from .transform import (
|
|
17
|
+
AssignOperation, BooleanExpression, ComparisonExpression,
|
|
18
|
+
ConditionalAssignOperation, CreateVariableOperation, DeleteVariableOperation,
|
|
19
|
+
ExecuteOperation, Operand, PredicateExpression,
|
|
20
|
+
RecodeMatch, RecodeOperation, RecodeResult, RecodeRule,
|
|
21
|
+
ReplaceValueLabelsOperation, SetFormatOperation,
|
|
22
|
+
SetMeasurementLevelOperation, SetVariableLabelOperation,
|
|
23
|
+
TransformationFrontendError, TransformationPlan, TypedValue, ValueLabel,
|
|
24
|
+
VariableDefinition, VariableSchema, transformation_plan_from_dict,
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
__all__ = [
|
|
28
|
+
"AssignOperation", "BooleanExpression", "ComparisonExpression",
|
|
29
|
+
"ConditionalAssignOperation", "ExecuteOperation", "Operand",
|
|
30
|
+
"PredicateExpression", "CreateVariableOperation", "DeleteVariableOperation",
|
|
31
|
+
"CapabilityDeclaration", "DoltConformanceSource", "LossReport",
|
|
32
|
+
"SpssFrontendCompilation", "TransformationError",
|
|
33
|
+
"TransformationFrontendError",
|
|
34
|
+
"RecodeMatch", "RecodeOperation", "RecodeResult", "RecodeRule",
|
|
35
|
+
"ReplaceValueLabelsOperation", "SetFormatOperation",
|
|
36
|
+
"SetMeasurementLevelOperation", "SetVariableLabelOperation",
|
|
37
|
+
"TransformationPlan", "TypedValue", "ValueLabel",
|
|
38
|
+
"VariableDefinition", "VariableSchema", "transformation_plan_from_dict",
|
|
39
|
+
"apply_spss_in_place", "apply_transformation_plan_in_place",
|
|
40
|
+
"compile_spss_syntax", "install_in_place_transformation_schema",
|
|
41
|
+
"UnsupportedOperationError", "capabilities", "capability_matrix",
|
|
42
|
+
"derive_sql_dataset", "dolt_state_snapshot", "execute_sql_transformation", "export_sav", "get_dataset",
|
|
43
|
+
"import_sav", "initialize_catalog", "inspect", "list_datasets",
|
|
44
|
+
"register_sql_transformation", "reconcile_derived_removals",
|
|
45
|
+
"reconcile_sql_transformation_runs",
|
|
46
|
+
"remove_derived_physical_relation", "retire_derived", "validate",
|
|
47
|
+
"validate_derived",
|
|
48
|
+
]
|
|
@@ -7,7 +7,13 @@ from typing import Any
|
|
|
7
7
|
from .core.results import result
|
|
8
8
|
from .spss import export_dataset, import_dataset, inspect_source
|
|
9
9
|
from .spss.sav import engine_identity
|
|
10
|
-
from .sql import
|
|
10
|
+
from .sql import (
|
|
11
|
+
DoltConformanceSource,
|
|
12
|
+
declared_profiles,
|
|
13
|
+
dolt_state_snapshot as _dolt_state_snapshot,
|
|
14
|
+
initialize_catalog as _initialize_catalog,
|
|
15
|
+
validate_dataset,
|
|
16
|
+
)
|
|
11
17
|
from .sql.catalog_api import catalog_dataset as _catalog_dataset, catalog_datasets as _catalog_datasets
|
|
12
18
|
from .sql.workflow import (
|
|
13
19
|
derive_dataset as _derive_dataset,
|
|
@@ -28,12 +34,17 @@ from .sql.inplace_transform import (
|
|
|
28
34
|
)
|
|
29
35
|
from .transform import TransformationPlan
|
|
30
36
|
from .sql.capabilities import (
|
|
31
|
-
SPECIFICATION_COMMIT, SPECIFICATION_RELEASE,
|
|
37
|
+
SPECIFICATION_COMMIT, SPECIFICATION_RELEASE, SPECIFICATION_STATUS,
|
|
38
|
+
active_connection, catalog_binding,
|
|
32
39
|
)
|
|
33
40
|
|
|
34
41
|
|
|
35
42
|
|
|
36
|
-
def capability_matrix(
|
|
43
|
+
def capability_matrix(
|
|
44
|
+
database_url: str | None = None,
|
|
45
|
+
*,
|
|
46
|
+
dolt_conformance_source: DoltConformanceSource | None = None,
|
|
47
|
+
) -> Mapping[str, Any]:
|
|
37
48
|
"""Return the pyspssio-backed SAV/ZSAV fidelity boundary.
|
|
38
49
|
|
|
39
50
|
supported means that the adapter has a tested faithful path.
|
|
@@ -44,7 +55,7 @@ def capability_matrix(database_url: str | None = None) -> Mapping[str, Any]:
|
|
|
44
55
|
"""
|
|
45
56
|
declaration = {
|
|
46
57
|
"specification": "OpenStatSpec",
|
|
47
|
-
"specification_status":
|
|
58
|
+
"specification_status": SPECIFICATION_STATUS,
|
|
48
59
|
"specification_release": SPECIFICATION_RELEASE,
|
|
49
60
|
"specification_commit": SPECIFICATION_COMMIT,
|
|
50
61
|
"profile": "SPSS SAV/ZSAV 1.0",
|
|
@@ -73,7 +84,7 @@ def capability_matrix(database_url: str | None = None) -> Mapping[str, Any]:
|
|
|
73
84
|
"multiple_response_sets": "supported",
|
|
74
85
|
"variable_alignment": "supported",
|
|
75
86
|
"variable_sets": "supported",
|
|
76
|
-
"compatible_variable_names": "
|
|
87
|
+
"compatible_variable_names": "requires-explicit-loss-consent",
|
|
77
88
|
"custom_attributes": {
|
|
78
89
|
"scalar_values": "supported",
|
|
79
90
|
"ordered_value_arrays": "supported",
|
|
@@ -86,9 +97,18 @@ def capability_matrix(database_url: str | None = None) -> Mapping[str, Any]:
|
|
|
86
97
|
"maximum_source_file_bytes": None,
|
|
87
98
|
"limit_basis": "runtime memory and active SQL connection limits",
|
|
88
99
|
},
|
|
89
|
-
"active_connection":
|
|
100
|
+
"active_connection": (
|
|
101
|
+
active_connection(
|
|
102
|
+
database_url,
|
|
103
|
+
dolt_conformance_source=dolt_conformance_source,
|
|
104
|
+
)
|
|
105
|
+
if database_url else None
|
|
106
|
+
),
|
|
90
107
|
"catalog_binding": catalog_binding(database_url) if database_url else None,
|
|
91
|
-
"sql_profiles": declared_profiles(
|
|
108
|
+
"sql_profiles": declared_profiles(
|
|
109
|
+
database_url,
|
|
110
|
+
dolt_conformance_source=dolt_conformance_source,
|
|
111
|
+
),
|
|
92
112
|
"optional_profiles": {
|
|
93
113
|
"sql_transformation_workflow": transformation_capabilities(database_url),
|
|
94
114
|
"spss_in_place_transformation": (
|
|
@@ -98,26 +118,84 @@ def capability_matrix(database_url: str | None = None) -> Mapping[str, Any]:
|
|
|
98
118
|
}
|
|
99
119
|
return declaration
|
|
100
120
|
|
|
101
|
-
def capabilities(
|
|
102
|
-
|
|
121
|
+
def capabilities(
|
|
122
|
+
database_url: str | None = None,
|
|
123
|
+
*,
|
|
124
|
+
dolt_conformance_source: DoltConformanceSource | None = None,
|
|
125
|
+
) -> Mapping[str, Any]:
|
|
126
|
+
return capability_matrix(
|
|
127
|
+
database_url, dolt_conformance_source=dolt_conformance_source,
|
|
128
|
+
)
|
|
103
129
|
|
|
104
130
|
|
|
105
131
|
def inspect(source: str | Path, /, **options: Any) -> Mapping[str, Any]:
|
|
106
132
|
return result(inspect_source(source, **options))
|
|
107
133
|
|
|
108
134
|
|
|
109
|
-
def
|
|
135
|
+
def dolt_state_snapshot(
|
|
136
|
+
*,
|
|
137
|
+
database_url: Any,
|
|
138
|
+
dolt_conformance_source: DoltConformanceSource | None = None,
|
|
139
|
+
) -> Mapping[str, Any]:
|
|
140
|
+
"""Return read-only Dolt branch, HEAD, status, and diff-summary evidence."""
|
|
141
|
+
return result(_dolt_state_snapshot(
|
|
142
|
+
database_url=database_url,
|
|
143
|
+
dolt_conformance_source=dolt_conformance_source,
|
|
144
|
+
))
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def initialize_catalog(
|
|
148
|
+
*,
|
|
149
|
+
database_url: Any,
|
|
150
|
+
dolt_conformance_source: DoltConformanceSource | None = None,
|
|
151
|
+
) -> Mapping[str, Any]:
|
|
152
|
+
"""Install or explicitly migrate a dedicated OpenStatSpec catalog."""
|
|
153
|
+
return result(_initialize_catalog(
|
|
154
|
+
database_url=database_url,
|
|
155
|
+
dolt_conformance_source=dolt_conformance_source,
|
|
156
|
+
))
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def import_sav(
|
|
160
|
+
source: str | Path, /, *, database_url: Any, dataset_id: str,
|
|
161
|
+
dolt_conformance_source: DoltConformanceSource | None = None,
|
|
162
|
+
**options: Any,
|
|
163
|
+
) -> Mapping[str, Any]:
|
|
110
164
|
"""Import one source file into one dedicated wide SQL table."""
|
|
111
|
-
return result(import_dataset(
|
|
165
|
+
return result(import_dataset(
|
|
166
|
+
source, database_url=database_url, dataset_id=dataset_id,
|
|
167
|
+
dolt_conformance_source=dolt_conformance_source,
|
|
168
|
+
**options,
|
|
169
|
+
))
|
|
112
170
|
|
|
113
171
|
|
|
114
|
-
def export_sav(
|
|
172
|
+
def export_sav(
|
|
173
|
+
*, database_url: Any, dataset_id: str, destination: str | Path,
|
|
174
|
+
dolt_conformance_source: DoltConformanceSource | None = None,
|
|
175
|
+
**options: Any,
|
|
176
|
+
) -> Mapping[str, Any]:
|
|
115
177
|
"""Export one database-resident conforming dataset to SAV/ZSAV."""
|
|
116
|
-
return result(export_dataset(
|
|
178
|
+
return result(export_dataset(
|
|
179
|
+
database_url=database_url, dataset_id=dataset_id,
|
|
180
|
+
destination=destination,
|
|
181
|
+
dolt_conformance_source=dolt_conformance_source,
|
|
182
|
+
**options,
|
|
183
|
+
))
|
|
117
184
|
|
|
118
185
|
|
|
119
|
-
def validate(
|
|
120
|
-
|
|
186
|
+
def validate(
|
|
187
|
+
*,
|
|
188
|
+
database_url: Any,
|
|
189
|
+
dataset_id: str,
|
|
190
|
+
dolt_conformance_source: DoltConformanceSource | None = None,
|
|
191
|
+
**options: Any,
|
|
192
|
+
) -> Mapping[str, Any]:
|
|
193
|
+
return result(validate_dataset(
|
|
194
|
+
database_url=database_url,
|
|
195
|
+
dataset_id=dataset_id,
|
|
196
|
+
dolt_conformance_source=dolt_conformance_source,
|
|
197
|
+
**options,
|
|
198
|
+
))
|
|
121
199
|
|
|
122
200
|
|
|
123
201
|
def list_datasets(*, database_url: Any, kind: str | None = None) -> Mapping[str, Any]:
|
|
@@ -144,8 +222,12 @@ def apply_spss_in_place(
|
|
|
144
222
|
*, database_url: Any, dataset_id: str, source_text: str,
|
|
145
223
|
actor: str, expected_branch: str | None = None,
|
|
146
224
|
expected_head: str | None = None,
|
|
225
|
+
dolt_conformance_source: DoltConformanceSource | None = None,
|
|
147
226
|
) -> Mapping[str, Any]:
|
|
148
|
-
"""Apply
|
|
227
|
+
"""Apply bounded sequential SPSS syntax to one SQL dataset/table.
|
|
228
|
+
|
|
229
|
+
Supports typed COMPUTE/IF predicates and dictionary metadata operations.
|
|
230
|
+
"""
|
|
149
231
|
return result(_apply_spss_in_place(
|
|
150
232
|
database_url=str(database_url),
|
|
151
233
|
dataset_id=dataset_id,
|
|
@@ -153,6 +235,7 @@ def apply_spss_in_place(
|
|
|
153
235
|
actor=actor,
|
|
154
236
|
expected_branch=expected_branch,
|
|
155
237
|
expected_head=expected_head,
|
|
238
|
+
dolt_conformance_source=dolt_conformance_source,
|
|
156
239
|
))
|
|
157
240
|
|
|
158
241
|
|
|
@@ -161,6 +244,7 @@ def apply_transformation_plan_in_place(
|
|
|
161
244
|
plan: TransformationPlan | Mapping[str, Any],
|
|
162
245
|
actor: str, expected_branch: str | None = None,
|
|
163
246
|
expected_head: str | None = None,
|
|
247
|
+
dolt_conformance_source: DoltConformanceSource | None = None,
|
|
164
248
|
) -> Mapping[str, Any]:
|
|
165
249
|
"""Apply a canonical plan to the same logical dataset and physical table."""
|
|
166
250
|
return result(_apply_transformation_plan_in_place(
|
|
@@ -170,12 +254,20 @@ def apply_transformation_plan_in_place(
|
|
|
170
254
|
actor=actor,
|
|
171
255
|
expected_branch=expected_branch,
|
|
172
256
|
expected_head=expected_head,
|
|
257
|
+
dolt_conformance_source=dolt_conformance_source,
|
|
173
258
|
))
|
|
174
259
|
|
|
175
260
|
|
|
176
|
-
def install_in_place_transformation_schema(
|
|
261
|
+
def install_in_place_transformation_schema(
|
|
262
|
+
*,
|
|
263
|
+
database_url: Any,
|
|
264
|
+
dolt_conformance_source: DoltConformanceSource | None = None,
|
|
265
|
+
) -> None:
|
|
177
266
|
"""Install the compact apply-audit relation before the first apply."""
|
|
178
|
-
_install_in_place_schema(
|
|
267
|
+
_install_in_place_schema(
|
|
268
|
+
database_url=str(database_url),
|
|
269
|
+
dolt_conformance_source=dolt_conformance_source,
|
|
270
|
+
)
|
|
179
271
|
|
|
180
272
|
|
|
181
273
|
def validate_derived(*, database_url: Any, derived_dataset_id: str) -> Mapping[str, Any]:
|