openstatspec 0.3.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {openstatspec-0.3.0 → openstatspec-0.4.0}/CHANGELOG.md +31 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/PKG-INFO +16 -38
- {openstatspec-0.3.0 → openstatspec-0.4.0}/README.md +15 -37
- openstatspec-0.4.0/docs/in-place-transformation.md +9 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/docs/release-readiness.md +31 -1
- openstatspec-0.4.0/docs/transformations.md +204 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/pyproject.toml +1 -1
- {openstatspec-0.3.0 → openstatspec-0.4.0}/src/openstatspec/__init__.py +13 -5
- {openstatspec-0.3.0 → openstatspec-0.4.0}/src/openstatspec/api.py +20 -1
- {openstatspec-0.3.0 → openstatspec-0.4.0}/src/openstatspec/cli.py +35 -3
- openstatspec-0.4.0/src/openstatspec/frontends/__init__.py +1 -0
- openstatspec-0.4.0/src/openstatspec/frontends/sas/.gitkeep +0 -0
- openstatspec-0.4.0/src/openstatspec/frontends/spss/__init__.py +26 -0
- {openstatspec-0.3.0/src/openstatspec/transform → openstatspec-0.4.0/src/openstatspec/frontends/spss}/binding.py +11 -44
- {openstatspec-0.3.0/src/openstatspec/transform → openstatspec-0.4.0/src/openstatspec/frontends/spss}/compiler.py +3 -2
- openstatspec-0.4.0/src/openstatspec/frontends/spss/execution.py +48 -0
- {openstatspec-0.3.0/src/openstatspec/transform → openstatspec-0.4.0/src/openstatspec/frontends/spss}/syntax.py +1 -1
- openstatspec-0.4.0/src/openstatspec/frontends/stata/.gitkeep +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/src/openstatspec/sql/inplace_transform.py +171 -45
- {openstatspec-0.3.0 → openstatspec-0.4.0}/src/openstatspec/transform/__init__.py +30 -11
- {openstatspec-0.3.0 → openstatspec-0.4.0}/src/openstatspec/transform/plan.py +0 -1
- openstatspec-0.4.0/src/openstatspec/transform/schema.py +46 -0
- openstatspec-0.4.0/src/openstatspec/transform/validation.py +178 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/tests/test_cli.py +63 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/tests/test_inplace_transform.py +207 -8
- {openstatspec-0.3.0 → openstatspec-0.4.0}/tests/test_transform_frontend.py +42 -5
- openstatspec-0.3.0/docs/in-place-transformation.md +0 -41
- {openstatspec-0.3.0 → openstatspec-0.4.0}/.github/verify_release_ref.py +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/.github/workflows/ci.yml +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/.github/workflows/release.yml +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/.gitignore +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/THIRD_PARTY_NOTICES.md +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/docs/architecture.md +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/docs/pyspssio-spike.md +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/docs/sav-profile.md +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/docs/sql-transformation-workflow.md +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/src/openstatspec/core/__init__.py +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/src/openstatspec/core/results.py +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/src/openstatspec/spss/__init__.py +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/src/openstatspec/spss/dictionary.py +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/src/openstatspec/spss/raw_dictionary.py +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/src/openstatspec/spss/sav.py +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/src/openstatspec/sql/__init__.py +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/src/openstatspec/sql/capabilities.py +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/src/openstatspec/sql/catalog_api.py +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/src/openstatspec/sql/normative.py +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/src/openstatspec/sql/profiles.py +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/src/openstatspec/sql/wide.py +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/src/openstatspec/sql/workflow.py +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/src/openstatspec/transform/errors.py +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/tests/conformance.py +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/tests/test_atomic_import.py +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/tests/test_attribute_catalog.py +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/tests/test_document_round_trip.py +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/tests/test_loss_reports.py +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/tests/test_normative_catalog.py +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/tests/test_official_conformance.py +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/tests/test_pyspssio_catalog_authority.py +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/tests/test_release_ref.py +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/tests/test_sav_sqlite.py +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/tests/test_spss_catalog_preflight.py +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/tests/test_sql_profiles.py +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/tests/test_sql_services.py +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/tests/test_sql_workflow.py +0 -0
- {openstatspec-0.3.0 → openstatspec-0.4.0}/tests/test_vls_compatible_names.py +0 -0
|
@@ -2,6 +2,37 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to this reference implementation are documented here.
|
|
4
4
|
|
|
5
|
+
## 0.4.0 — 2026-07-31
|
|
6
|
+
|
|
7
|
+
### Added
|
|
8
|
+
|
|
9
|
+
- Added a public canonical transformation-plan in-place API accepting either
|
|
10
|
+
typed plan objects or strictly validated mappings.
|
|
11
|
+
- Added apply-plan and install-in-place-schema CLI workflows alongside the
|
|
12
|
+
compatible apply-spss command.
|
|
13
|
+
- Added explicit audit provenance distinguishing canonical plan documents from
|
|
14
|
+
SPSS syntax sources.
|
|
15
|
+
- Added a transformation manual covering the canonical core, frontend boundary,
|
|
16
|
+
execution invariants, CLI/API workflows, and extension rules.
|
|
17
|
+
|
|
18
|
+
### Changed
|
|
19
|
+
|
|
20
|
+
- Moved the SPSS parser, binder, compiler, and convenience execution adapter to
|
|
21
|
+
the dedicated openstatspec.frontends.spss package.
|
|
22
|
+
- Kept openstatspec.transform language-neutral with canonical plan, schema, and
|
|
23
|
+
live-schema validation modules while preserving previous SPSS import paths.
|
|
24
|
+
- Reserved empty Stata and SAS frontend directories without exposing parsers,
|
|
25
|
+
capabilities, CLI choices, or support claims.
|
|
26
|
+
- Strengthened pre-mutation target-type checks and target-scoped physical table
|
|
27
|
+
identity guards. The executor continues to modify the same dataset and table
|
|
28
|
+
without creating OpenStatSpec rollback, copy, snapshot, or history layers.
|
|
29
|
+
|
|
30
|
+
### Specification basis
|
|
31
|
+
|
|
32
|
+
- Conformance and release validation continue to use OpenStatSpec specification
|
|
33
|
+
release v0.2.0 at exact commit
|
|
34
|
+
79339ec3d8f8aa81789b7e85f6b8afa6f1374e50.
|
|
35
|
+
|
|
5
36
|
## 0.3.0 — 2026-07-31
|
|
6
37
|
|
|
7
38
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: openstatspec
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Reference adapter for the OpenStatSpec relational contract
|
|
5
5
|
Author: OpenStatSpec
|
|
6
6
|
License: Apache-2.0
|
|
@@ -45,6 +45,10 @@ import.
|
|
|
45
45
|
capabilities, and loss reports.
|
|
46
46
|
- `openstatspec.sql`: database connection and wide-table/catalog operations.
|
|
47
47
|
- `openstatspec.spss`: SAV/ZSAV adapter boundary.
|
|
48
|
+
- `openstatspec.transform`: canonical plans, frontend-neutral schema concepts,
|
|
49
|
+
and plan validation.
|
|
50
|
+
- `openstatspec.frontends.spss`: the SPSS-like syntax frontend and convenience
|
|
51
|
+
execution adapter.
|
|
48
52
|
|
|
49
53
|
## Intended workflow
|
|
50
54
|
|
|
@@ -82,43 +86,17 @@ implemented capability boundary.
|
|
|
82
86
|
|
|
83
87
|
## SPSS-like transformation frontend
|
|
84
88
|
|
|
85
|
-
The
|
|
86
|
-
`VALUE LABELS
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
install_in_place_transformation_schema,
|
|
97
|
-
)
|
|
98
|
-
|
|
99
|
-
database_url = "postgresql+psycopg://user:password@host/database"
|
|
100
|
-
install_in_place_transformation_schema(database_url=database_url)
|
|
101
|
-
result = apply_spss_in_place(
|
|
102
|
-
database_url=database_url,
|
|
103
|
-
dataset_id="...",
|
|
104
|
-
actor="agent@example.org",
|
|
105
|
-
source_text="""
|
|
106
|
-
RECODE age (18 THRU 34 = 1) (35 THRU 64 = 2) INTO age_group.
|
|
107
|
-
VARIABLE LABELS age_group 'Age group'.
|
|
108
|
-
VALUE LABELS age_group 1 '18-34' 2 '35-64'.
|
|
109
|
-
""",
|
|
110
|
-
)
|
|
111
|
-
```
|
|
112
|
-
|
|
113
|
-
Existing-target recodes and metadata mutations use direct DML. SQLite and
|
|
114
|
-
PostgreSQL may also add a target column in the same native transaction. MySQL,
|
|
115
|
-
MariaDB, and Dolt reject such schema-changing plans before mutation because
|
|
116
|
-
their implicit-commit DDL could otherwise leave a partial apply. On Dolt, the
|
|
117
|
-
caller additionally supplies expected branch and HEAD identities, and the
|
|
118
|
-
working set must be clean. The transformer never calls `DOLT_COMMIT`.
|
|
119
|
-
|
|
120
|
-
See [in-place transformations](docs/in-place-transformation.md) for the full
|
|
121
|
-
execution boundary and CLI form.
|
|
89
|
+
The SPSS-like frontend lowers supported `RECODE`, `VARIABLE LABELS`, and
|
|
90
|
+
`VALUE LABELS` syntax into a language-neutral canonical plan. The in-place
|
|
91
|
+
path applies it to the same logical dataset, physical wide table, and metadata
|
|
92
|
+
catalog without a derived dataset, copied table, snapshot, or separate
|
|
93
|
+
rollback/history layer. Dolt remains the sole versioning layer for Dolt-backed
|
|
94
|
+
edits, and the transformer never calls `DOLT_COMMIT`.
|
|
95
|
+
|
|
96
|
+
See the [dataset transformations manual](docs/transformations.md) for schema
|
|
97
|
+
installation, Python and CLI surfaces, database invariants, audit provenance,
|
|
98
|
+
package layout, and extension guidance. Stata and SAS are unimplemented
|
|
99
|
+
placeholders.
|
|
122
100
|
|
|
123
101
|
## Current support status
|
|
124
102
|
|
|
@@ -26,6 +26,10 @@ import.
|
|
|
26
26
|
capabilities, and loss reports.
|
|
27
27
|
- `openstatspec.sql`: database connection and wide-table/catalog operations.
|
|
28
28
|
- `openstatspec.spss`: SAV/ZSAV adapter boundary.
|
|
29
|
+
- `openstatspec.transform`: canonical plans, frontend-neutral schema concepts,
|
|
30
|
+
and plan validation.
|
|
31
|
+
- `openstatspec.frontends.spss`: the SPSS-like syntax frontend and convenience
|
|
32
|
+
execution adapter.
|
|
29
33
|
|
|
30
34
|
## Intended workflow
|
|
31
35
|
|
|
@@ -63,43 +67,17 @@ implemented capability boundary.
|
|
|
63
67
|
|
|
64
68
|
## SPSS-like transformation frontend
|
|
65
69
|
|
|
66
|
-
The
|
|
67
|
-
`VALUE LABELS
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
install_in_place_transformation_schema,
|
|
78
|
-
)
|
|
79
|
-
|
|
80
|
-
database_url = "postgresql+psycopg://user:password@host/database"
|
|
81
|
-
install_in_place_transformation_schema(database_url=database_url)
|
|
82
|
-
result = apply_spss_in_place(
|
|
83
|
-
database_url=database_url,
|
|
84
|
-
dataset_id="...",
|
|
85
|
-
actor="agent@example.org",
|
|
86
|
-
source_text="""
|
|
87
|
-
RECODE age (18 THRU 34 = 1) (35 THRU 64 = 2) INTO age_group.
|
|
88
|
-
VARIABLE LABELS age_group 'Age group'.
|
|
89
|
-
VALUE LABELS age_group 1 '18-34' 2 '35-64'.
|
|
90
|
-
""",
|
|
91
|
-
)
|
|
92
|
-
```
|
|
93
|
-
|
|
94
|
-
Existing-target recodes and metadata mutations use direct DML. SQLite and
|
|
95
|
-
PostgreSQL may also add a target column in the same native transaction. MySQL,
|
|
96
|
-
MariaDB, and Dolt reject such schema-changing plans before mutation because
|
|
97
|
-
their implicit-commit DDL could otherwise leave a partial apply. On Dolt, the
|
|
98
|
-
caller additionally supplies expected branch and HEAD identities, and the
|
|
99
|
-
working set must be clean. The transformer never calls `DOLT_COMMIT`.
|
|
100
|
-
|
|
101
|
-
See [in-place transformations](docs/in-place-transformation.md) for the full
|
|
102
|
-
execution boundary and CLI form.
|
|
70
|
+
The SPSS-like frontend lowers supported `RECODE`, `VARIABLE LABELS`, and
|
|
71
|
+
`VALUE LABELS` syntax into a language-neutral canonical plan. The in-place
|
|
72
|
+
path applies it to the same logical dataset, physical wide table, and metadata
|
|
73
|
+
catalog without a derived dataset, copied table, snapshot, or separate
|
|
74
|
+
rollback/history layer. Dolt remains the sole versioning layer for Dolt-backed
|
|
75
|
+
edits, and the transformer never calls `DOLT_COMMIT`.
|
|
76
|
+
|
|
77
|
+
See the [dataset transformations manual](docs/transformations.md) for schema
|
|
78
|
+
installation, Python and CLI surfaces, database invariants, audit provenance,
|
|
79
|
+
package layout, and extension guidance. Stata and SAS are unimplemented
|
|
80
|
+
placeholders.
|
|
103
81
|
|
|
104
82
|
## Current support status
|
|
105
83
|
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# In-place transformations
|
|
2
|
+
|
|
3
|
+
This page has moved to the comprehensive
|
|
4
|
+
[dataset transformations manual](transformations.md).
|
|
5
|
+
|
|
6
|
+
The manual covers the canonical plan, SPSS-like frontend, executor, audit
|
|
7
|
+
schema, Python and CLI surfaces, database and Dolt invariants, provenance,
|
|
8
|
+
package layout, and future frontend boundary. Stata and SAS remain
|
|
9
|
+
unimplemented placeholders.
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# 0.
|
|
1
|
+
# 0.4.0 release readiness
|
|
2
2
|
|
|
3
3
|
This page records the expected release contract, not a publication event.
|
|
4
4
|
Creating a version tag remains a separate maintainer action.
|
|
@@ -56,6 +56,36 @@ otherwise the strict export policy reports and blocks the loss. A caller
|
|
|
56
56
|
that supplies consent for an available loss code receives the
|
|
57
57
|
machine-readable loss report with the export result.
|
|
58
58
|
|
|
59
|
+
## Transformation release gates
|
|
60
|
+
|
|
61
|
+
The canonical transformation core and SPSS syntax frontend are separate public
|
|
62
|
+
surfaces. A release must run the specification-owned canonical-plan and SPSS
|
|
63
|
+
frontend conformance fixtures, then exercise both the generic plan apply API
|
|
64
|
+
and the SPSS compatibility apply path.
|
|
65
|
+
|
|
66
|
+
The gate must prove that:
|
|
67
|
+
|
|
68
|
+
- a TransformationPlan object and its strict JSON mapping produce the same
|
|
69
|
+
plan hash and in-place result;
|
|
70
|
+
- top-level SPSS compiler imports and legacy openstatspec.transform re-exports
|
|
71
|
+
still load from an installed wheel;
|
|
72
|
+
- install-in-place-schema, apply-plan, and apply-spss execute their documented
|
|
73
|
+
CLI workflows;
|
|
74
|
+
- invalid or unsupported plans fail before the first data or metadata mutation;
|
|
75
|
+
- successful applies retain the same dataset ID, physical schema/table
|
|
76
|
+
identity, dataset count, and persistent physical data-table count;
|
|
77
|
+
- audit rows distinguish canonical plans from SPSS syntax, preserve correct
|
|
78
|
+
source/plan hashes and frontend contract, and contain no copied data;
|
|
79
|
+
- no OpenStatSpec rollback, snapshot, staging, copy, derived-dataset, or
|
|
80
|
+
parallel history artifacts are created; and
|
|
81
|
+
- Dolt checks expected branch, HEAD, and a clean working set without committing
|
|
82
|
+
or changing HEAD; other supported SQL connections remain allowed.
|
|
83
|
+
|
|
84
|
+
The built wheel must contain the generic openstatspec.transform modules and the
|
|
85
|
+
implemented openstatspec.frontends.spss package. Stata and SAS remain empty
|
|
86
|
+
source-tree placeholders and must expose no compiler, apply API, CLI choice,
|
|
87
|
+
capability claim, or implied support.
|
|
88
|
+
|
|
59
89
|
## Maintainer checks before tagging
|
|
60
90
|
|
|
61
91
|
1. Publish the pinned `openstatspec-pyspssio==0.5.1.post2` engine distribution
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
# Dataset transformations
|
|
2
|
+
|
|
3
|
+
OpenStatSpec separates transformation syntax, canonical meaning, and database
|
|
4
|
+
mutation. This lets multiple language frontends produce the same plan without
|
|
5
|
+
coupling the executor to any one language.
|
|
6
|
+
|
|
7
|
+
The implemented frontend accepts a small SPSS-like subset: `RECODE`,
|
|
8
|
+
`VARIABLE LABELS`, and `VALUE LABELS`. Stata and SAS are not implemented.
|
|
9
|
+
|
|
10
|
+
## Architecture
|
|
11
|
+
|
|
12
|
+
The path has three layers:
|
|
13
|
+
|
|
14
|
+
1. A **frontend** parses source text and binds names and types against an
|
|
15
|
+
explicit input schema. It performs no database mutation or SQL generation.
|
|
16
|
+
2. The **canonical plan** contains ordered, typed, language-neutral operations.
|
|
17
|
+
Its canonical JSON and hash are independent of SQL dialect and source
|
|
18
|
+
formatting.
|
|
19
|
+
3. The **in-place executor** validates the complete plan against the live
|
|
20
|
+
dataset, then directly mutates that dataset's existing wide table and
|
|
21
|
+
metadata catalog. It does not parse frontend syntax.
|
|
22
|
+
|
|
23
|
+
This is a trust boundary. JSON plans must pass
|
|
24
|
+
`transformation_plan_from_dict()` and live-schema validation before mutation;
|
|
25
|
+
they are never treated as arbitrary SQL.
|
|
26
|
+
|
|
27
|
+
## Install the audit schema
|
|
28
|
+
|
|
29
|
+
Install the compact audit relation once before the first apply:
|
|
30
|
+
|
|
31
|
+
```python
|
|
32
|
+
import openstatspec
|
|
33
|
+
|
|
34
|
+
openstatspec.install_in_place_transformation_schema(
|
|
35
|
+
database_url="sqlite:///survey.sqlite",
|
|
36
|
+
)
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
Installation is separate because schema DDL may commit independently on some
|
|
40
|
+
engines. Apply fails before changing data or metadata when the audit relation is
|
|
41
|
+
absent, and never creates or migrates schema-management objects itself.
|
|
42
|
+
|
|
43
|
+
The CLI equivalent is:
|
|
44
|
+
|
|
45
|
+
```text
|
|
46
|
+
openstatspec install-in-place-schema --database-url sqlite:///survey.sqlite
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
## Generic canonical-plan apply
|
|
50
|
+
|
|
51
|
+
`openstatspec.transform.TransformationPlan` is the generic model. Numeric
|
|
52
|
+
constants retain exact binary64 bits, strings retain exact Unicode text, and
|
|
53
|
+
operation order is significant. Use `transformation_plan_from_dict()` for
|
|
54
|
+
untrusted mappings. `canonical_plan_json()` and `canonical_plan_hash()`
|
|
55
|
+
produce stable audit identities.
|
|
56
|
+
|
|
57
|
+
The public API is:
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
result = openstatspec.apply_transformation_plan_in_place(
|
|
61
|
+
database_url="sqlite:///survey.sqlite",
|
|
62
|
+
dataset_id="responses",
|
|
63
|
+
plan=plan,
|
|
64
|
+
actor="agent@example.org",
|
|
65
|
+
expected_branch=None,
|
|
66
|
+
expected_head=None,
|
|
67
|
+
)
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
`plan` accepts a `TransformationPlan` or a mapping handled by the strict
|
|
71
|
+
loader. The executor recomputes the hash and validates the whole plan before
|
|
72
|
+
mutation.
|
|
73
|
+
|
|
74
|
+
The corresponding CLI is:
|
|
75
|
+
|
|
76
|
+
```text
|
|
77
|
+
openstatspec apply-plan --database-url sqlite:///survey.sqlite \
|
|
78
|
+
--dataset-id responses --actor agent@example.org --plan-file plan.json
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
## SPSS-like frontend
|
|
82
|
+
|
|
83
|
+
The pure compiler works without a database:
|
|
84
|
+
|
|
85
|
+
```python
|
|
86
|
+
from openstatspec import VariableDefinition, VariableSchema, compile_spss_syntax
|
|
87
|
+
|
|
88
|
+
schema = VariableSchema((VariableDefinition("age", "numeric"),))
|
|
89
|
+
compilation = compile_spss_syntax(
|
|
90
|
+
"RECODE age (18 THRU 34 = 1) (35 THRU 64 = 2).",
|
|
91
|
+
schema,
|
|
92
|
+
)
|
|
93
|
+
print(compilation.plan.canonical_json())
|
|
94
|
+
print(compilation.plan_hash)
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
It normalizes line endings, records the source hash, parses and sequentially
|
|
98
|
+
binds the supported subset, and returns a canonical plan plus output schema.
|
|
99
|
+
|
|
100
|
+
For database-connected use, the compatibility wrapper loads the live schema,
|
|
101
|
+
compiles the source, and invokes the in-place path:
|
|
102
|
+
|
|
103
|
+
```python
|
|
104
|
+
result = openstatspec.apply_spss_in_place(
|
|
105
|
+
database_url="sqlite:///survey.sqlite",
|
|
106
|
+
dataset_id="responses",
|
|
107
|
+
actor="agent@example.org",
|
|
108
|
+
source_text="""
|
|
109
|
+
RECODE age (18 THRU 34 = 1) (35 THRU 64 = 2).
|
|
110
|
+
VARIABLE LABELS age 'Age group'.
|
|
111
|
+
VALUE LABELS age 1 '18-34' 2 '35-64'.
|
|
112
|
+
""",
|
|
113
|
+
)
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
`openstatspec.compile_spss_syntax` and
|
|
117
|
+
`openstatspec.apply_spss_in_place` remain compatibility APIs.
|
|
118
|
+
|
|
119
|
+
The current CLI wrapper is:
|
|
120
|
+
|
|
121
|
+
```text
|
|
122
|
+
openstatspec apply-spss --database-url sqlite:///survey.sqlite \
|
|
123
|
+
--dataset-id responses --actor agent@example.org --syntax-file transform.sps
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
On Dolt, also pass `--expected-branch` and `--expected-head`.
|
|
127
|
+
|
|
128
|
+
## Database and Dolt invariants
|
|
129
|
+
|
|
130
|
+
Every successful apply preserves the logical `dataset_id` and physical
|
|
131
|
+
schema/table identity. It creates no derived dataset, output table, full-table
|
|
132
|
+
copy, staging table, snapshot, rollback artifact, or recovery/history layer.
|
|
133
|
+
Existing-target recodes use direct `UPDATE`; label operations mutate existing
|
|
134
|
+
catalog rows.
|
|
135
|
+
|
|
136
|
+
SQLite and PostgreSQL may add a numeric target where native transactions make
|
|
137
|
+
the complete operation atomic. MySQL, MariaDB, and Dolt reject target-creating
|
|
138
|
+
plans before the first mutation because implicit-commit DDL could leave a
|
|
139
|
+
partial apply. Their target column and metadata must already exist.
|
|
140
|
+
|
|
141
|
+
Dolt is the sole history, diff, branch, and rollback layer for Dolt-backed
|
|
142
|
+
datasets. Before mutation, the executor verifies the expected branch and
|
|
143
|
+
`HEAD` and requires clean `dolt_status`. Success changes the same working set
|
|
144
|
+
without changing `HEAD`. OpenStatSpec does not call `DOLT_COMMIT`, switch
|
|
145
|
+
branches, merge, reset, tag, or create a hidden recovery commit. The caller
|
|
146
|
+
reviews `dolt diff` and separately decides whether to commit or restore.
|
|
147
|
+
|
|
148
|
+
## Audit and provenance
|
|
149
|
+
|
|
150
|
+
Each success writes one compact `transformation_apply` row in the data and
|
|
151
|
+
metadata transaction. It records dataset and relation identity, actor, plan
|
|
152
|
+
hash, operation count, timestamps, database profile, and relevant Dolt
|
|
153
|
+
branch/HEAD. SPSS source also has a normalized source hash.
|
|
154
|
+
|
|
155
|
+
The row stores no case values and references no copied state. It is provenance,
|
|
156
|
+
not an undo log or substitute for Dolt history. `source_kind` distinguishes a
|
|
157
|
+
direct `canonical_plan` from `spss_syntax`. For a direct plan, the canonical
|
|
158
|
+
JSON document is itself the source artifact, so its source hash equals the plan
|
|
159
|
+
hash and `frontend_contract` is null. SPSS applies record the normalized
|
|
160
|
+
syntax hash and SPSS frontend contract.
|
|
161
|
+
|
|
162
|
+
## Package layout
|
|
163
|
+
|
|
164
|
+
The intended boundary is:
|
|
165
|
+
|
|
166
|
+
```text
|
|
167
|
+
openstatspec
|
|
168
|
+
├── transform
|
|
169
|
+
│ ├── plan.py
|
|
170
|
+
│ ├── schema.py
|
|
171
|
+
│ ├── validation.py
|
|
172
|
+
│ └── errors.py
|
|
173
|
+
└── frontends
|
|
174
|
+
├── spss
|
|
175
|
+
│ ├── syntax.py
|
|
176
|
+
│ ├── binding.py
|
|
177
|
+
│ ├── compiler.py
|
|
178
|
+
│ └── execution.py
|
|
179
|
+
├── stata # empty placeholder; not implemented
|
|
180
|
+
└── sas # empty placeholder; not implemented
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
Current imports from `openstatspec` and `openstatspec.transform` remain
|
|
184
|
+
compatibility contracts if implementation files move. Stata and SAS provide no
|
|
185
|
+
parser, compiler, apply API, CLI choice, or support claim.
|
|
186
|
+
|
|
187
|
+
## Extension guidance
|
|
188
|
+
|
|
189
|
+
There is no plugin discovery or Python entry-point protocol. A future built-in
|
|
190
|
+
frontend must parse with source spans, bind against `VariableSchema` without
|
|
191
|
+
database access, emit only canonical operations, produce deterministic hashes,
|
|
192
|
+
share the generic validator/executor, and add specification-owned conformance
|
|
193
|
+
fixtures.
|
|
194
|
+
|
|
195
|
+
An external plugin protocol should be added only for a real external frontend.
|
|
196
|
+
It would need explicit identity, contract compatibility, deterministic
|
|
197
|
+
compilation, capabilities, stable error semantics, and a trust policy.
|
|
198
|
+
|
|
199
|
+
Transformation conformance covers canonical plan fixtures, strict invalid-plan
|
|
200
|
+
cases, frontend source/plan hashes and diagnostics, wrapper-to-plan equivalence,
|
|
201
|
+
preflight-before-mutation, dataset/table identity, metadata and audit rows, CLI
|
|
202
|
+
compatibility, and service evidence before database execution support is
|
|
203
|
+
claimed. Stata and SAS need their own fixtures and implementations before their
|
|
204
|
+
placeholder status can change.
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
"""Public API for the OpenStatSpec Python reference implementation."""
|
|
2
2
|
|
|
3
3
|
from .api import (
|
|
4
|
-
apply_spss_in_place,
|
|
4
|
+
apply_spss_in_place, apply_transformation_plan_in_place,
|
|
5
|
+
capabilities, capability_matrix, derive_sql_dataset,
|
|
5
6
|
execute_sql_transformation,
|
|
6
7
|
export_sav, get_dataset, import_sav, inspect,
|
|
7
8
|
install_in_place_transformation_schema, list_datasets,
|
|
@@ -11,17 +12,24 @@ from .api import (
|
|
|
11
12
|
)
|
|
12
13
|
from .core import CapabilityDeclaration, LossReport, UnsupportedOperationError
|
|
13
14
|
from .sql.workflow import TransformationError
|
|
15
|
+
from .frontends.spss import SpssFrontendCompilation, compile_spss_syntax
|
|
14
16
|
from .transform import (
|
|
15
|
-
|
|
16
|
-
|
|
17
|
+
RecodeMatch, RecodeOperation, RecodeResult, RecodeRule,
|
|
18
|
+
ReplaceValueLabelsOperation, SetVariableLabelOperation,
|
|
19
|
+
TransformationFrontendError, TransformationPlan, TypedValue, ValueLabel,
|
|
20
|
+
VariableDefinition, VariableSchema, transformation_plan_from_dict,
|
|
17
21
|
)
|
|
18
22
|
|
|
19
23
|
__all__ = [
|
|
20
24
|
"CapabilityDeclaration", "LossReport", "SpssFrontendCompilation",
|
|
21
25
|
"TransformationError", "TransformationFrontendError",
|
|
22
|
-
"
|
|
26
|
+
"RecodeMatch", "RecodeOperation", "RecodeResult", "RecodeRule",
|
|
27
|
+
"ReplaceValueLabelsOperation", "SetVariableLabelOperation",
|
|
28
|
+
"TransformationPlan", "TypedValue", "ValueLabel",
|
|
29
|
+
"VariableDefinition", "VariableSchema", "transformation_plan_from_dict",
|
|
23
30
|
"UnsupportedOperationError", "capabilities", "capability_matrix",
|
|
24
|
-
"apply_spss_in_place", "
|
|
31
|
+
"apply_spss_in_place", "apply_transformation_plan_in_place",
|
|
32
|
+
"compile_spss_syntax", "derive_sql_dataset",
|
|
25
33
|
"execute_sql_transformation", "export_sav",
|
|
26
34
|
"get_dataset", "import_sav", "inspect",
|
|
27
35
|
"install_in_place_transformation_schema", "list_datasets",
|
|
@@ -20,11 +20,13 @@ from .sql.workflow import (
|
|
|
20
20
|
remove_derived_relation as _remove_derived_relation,
|
|
21
21
|
retire_derived_dataset as _retire_derived_dataset,
|
|
22
22
|
)
|
|
23
|
+
from .frontends.spss.execution import apply_spss_in_place as _apply_spss_in_place
|
|
23
24
|
from .sql.inplace_transform import (
|
|
24
|
-
|
|
25
|
+
apply_transformation_plan_in_place as _apply_transformation_plan_in_place,
|
|
25
26
|
in_place_transformation_capabilities,
|
|
26
27
|
install_in_place_transformation_schema as _install_in_place_schema,
|
|
27
28
|
)
|
|
29
|
+
from .transform import TransformationPlan
|
|
28
30
|
from .sql.capabilities import (
|
|
29
31
|
SPECIFICATION_COMMIT, SPECIFICATION_RELEASE, active_connection, catalog_binding,
|
|
30
32
|
)
|
|
@@ -154,6 +156,23 @@ def apply_spss_in_place(
|
|
|
154
156
|
))
|
|
155
157
|
|
|
156
158
|
|
|
159
|
+
def apply_transformation_plan_in_place(
|
|
160
|
+
*, database_url: Any, dataset_id: str,
|
|
161
|
+
plan: TransformationPlan | Mapping[str, Any],
|
|
162
|
+
actor: str, expected_branch: str | None = None,
|
|
163
|
+
expected_head: str | None = None,
|
|
164
|
+
) -> Mapping[str, Any]:
|
|
165
|
+
"""Apply a canonical plan to the same logical dataset and physical table."""
|
|
166
|
+
return result(_apply_transformation_plan_in_place(
|
|
167
|
+
database_url=str(database_url),
|
|
168
|
+
dataset_id=dataset_id,
|
|
169
|
+
plan=plan,
|
|
170
|
+
actor=actor,
|
|
171
|
+
expected_branch=expected_branch,
|
|
172
|
+
expected_head=expected_head,
|
|
173
|
+
))
|
|
174
|
+
|
|
175
|
+
|
|
157
176
|
def install_in_place_transformation_schema(*, database_url: Any) -> None:
|
|
158
177
|
"""Install the compact apply-audit relation before the first apply."""
|
|
159
178
|
_install_in_place_schema(database_url=str(database_url))
|
|
@@ -5,10 +5,12 @@ from collections.abc import Sequence
|
|
|
5
5
|
from pathlib import Path
|
|
6
6
|
|
|
7
7
|
from .api import (
|
|
8
|
-
apply_spss_in_place,
|
|
8
|
+
apply_spss_in_place, apply_transformation_plan_in_place,
|
|
9
|
+
capability_matrix, derive_sql_dataset,
|
|
9
10
|
execute_sql_transformation,
|
|
10
11
|
export_sav, get_dataset, import_sav, inspect, list_datasets,
|
|
11
|
-
|
|
12
|
+
install_in_place_transformation_schema, register_sql_transformation,
|
|
13
|
+
validate, validate_derived,
|
|
12
14
|
)
|
|
13
15
|
|
|
14
16
|
|
|
@@ -77,7 +79,7 @@ def main(argv: Sequence[str] | None = None) -> int:
|
|
|
77
79
|
|
|
78
80
|
apply_spss = commands.add_parser(
|
|
79
81
|
"apply-spss",
|
|
80
|
-
help="
|
|
82
|
+
help="compile supported SPSS syntax and apply it in-place",
|
|
81
83
|
)
|
|
82
84
|
apply_spss.add_argument("--database-url", required=True)
|
|
83
85
|
apply_spss.add_argument("--dataset-id", required=True)
|
|
@@ -88,6 +90,23 @@ def main(argv: Sequence[str] | None = None) -> int:
|
|
|
88
90
|
syntax_source.add_argument("--syntax")
|
|
89
91
|
syntax_source.add_argument("--syntax-file")
|
|
90
92
|
|
|
93
|
+
apply_plan = commands.add_parser(
|
|
94
|
+
"apply-plan",
|
|
95
|
+
help="apply a canonical transformation plan in-place",
|
|
96
|
+
)
|
|
97
|
+
apply_plan.add_argument("--database-url", required=True)
|
|
98
|
+
apply_plan.add_argument("--dataset-id", required=True)
|
|
99
|
+
apply_plan.add_argument("--actor", required=True)
|
|
100
|
+
apply_plan.add_argument("--expected-branch")
|
|
101
|
+
apply_plan.add_argument("--expected-head")
|
|
102
|
+
apply_plan.add_argument("--plan-file", required=True)
|
|
103
|
+
|
|
104
|
+
install_in_place = commands.add_parser(
|
|
105
|
+
"install-in-place-schema",
|
|
106
|
+
help="install or upgrade the compact in-place apply audit schema",
|
|
107
|
+
)
|
|
108
|
+
install_in_place.add_argument("--database-url", required=True)
|
|
109
|
+
|
|
91
110
|
derived_validator = commands.add_parser("validate-derived", help="validate a derived dataset")
|
|
92
111
|
derived_validator.add_argument("--database-url", required=True)
|
|
93
112
|
derived_validator.add_argument("--derived-dataset-id", required=True)
|
|
@@ -151,6 +170,19 @@ def main(argv: Sequence[str] | None = None) -> int:
|
|
|
151
170
|
expected_branch=args.expected_branch,
|
|
152
171
|
expected_head=args.expected_head,
|
|
153
172
|
)
|
|
173
|
+
elif args.command == "apply-plan":
|
|
174
|
+
plan = json.loads(Path(args.plan_file).read_text(encoding="utf-8"))
|
|
175
|
+
output = apply_transformation_plan_in_place(
|
|
176
|
+
database_url=args.database_url,
|
|
177
|
+
dataset_id=args.dataset_id,
|
|
178
|
+
plan=plan,
|
|
179
|
+
actor=args.actor,
|
|
180
|
+
expected_branch=args.expected_branch,
|
|
181
|
+
expected_head=args.expected_head,
|
|
182
|
+
)
|
|
183
|
+
elif args.command == "install-in-place-schema":
|
|
184
|
+
install_in_place_transformation_schema(database_url=args.database_url)
|
|
185
|
+
output = {"status": "installed"}
|
|
154
186
|
else:
|
|
155
187
|
output = validate_derived(
|
|
156
188
|
database_url=args.database_url, derived_dataset_id=args.derived_dataset_id,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Syntax frontends that lower external languages to canonical plans."""
|
|
File without changes
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""SPSS syntax frontend for canonical OpenStatSpec transformation plans."""
|
|
2
|
+
|
|
3
|
+
from .binding import bind_spss_syntax
|
|
4
|
+
from .compiler import SpssFrontendCompilation, compile_spss_syntax
|
|
5
|
+
from .syntax import (
|
|
6
|
+
SpssSyntaxProgram,
|
|
7
|
+
normalize_spss_source,
|
|
8
|
+
parse_spss_syntax,
|
|
9
|
+
spss_source_hash,
|
|
10
|
+
tokenize_spss,
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
SPSS_FRONTEND_CONTRACT = "openstatspec-spss-syntax-frontend-v0.1"
|
|
15
|
+
|
|
16
|
+
__all__ = [
|
|
17
|
+
"SPSS_FRONTEND_CONTRACT",
|
|
18
|
+
"SpssFrontendCompilation",
|
|
19
|
+
"SpssSyntaxProgram",
|
|
20
|
+
"bind_spss_syntax",
|
|
21
|
+
"compile_spss_syntax",
|
|
22
|
+
"normalize_spss_source",
|
|
23
|
+
"parse_spss_syntax",
|
|
24
|
+
"spss_source_hash",
|
|
25
|
+
"tokenize_spss",
|
|
26
|
+
]
|