openstatspec 0.1.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- openstatspec-0.3.0/.github/verify_release_ref.py +63 -0
- {openstatspec-0.1.0 → openstatspec-0.3.0}/.github/workflows/ci.yml +93 -4
- {openstatspec-0.1.0 → openstatspec-0.3.0}/.github/workflows/release.yml +30 -11
- openstatspec-0.3.0/CHANGELOG.md +92 -0
- openstatspec-0.3.0/PKG-INFO +180 -0
- openstatspec-0.3.0/README.md +161 -0
- openstatspec-0.3.0/docs/architecture.md +30 -0
- openstatspec-0.3.0/docs/in-place-transformation.md +41 -0
- {openstatspec-0.1.0 → openstatspec-0.3.0}/docs/release-readiness.md +28 -12
- {openstatspec-0.1.0 → openstatspec-0.3.0}/docs/sav-profile.md +23 -6
- openstatspec-0.3.0/docs/sql-transformation-workflow.md +204 -0
- {openstatspec-0.1.0 → openstatspec-0.3.0}/pyproject.toml +3 -3
- openstatspec-0.3.0/src/openstatspec/__init__.py +32 -0
- {openstatspec-0.1.0 → openstatspec-0.3.0}/src/openstatspec/api.py +93 -1
- openstatspec-0.3.0/src/openstatspec/cli.py +159 -0
- {openstatspec-0.1.0 → openstatspec-0.3.0}/src/openstatspec/spss/raw_dictionary.py +259 -28
- {openstatspec-0.1.0 → openstatspec-0.3.0}/src/openstatspec/spss/sav.py +7 -1
- openstatspec-0.3.0/src/openstatspec/sql/capabilities.py +452 -0
- openstatspec-0.3.0/src/openstatspec/sql/catalog_api.py +163 -0
- openstatspec-0.3.0/src/openstatspec/sql/inplace_transform.py +556 -0
- {openstatspec-0.1.0 → openstatspec-0.3.0}/src/openstatspec/sql/normative.py +70 -6
- {openstatspec-0.1.0 → openstatspec-0.3.0}/src/openstatspec/sql/profiles.py +20 -3
- {openstatspec-0.1.0 → openstatspec-0.3.0}/src/openstatspec/sql/wide.py +260 -106
- openstatspec-0.3.0/src/openstatspec/sql/workflow.py +2572 -0
- openstatspec-0.3.0/src/openstatspec/transform/__init__.py +33 -0
- openstatspec-0.3.0/src/openstatspec/transform/binding.py +298 -0
- openstatspec-0.3.0/src/openstatspec/transform/compiler.py +50 -0
- openstatspec-0.3.0/src/openstatspec/transform/errors.py +59 -0
- openstatspec-0.3.0/src/openstatspec/transform/plan.py +416 -0
- openstatspec-0.3.0/src/openstatspec/transform/syntax.py +478 -0
- {openstatspec-0.1.0 → openstatspec-0.3.0}/tests/conformance.py +19 -6
- openstatspec-0.3.0/tests/test_atomic_import.py +358 -0
- {openstatspec-0.1.0 → openstatspec-0.3.0}/tests/test_cli.py +3 -3
- openstatspec-0.3.0/tests/test_inplace_transform.py +309 -0
- {openstatspec-0.1.0 → openstatspec-0.3.0}/tests/test_official_conformance.py +4 -2
- openstatspec-0.3.0/tests/test_release_ref.py +97 -0
- openstatspec-0.3.0/tests/test_sql_profiles.py +520 -0
- openstatspec-0.3.0/tests/test_sql_services.py +326 -0
- openstatspec-0.3.0/tests/test_sql_workflow.py +1737 -0
- openstatspec-0.3.0/tests/test_transform_frontend.py +330 -0
- openstatspec-0.3.0/tests/test_vls_compatible_names.py +143 -0
- openstatspec-0.1.0/CHANGELOG.md +0 -28
- openstatspec-0.1.0/PKG-INFO +0 -90
- openstatspec-0.1.0/README.md +0 -73
- openstatspec-0.1.0/docs/architecture.md +0 -14
- openstatspec-0.1.0/src/openstatspec/__init__.py +0 -6
- openstatspec-0.1.0/src/openstatspec/cli.py +0 -42
- openstatspec-0.1.0/src/openstatspec/sql/capabilities.py +0 -260
- openstatspec-0.1.0/tests/test_atomic_import.py +0 -115
- openstatspec-0.1.0/tests/test_sql_profiles.py +0 -112
- openstatspec-0.1.0/tests/test_sql_services.py +0 -77
- {openstatspec-0.1.0 → openstatspec-0.3.0}/.gitignore +0 -0
- {openstatspec-0.1.0 → openstatspec-0.3.0}/THIRD_PARTY_NOTICES.md +0 -0
- {openstatspec-0.1.0 → openstatspec-0.3.0}/docs/pyspssio-spike.md +0 -0
- {openstatspec-0.1.0 → openstatspec-0.3.0}/src/openstatspec/core/__init__.py +0 -0
- {openstatspec-0.1.0 → openstatspec-0.3.0}/src/openstatspec/core/results.py +0 -0
- {openstatspec-0.1.0 → openstatspec-0.3.0}/src/openstatspec/spss/__init__.py +0 -0
- {openstatspec-0.1.0 → openstatspec-0.3.0}/src/openstatspec/spss/dictionary.py +0 -0
- {openstatspec-0.1.0 → openstatspec-0.3.0}/src/openstatspec/sql/__init__.py +0 -0
- {openstatspec-0.1.0 → openstatspec-0.3.0}/tests/test_attribute_catalog.py +0 -0
- {openstatspec-0.1.0 → openstatspec-0.3.0}/tests/test_document_round_trip.py +0 -0
- {openstatspec-0.1.0 → openstatspec-0.3.0}/tests/test_loss_reports.py +0 -0
- {openstatspec-0.1.0 → openstatspec-0.3.0}/tests/test_normative_catalog.py +0 -0
- {openstatspec-0.1.0 → openstatspec-0.3.0}/tests/test_pyspssio_catalog_authority.py +0 -0
- {openstatspec-0.1.0 → openstatspec-0.3.0}/tests/test_sav_sqlite.py +0 -0
- {openstatspec-0.1.0 → openstatspec-0.3.0}/tests/test_spss_catalog_preflight.py +0 -0
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""Fail closed unless the requested release tag names HEAD and the package version."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
import subprocess
|
|
8
|
+
import sys
|
|
9
|
+
import tomllib
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _git(*arguments: str) -> str:
|
|
13
|
+
completed = subprocess.run(
|
|
14
|
+
["git", *arguments],
|
|
15
|
+
check=False,
|
|
16
|
+
stdout=subprocess.PIPE,
|
|
17
|
+
stderr=subprocess.PIPE,
|
|
18
|
+
text=True,
|
|
19
|
+
)
|
|
20
|
+
if completed.returncode:
|
|
21
|
+
detail = completed.stderr.strip() or completed.stdout.strip()
|
|
22
|
+
raise RuntimeError(detail or f"git {' '.join(arguments)} failed")
|
|
23
|
+
return completed.stdout.strip()
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def verify_release_ref() -> None:
|
|
27
|
+
release_tag = os.environ.get("RELEASE_TAG", "")
|
|
28
|
+
version = str(
|
|
29
|
+
tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8"))["project"]["version"]
|
|
30
|
+
)
|
|
31
|
+
expected_tag = f"v{version}"
|
|
32
|
+
if release_tag != expected_tag:
|
|
33
|
+
raise RuntimeError(
|
|
34
|
+
f"release tag {release_tag!r} does not match package version {version!r}"
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
tag_ref = f"refs/tags/{release_tag}"
|
|
38
|
+
exists = subprocess.run(
|
|
39
|
+
["git", "show-ref", "--verify", "--quiet", tag_ref],
|
|
40
|
+
check=False,
|
|
41
|
+
)
|
|
42
|
+
if exists.returncode:
|
|
43
|
+
raise RuntimeError(f"release tag ref {tag_ref!r} does not exist")
|
|
44
|
+
|
|
45
|
+
tag_commit = _git("rev-parse", "--verify", f"{tag_ref}^{{commit}}")
|
|
46
|
+
head_commit = _git("rev-parse", "--verify", "HEAD^{commit}")
|
|
47
|
+
if tag_commit != head_commit:
|
|
48
|
+
raise RuntimeError(
|
|
49
|
+
f"release tag {release_tag!r} resolves to {tag_commit}, not HEAD {head_commit}"
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def main() -> int:
|
|
54
|
+
try:
|
|
55
|
+
verify_release_ref()
|
|
56
|
+
except (KeyError, OSError, RuntimeError, tomllib.TOMLDecodeError) as error:
|
|
57
|
+
print(f"release ref verification failed: {error}", file=sys.stderr)
|
|
58
|
+
return 1
|
|
59
|
+
return 0
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
if __name__ == "__main__":
|
|
63
|
+
raise SystemExit(main())
|
|
@@ -20,7 +20,7 @@ jobs:
|
|
|
20
20
|
uses: actions/checkout@v7
|
|
21
21
|
with: &specification-checkout
|
|
22
22
|
repository: OpenStatSpec/specification
|
|
23
|
-
ref:
|
|
23
|
+
ref: 79339ec3d8f8aa81789b7e85f6b8afa6f1374e50
|
|
24
24
|
path: openstatspec-specification
|
|
25
25
|
- name: Checkout required SPSS engine
|
|
26
26
|
uses: actions/checkout@v7
|
|
@@ -63,7 +63,7 @@ jobs:
|
|
|
63
63
|
strategy:
|
|
64
64
|
fail-fast: false
|
|
65
65
|
matrix:
|
|
66
|
-
postgres: ["17", "18"]
|
|
66
|
+
postgres: ["17.10", "18.4"]
|
|
67
67
|
services:
|
|
68
68
|
postgres:
|
|
69
69
|
image: postgres:${{ matrix.postgres }}
|
|
@@ -77,6 +77,7 @@ jobs:
|
|
|
77
77
|
--health-interval 10s --health-timeout 5s --health-retries 5
|
|
78
78
|
env:
|
|
79
79
|
OPENSTATSPEC_POSTGRES_URL: postgresql+psycopg://openstatspec:openstatspec@localhost:5432/openstatspec
|
|
80
|
+
OPENSTATSPEC_EXPECTED_POSTGRES_VERSION: ${{ matrix.postgres }}
|
|
80
81
|
steps: &sql-test-steps
|
|
81
82
|
- uses: actions/checkout@v7
|
|
82
83
|
- name: Checkout OpenStatSpec specification fixtures
|
|
@@ -99,7 +100,7 @@ jobs:
|
|
|
99
100
|
strategy:
|
|
100
101
|
fail-fast: false
|
|
101
102
|
matrix:
|
|
102
|
-
mysql: ["8.4", "9.7"]
|
|
103
|
+
mysql: ["8.4.11", "9.7.2"]
|
|
103
104
|
services:
|
|
104
105
|
mysql:
|
|
105
106
|
image: mysql:${{ matrix.mysql }}
|
|
@@ -114,6 +115,7 @@ jobs:
|
|
|
114
115
|
--health-interval 10s --health-timeout 5s --health-retries 10
|
|
115
116
|
env:
|
|
116
117
|
OPENSTATSPEC_MYSQL_URL: mysql+pymysql://openstatspec:openstatspec@localhost:3306/openstatspec
|
|
118
|
+
OPENSTATSPEC_EXPECTED_MYSQL_VERSION: ${{ matrix.mysql }}
|
|
117
119
|
steps: *sql-test-steps
|
|
118
120
|
|
|
119
121
|
mariadb-integration:
|
|
@@ -122,7 +124,7 @@ jobs:
|
|
|
122
124
|
strategy:
|
|
123
125
|
fail-fast: false
|
|
124
126
|
matrix:
|
|
125
|
-
mariadb: ["11.4", "11.8", "12.3"]
|
|
127
|
+
mariadb: ["11.4.12", "11.8.8", "12.3.2"]
|
|
126
128
|
services:
|
|
127
129
|
mariadb:
|
|
128
130
|
image: mariadb:${{ matrix.mariadb }}
|
|
@@ -137,4 +139,91 @@ jobs:
|
|
|
137
139
|
--health-interval 10s --health-timeout 5s --health-retries 10
|
|
138
140
|
env:
|
|
139
141
|
OPENSTATSPEC_MARIADB_URL: mysql+pymysql://openstatspec:openstatspec@localhost:3306/openstatspec
|
|
142
|
+
OPENSTATSPEC_EXPECTED_MARIADB_VERSION: ${{ matrix.mariadb }}
|
|
140
143
|
steps: *sql-test-steps
|
|
144
|
+
|
|
145
|
+
dolt-integration:
|
|
146
|
+
name: Dolt ${{ matrix.dolt }} integration
|
|
147
|
+
runs-on: ubuntu-latest
|
|
148
|
+
strategy:
|
|
149
|
+
fail-fast: false
|
|
150
|
+
matrix:
|
|
151
|
+
include:
|
|
152
|
+
- dolt: "2.2.2"
|
|
153
|
+
image: dolthub/dolt:2.2.2@sha256:ae85afdd95317f0a982cab87fa3092a5a2e6689bc5de822ad7ae4f65b86d39a1
|
|
154
|
+
- dolt: "2.2.3"
|
|
155
|
+
image: dolthub/dolt:2.2.3@sha256:1a651e738a2e7275b9c69fec8c5f42c6e23954314405c412f4dc9287ee7c0080
|
|
156
|
+
env:
|
|
157
|
+
OPENSTATSPEC_DOLT_URL: mysql+pymysql://openstatspec:openstatspec@127.0.0.1:13308/openstatspec
|
|
158
|
+
OPENSTATSPEC_EXPECTED_DOLT_VERSION: ${{ matrix.dolt }}
|
|
159
|
+
OPENSTATSPEC_DOLT_IMAGE: ${{ matrix.image }}
|
|
160
|
+
steps:
|
|
161
|
+
- uses: actions/checkout@v7
|
|
162
|
+
- name: Checkout OpenStatSpec specification fixtures
|
|
163
|
+
uses: actions/checkout@v7
|
|
164
|
+
with: *specification-checkout
|
|
165
|
+
- name: Checkout required SPSS engine
|
|
166
|
+
uses: actions/checkout@v7
|
|
167
|
+
with: *engine-checkout
|
|
168
|
+
- uses: actions/setup-python@v7
|
|
169
|
+
with:
|
|
170
|
+
python-version: "3.12"
|
|
171
|
+
- name: Initialize Dolt ${{ matrix.dolt }}
|
|
172
|
+
run: |
|
|
173
|
+
mkdir -p "$RUNNER_TEMP/openstatspec-dolt"
|
|
174
|
+
docker run --rm \
|
|
175
|
+
--volume "$RUNNER_TEMP/openstatspec-dolt:/var/lib/dolt" \
|
|
176
|
+
--workdir /var/lib/dolt \
|
|
177
|
+
"$OPENSTATSPEC_DOLT_IMAGE" init \
|
|
178
|
+
--name "OpenStatSpec CI" --email "ci@openstatspec.org"
|
|
179
|
+
docker run --rm \
|
|
180
|
+
--volume "$RUNNER_TEMP/openstatspec-dolt:/var/lib/dolt" \
|
|
181
|
+
--workdir /var/lib/dolt \
|
|
182
|
+
"$OPENSTATSPEC_DOLT_IMAGE" sql \
|
|
183
|
+
-q "CREATE DATABASE openstatspec"
|
|
184
|
+
docker run --rm \
|
|
185
|
+
--volume "$RUNNER_TEMP/openstatspec-dolt:/var/lib/dolt" \
|
|
186
|
+
--workdir /var/lib/dolt \
|
|
187
|
+
"$OPENSTATSPEC_DOLT_IMAGE" sql \
|
|
188
|
+
-q "CREATE USER 'openstatspec'@'%' IDENTIFIED BY 'openstatspec'; GRANT ALL PRIVILEGES ON openstatspec.* TO 'openstatspec'@'%'"
|
|
189
|
+
docker run --detach --name openstatspec-dolt \
|
|
190
|
+
--publish 127.0.0.1:13308:3306 \
|
|
191
|
+
--volume "$RUNNER_TEMP/openstatspec-dolt:/var/lib/dolt" \
|
|
192
|
+
--workdir /var/lib/dolt \
|
|
193
|
+
"$OPENSTATSPEC_DOLT_IMAGE" sql-server -H 0.0.0.0 -l warning
|
|
194
|
+
- run: python -m pip install --upgrade pip
|
|
195
|
+
- run: python -m pip install ./openstatspec-pyspssio
|
|
196
|
+
- run: python -m pip install -e ".[dev,sql]"
|
|
197
|
+
- name: Wait for Dolt
|
|
198
|
+
run: |
|
|
199
|
+
python - <<'PY'
|
|
200
|
+
import time
|
|
201
|
+
import pymysql
|
|
202
|
+
|
|
203
|
+
for attempt in range(30):
|
|
204
|
+
try:
|
|
205
|
+
connection = pymysql.connect(
|
|
206
|
+
host="127.0.0.1",
|
|
207
|
+
port=13308,
|
|
208
|
+
user="openstatspec",
|
|
209
|
+
password="openstatspec",
|
|
210
|
+
database="openstatspec",
|
|
211
|
+
)
|
|
212
|
+
except pymysql.MySQLError:
|
|
213
|
+
if attempt == 29:
|
|
214
|
+
raise
|
|
215
|
+
time.sleep(2)
|
|
216
|
+
else:
|
|
217
|
+
connection.close()
|
|
218
|
+
break
|
|
219
|
+
PY
|
|
220
|
+
- run: python -m pytest -m services
|
|
221
|
+
- name: Show Dolt logs on failure and stop server
|
|
222
|
+
if: always()
|
|
223
|
+
run: |
|
|
224
|
+
if docker inspect openstatspec-dolt >/dev/null 2>&1; then
|
|
225
|
+
if [ "${{ job.status }}" != "success" ]; then
|
|
226
|
+
docker logs openstatspec-dolt
|
|
227
|
+
fi
|
|
228
|
+
docker rm --force openstatspec-dolt
|
|
229
|
+
fi
|
|
@@ -7,6 +7,12 @@ on:
|
|
|
7
7
|
push:
|
|
8
8
|
tags:
|
|
9
9
|
- "v*"
|
|
10
|
+
workflow_dispatch:
|
|
11
|
+
inputs:
|
|
12
|
+
release_ref:
|
|
13
|
+
description: "Existing release tag to rebuild and publish idempotently"
|
|
14
|
+
required: true
|
|
15
|
+
type: string
|
|
10
16
|
|
|
11
17
|
permissions:
|
|
12
18
|
contents: read
|
|
@@ -16,11 +22,32 @@ jobs:
|
|
|
16
22
|
runs-on: ubuntu-latest
|
|
17
23
|
steps:
|
|
18
24
|
- uses: actions/checkout@v7
|
|
25
|
+
with:
|
|
26
|
+
ref: refs/tags/${{ inputs.release_ref || github.ref_name }}
|
|
27
|
+
fetch-depth: 0
|
|
28
|
+
- uses: actions/setup-python@v7
|
|
29
|
+
with:
|
|
30
|
+
python-version: "3.13"
|
|
31
|
+
- name: Verify release tag exists and targets this package version
|
|
32
|
+
env:
|
|
33
|
+
RELEASE_TAG: ${{ inputs.release_ref || github.ref_name }}
|
|
34
|
+
run: |
|
|
35
|
+
set -eu
|
|
36
|
+
git fetch --force --no-tags origin "refs/tags/${RELEASE_TAG}:refs/tags/${RELEASE_TAG}"
|
|
37
|
+
python .github/verify_release_ref.py
|
|
38
|
+
- run: python -m pip install --upgrade pip build
|
|
39
|
+
- name: Verify required SPSS engine is publicly installable
|
|
40
|
+
run: >-
|
|
41
|
+
python -m pip download --only-binary=:all: --no-deps
|
|
42
|
+
--dest /tmp/openstatspec-engine-release
|
|
43
|
+
openstatspec-pyspssio==0.5.1.post2
|
|
44
|
+
- name: Build clean release distributions
|
|
45
|
+
run: python -m build
|
|
19
46
|
- name: Checkout normative specification fixtures
|
|
20
47
|
uses: actions/checkout@v7
|
|
21
48
|
with:
|
|
22
49
|
repository: OpenStatSpec/specification
|
|
23
|
-
ref:
|
|
50
|
+
ref: 79339ec3d8f8aa81789b7e85f6b8afa6f1374e50
|
|
24
51
|
path: openstatspec-specification
|
|
25
52
|
- name: Checkout required SPSS engine
|
|
26
53
|
uses: actions/checkout@v7
|
|
@@ -28,19 +55,9 @@ jobs:
|
|
|
28
55
|
repository: TonisOrmisson/pyspssio
|
|
29
56
|
ref: e069adf33c70bcd9e8e6ee495106479463a84fa2
|
|
30
57
|
path: openstatspec-pyspssio
|
|
31
|
-
- uses: actions/setup-python@v7
|
|
32
|
-
with:
|
|
33
|
-
python-version: "3.13"
|
|
34
|
-
- run: python -m pip install --upgrade pip build
|
|
35
|
-
- name: Verify required SPSS engine is publicly installable
|
|
36
|
-
run: >-
|
|
37
|
-
python -m pip download --only-binary=:all: --no-deps
|
|
38
|
-
--dest /tmp/openstatspec-engine-release
|
|
39
|
-
openstatspec-pyspssio==0.5.1.post2
|
|
40
58
|
- run: python -m pip install ./openstatspec-pyspssio
|
|
41
59
|
- run: python -m pip install -e ".[dev]"
|
|
42
60
|
- run: python -m pytest -m "not services"
|
|
43
|
-
- run: python -m build
|
|
44
61
|
- run: python -m venv /tmp/openstatspec-release-smoke
|
|
45
62
|
- run: /tmp/openstatspec-release-smoke/bin/python -m pip install ./openstatspec-pyspssio
|
|
46
63
|
- run: /tmp/openstatspec-release-smoke/bin/python -m pip install dist/*.whl
|
|
@@ -64,3 +81,5 @@ jobs:
|
|
|
64
81
|
name: python-distributions
|
|
65
82
|
path: dist/
|
|
66
83
|
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
84
|
+
with:
|
|
85
|
+
skip-existing: true
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this reference implementation are documented here.
|
|
4
|
+
|
|
5
|
+
## 0.3.0 — 2026-07-31
|
|
6
|
+
|
|
7
|
+
### Added
|
|
8
|
+
|
|
9
|
+
- Added a bounded SPSS-like `RECODE`, `VARIABLE LABELS`, and `VALUE LABELS`
|
|
10
|
+
parser with stable diagnostics and conformance fixtures.
|
|
11
|
+
- Added canonical Transformation Plan serialization, RFC 8785 hashes, schema
|
|
12
|
+
binding, deterministic typed operations, and an agent-facing CLI/API.
|
|
13
|
+
- Added product-neutral in-place execution with same dataset/table identity,
|
|
14
|
+
direct data and metadata mutation, and compact operation audit. Dolt adds
|
|
15
|
+
expected branch/HEAD and clean-working-set checks.
|
|
16
|
+
|
|
17
|
+
### Changed
|
|
18
|
+
|
|
19
|
+
- Version history, diff, rollback, restoration, and commit remain database
|
|
20
|
+
responsibilities; the transformer creates no derived/copy/snapshot/recovery
|
|
21
|
+
layer and performs no `DOLT_COMMIT`.
|
|
22
|
+
- SQL server support now distinguishes conservative family claims from exact
|
|
23
|
+
CI evidence: PostgreSQL 17.x/18.x at 17.10/18.4, MySQL 8.4.x/9.7.x at
|
|
24
|
+
8.4.11/9.7.2, and MariaDB 11.4.x/11.8.x/12.3.x at
|
|
25
|
+
11.4.12/11.8.8/12.3.2. Each service job verifies the live normalized version.
|
|
26
|
+
- Dolt remains an independent core profile, now claiming canonical stable
|
|
27
|
+
versions `>=2.2.2,<2.3.0` and running its complete service suite at exact
|
|
28
|
+
versions 2.2.2 and 2.2.3 from immutable container-image pins.
|
|
29
|
+
SQLite retains its core `>=3.24.0,<4.0.0` and optional workflow
|
|
30
|
+
`>=3.35.0,<4.0.0` tiers. MSSQL remains unsupported and roadmap-only.
|
|
31
|
+
|
|
32
|
+
### Specification basis
|
|
33
|
+
|
|
34
|
+
- CI, release validation, and machine-readable capabilities use OpenStatSpec
|
|
35
|
+
specification release `v0.2.0` at exact commit
|
|
36
|
+
`79339ec3d8f8aa81789b7e85f6b8afa6f1374e50`.
|
|
37
|
+
|
|
38
|
+
## 0.2.0 — 2026-07-30
|
|
39
|
+
|
|
40
|
+
This release extends the reference adapter while keeping imported source
|
|
41
|
+
datasets immutable and all unsupported paths fail-closed.
|
|
42
|
+
|
|
43
|
+
### Added
|
|
44
|
+
|
|
45
|
+
- An optional SQLite-only Transformation Workflow for immutable, versioned,
|
|
46
|
+
parameterized SELECT definitions; materialized derived datasets; lineage and
|
|
47
|
+
weights; public catalog APIs; and CLI operations.
|
|
48
|
+
- An independent Dolt core SQL profile for exact product version 2.2.2, with
|
|
49
|
+
service-backed coverage, conservative adapter limits, complete compensating
|
|
50
|
+
cleanup, and pre-DDL rejection of unsupported values. Transformation Workflow
|
|
51
|
+
support on Dolt is not claimed.
|
|
52
|
+
|
|
53
|
+
### Changed
|
|
54
|
+
|
|
55
|
+
- SAV and ZSAV validation now rewrites legacy compatible names consistently
|
|
56
|
+
across type-2 records and subtype-13/subtype-14 long-name metadata.
|
|
57
|
+
- Transformation publication, interrupted-run reconciliation, retirement, and
|
|
58
|
+
physical-removal recovery now preserve auditable state and fail closed on
|
|
59
|
+
catalog or relation drift.
|
|
60
|
+
- Release automation builds distributions before test-only source checkouts and
|
|
61
|
+
supports idempotent rebuilds of an existing release tag.
|
|
62
|
+
|
|
63
|
+
### Specification basis
|
|
64
|
+
|
|
65
|
+
- CI and release validation are pinned to OpenStatSpec specification release
|
|
66
|
+
`v1.0.0-rc.1` at exact commit
|
|
67
|
+
`fef0dc6f4b17ff7141dad3f49d0524c63efbfed5`.
|
|
68
|
+
|
|
69
|
+
## 0.1.0 — 2026-07-29
|
|
70
|
+
|
|
71
|
+
First public reference implementation of the OpenStatSpec strict wide-table
|
|
72
|
+
SPSS profile.
|
|
73
|
+
|
|
74
|
+
### Included
|
|
75
|
+
|
|
76
|
+
- Import of unencrypted SAV and ZSAV sources into one dedicated wide SQL table
|
|
77
|
+
and its metadata catalog.
|
|
78
|
+
- Export of supported dataset semantics to SAV and ZSAV.
|
|
79
|
+
- SQLite, PostgreSQL, MySQL, and MariaDB profiles, including service-backed
|
|
80
|
+
PostgreSQL 17/18, MySQL 8.4/9.7, and MariaDB 11.4/11.8/12.3 CI coverage.
|
|
81
|
+
- Preflight checks for target profile limits, atomic imports, validation, a
|
|
82
|
+
command-line interface, and machine-readable capability and loss reports.
|
|
83
|
+
|
|
84
|
+
### Important boundaries
|
|
85
|
+
|
|
86
|
+
- This release promises semantic round-trip equivalence only for supported
|
|
87
|
+
features; it never promises byte-identical SAV/ZSAV output.
|
|
88
|
+
- The adapter requires explicit `allow_loss` consent before exporting known
|
|
89
|
+
unsupported or lossy dictionary semantics.
|
|
90
|
+
- Multiple-response sets, variable alignment, variable sets, and custom
|
|
91
|
+
attributes have explicit capability diagnostics; see the SAV profile.
|
|
92
|
+
- Encrypted SPSS files are not supported.
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: openstatspec
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Reference adapter for the OpenStatSpec relational contract
|
|
5
|
+
Author: OpenStatSpec
|
|
6
|
+
License: Apache-2.0
|
|
7
|
+
Requires-Python: >=3.11
|
|
8
|
+
Requires-Dist: openstatspec-pyspssio==0.5.1.post2
|
|
9
|
+
Requires-Dist: pandas>=2.2
|
|
10
|
+
Requires-Dist: rfc8785>=0.1.4
|
|
11
|
+
Requires-Dist: sqlalchemy>=2.0
|
|
12
|
+
Requires-Dist: sqlglot>=30.14
|
|
13
|
+
Provides-Extra: dev
|
|
14
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
15
|
+
Provides-Extra: sql
|
|
16
|
+
Requires-Dist: psycopg[binary]>=3.2; extra == 'sql'
|
|
17
|
+
Requires-Dist: pymysql>=1.1; extra == 'sql'
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
|
|
20
|
+
# OpenStatSpec Python
|
|
21
|
+
|
|
22
|
+
The reference Python implementation of the OpenStatSpec specification.
|
|
23
|
+
|
|
24
|
+
This package implements the specification; it does not define or extend it.
|
|
25
|
+
The normative model lives in the `OpenStatSpec/specification` repository.
|
|
26
|
+
|
|
27
|
+
## Boundaries
|
|
28
|
+
|
|
29
|
+
For each supported import, one source dataset becomes one dedicated wide SQL
|
|
30
|
+
table. Cases are rows and source variables are physical SQL columns. The
|
|
31
|
+
singular UUID-keyed tables from the specification (`dataset`, `variable`,
|
|
32
|
+
`operation`, `fidelity_event`, and related metadata tables) are the public
|
|
33
|
+
catalog contract. Historical `*_catalog` tables are an internal compatibility
|
|
34
|
+
layer for the current exporter and are not the standard database interface.
|
|
35
|
+
The adapter does not reshape data, create EAV
|
|
36
|
+
or long-form tables, or harmonize studies or waves.
|
|
37
|
+
|
|
38
|
+
Unsupported source features, SQL targets, or export paths fail explicitly.
|
|
39
|
+
There is no silent truncation, type conversion, metadata loss, or partial
|
|
40
|
+
import.
|
|
41
|
+
|
|
42
|
+
## Package layout
|
|
43
|
+
|
|
44
|
+
- `openstatspec.core`: pure standard concepts, validation, versions,
|
|
45
|
+
capabilities, and loss reports.
|
|
46
|
+
- `openstatspec.sql`: database connection and wide-table/catalog operations.
|
|
47
|
+
- `openstatspec.spss`: SAV/ZSAV adapter boundary.
|
|
48
|
+
|
|
49
|
+
## Intended workflow
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
from openstatspec import export_sav, import_sav
|
|
53
|
+
|
|
54
|
+
import_sav("responses.sav", database_url="postgresql+psycopg://user:password@server/database", dataset_id="responses-2026")
|
|
55
|
+
export_sav(database_url="postgresql+psycopg://user:password@server/database", dataset_id="responses-2026", destination="responses-roundtrip.sav")
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
```text
|
|
59
|
+
openstatspec import responses.sav --database-url postgresql+psycopg://... --dataset-id responses-2026
|
|
60
|
+
openstatspec export --database-url postgresql+psycopg://... --dataset-id responses-2026 --output responses-roundtrip.sav
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
## Optional database-first SQL workflow
|
|
64
|
+
|
|
65
|
+
Imported datasets remain immutable source records. The optional SQL
|
|
66
|
+
transformation profile can register versioned, parameterized SQLite SELECT queries,
|
|
67
|
+
materialize results, record lineage and weights, and expose
|
|
68
|
+
derived datasets through a public catalog API. It uses a separate profile
|
|
69
|
+
catalog and never presents SQL output as an imported source dataset.
|
|
70
|
+
Workflow operations support SQLite only in this milestone and fail closed on
|
|
71
|
+
PostgreSQL/MySQL/MariaDB; core import/export database support is unchanged.
|
|
72
|
+
The core SQLite import/export profile accepts SQLite `>=3.24.0,<4.0.0`; the
|
|
73
|
+
optional transformation workflow deliberately has the narrower
|
|
74
|
+
`>=3.35.0,<4.0.0` runtime preflight. These independent tiers do not change the
|
|
75
|
+
server-profile matrix. Microsoft SQL Server is not supported; its future
|
|
76
|
+
dialect is scoped only in the specification's
|
|
77
|
+
[MSSQL roadmap](https://github.com/OpenStatSpec/specification/blob/main/docs/mssql-dialect-roadmap.md).
|
|
78
|
+
|
|
79
|
+
See [the SQL transformation workflow](docs/sql-transformation-workflow.md) for
|
|
80
|
+
Python and CLI examples, migration behavior, hashing, atomicity, and the exact
|
|
81
|
+
implemented capability boundary.
|
|
82
|
+
|
|
83
|
+
## SPSS-like transformation frontend
|
|
84
|
+
|
|
85
|
+
The transformation frontend accepts `RECODE`, `VARIABLE LABELS`, and
|
|
86
|
+
`VALUE LABELS`, lowers them to a canonical OpenStatSpec Transformation Plan,
|
|
87
|
+
and mutates the same logical dataset and physical wide table. SQLite,
|
|
88
|
+
PostgreSQL, MySQL, MariaDB, and Dolt connections are allowed. It creates no
|
|
89
|
+
derived dataset, copied table, snapshot, or OpenStatSpec rollback/version layer.
|
|
90
|
+
|
|
91
|
+
Install the compact operation-audit relation separately, then apply syntax:
|
|
92
|
+
|
|
93
|
+
```python
|
|
94
|
+
from openstatspec import (
|
|
95
|
+
apply_spss_in_place,
|
|
96
|
+
install_in_place_transformation_schema,
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
database_url = "postgresql+psycopg://user:password@host/database"
|
|
100
|
+
install_in_place_transformation_schema(database_url=database_url)
|
|
101
|
+
result = apply_spss_in_place(
|
|
102
|
+
database_url=database_url,
|
|
103
|
+
dataset_id="...",
|
|
104
|
+
actor="agent@example.org",
|
|
105
|
+
source_text="""
|
|
106
|
+
RECODE age (18 THRU 34 = 1) (35 THRU 64 = 2) INTO age_group.
|
|
107
|
+
VARIABLE LABELS age_group 'Age group'.
|
|
108
|
+
VALUE LABELS age_group 1 '18-34' 2 '35-64'.
|
|
109
|
+
""",
|
|
110
|
+
)
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
Existing-target recodes and metadata mutations use direct DML. SQLite and
|
|
114
|
+
PostgreSQL may also add a target column in the same native transaction. MySQL,
|
|
115
|
+
MariaDB, and Dolt reject such schema-changing plans before mutation because
|
|
116
|
+
their implicit-commit DDL could otherwise leave a partial apply. On Dolt, the
|
|
117
|
+
caller additionally supplies expected branch and HEAD identities, and the
|
|
118
|
+
working set must be clean. The transformer never calls `DOLT_COMMIT`.
|
|
119
|
+
|
|
120
|
+
See [in-place transformations](docs/in-place-transformation.md) for the full
|
|
121
|
+
execution boundary and CLI form.
|
|
122
|
+
|
|
123
|
+
## Current support status
|
|
124
|
+
|
|
125
|
+
The adapter requires `openstatspec-pyspssio==0.5.1.post2` as its sole SPSS
|
|
126
|
+
engine. Its import module remains `pyspssio`; the exact source commit is recorded
|
|
127
|
+
in operation metadata. There is no fallback reader or writer. It supports unencrypted SAV and ZSAV import and
|
|
128
|
+
SAV/ZSAV export for the semantics exposed by that engine. SQLite is the local
|
|
129
|
+
reference path.
|
|
130
|
+
PostgreSQL, MySQL, MariaDB, and Dolt are each covered by separate service-backed CI
|
|
131
|
+
conformance checks. Dolt support is an independent core profile for the
|
|
132
|
+
canonical stable range `>=2.2.2,<2.3.0`; earlier patches, other families,
|
|
133
|
+
noncanonical versions, and unknown MySQL-wire products fail closed.
|
|
134
|
+
|
|
135
|
+
The supported family claims are broader than the deliberately exact CI
|
|
136
|
+
evidence points: PostgreSQL 17.x/18.x is exercised at 17.10/18.4, MySQL
|
|
137
|
+
8.4.x/9.7.x at 8.4.11/9.7.2, and MariaDB 11.4.x/11.8.x/12.3.x at
|
|
138
|
+
11.4.12/11.8.8/12.3.2. Each service job checks the normalized live server
|
|
139
|
+
version against its exact matrix entry before that run can count as evidence.
|
|
140
|
+
Dolt claims the conservative 2.2.x range `>=2.2.2,<2.3.0`; its full service
|
|
141
|
+
suite is exercised independently at exact versions 2.2.2 and 2.2.3 using
|
|
142
|
+
immutable container-image digests.
|
|
143
|
+
|
|
144
|
+
| Engine/profile | Runtime supported policy | Exact CI-tested versions |
|
|
145
|
+
| --- | --- | --- |
|
|
146
|
+
| SQLite core / optional workflow | Core `>=3.24.0,<4.0.0`; optional workflow `>=3.35.0,<4.0.0` | Runtime-provided SQLite on Python 3.11–3.14 runners; not a pinned server image |
|
|
147
|
+
| PostgreSQL | 17.x and 18.x | 17.10 and 18.4 |
|
|
148
|
+
| MySQL | 8.4.x and 9.7.x | 8.4.11 and 9.7.2 |
|
|
149
|
+
| MariaDB | 11.4.x, 11.8.x, and 12.3.x | 11.4.12, 11.8.8, and 12.3.2 |
|
|
150
|
+
| Dolt | 2.2.x with `>=2.2.2,<2.3.0` | 2.2.2 and 2.2.3 |
|
|
151
|
+
|
|
152
|
+
Microsoft SQL Server (MSSQL) remains roadmap-only and is not a supported
|
|
153
|
+
runtime profile; see the specification's [MSSQL roadmap](https://github.com/OpenStatSpec/specification/blob/main/docs/mssql-dialect-roadmap.md).
|
|
154
|
+
|
|
155
|
+
Use these explicit SQLAlchemy URLs:
|
|
156
|
+
|
|
157
|
+
- SQLite: `sqlite:///dataset.sqlite`
|
|
158
|
+
- PostgreSQL: `postgresql+psycopg://user:password@host/database`
|
|
159
|
+
- MySQL/MariaDB: `mysql+pymysql://user:password@host/database`
|
|
160
|
+
- Dolt `>=2.2.2,<2.3.0`: `mysql+pymysql://user:password@host/database` (detected by server identity)
|
|
161
|
+
|
|
162
|
+
The Dolt core profile supports strict wide-table import, validation, and export;
|
|
163
|
+
the separate Transformation Workflow is unsupported.
|
|
164
|
+
|
|
165
|
+
Run `openstatspec capabilities` before an integration to inspect the
|
|
166
|
+
machine-readable feature matrix. Export is deliberately strict: if known
|
|
167
|
+
dictionary semantics cannot be reproduced, it stops until you pass the exact
|
|
168
|
+
diagnostic code with `--allow-loss`. This avoids silent loss while making an
|
|
169
|
+
intentional lossy export auditable.
|
|
170
|
+
|
|
171
|
+
The matrix is also available to Python callers as
|
|
172
|
+
`openstatspec.capability_matrix()`. It distinguishes supported semantics from
|
|
173
|
+
unobservable and fail-closed paths; see the SAV profile for the exact
|
|
174
|
+
the openstatspec-pyspssio boundary.
|
|
175
|
+
|
|
176
|
+
See [the SAV profile](docs/sav-profile.md) for feature boundaries and
|
|
177
|
+
[release readiness](docs/release-readiness.md) for the pre-tag checklist.
|
|
178
|
+
Read [third-party notices](THIRD_PARTY_NOTICES.md) before distributing a
|
|
179
|
+
bundled application: the required engine includes IBM redistributables under
|
|
180
|
+
separate terms.
|