openstatspec 0.1.0__tar.gz → 0.2.0__tar.gz

openstatspec-0.2.0/.github/verify_release_ref.py

@@ -0,0 +1,63 @@
+"""Fail closed unless the requested release tag names HEAD and the package version."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tomllib
+
+
+def _git(*arguments: str) -> str:
+    completed = subprocess.run(
+        ["git", *arguments],
+        check=False,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+    )
+    if completed.returncode:
+        detail = completed.stderr.strip() or completed.stdout.strip()
+        raise RuntimeError(detail or f"git {' '.join(arguments)} failed")
+    return completed.stdout.strip()
+
+
+def verify_release_ref() -> None:
+    release_tag = os.environ.get("RELEASE_TAG", "")
+    version = str(
+        tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8"))["project"]["version"]
+    )
+    expected_tag = f"v{version}"
+    if release_tag != expected_tag:
+        raise RuntimeError(
+            f"release tag {release_tag!r} does not match package version {version!r}"
+        )
+
+    tag_ref = f"refs/tags/{release_tag}"
+    exists = subprocess.run(
+        ["git", "show-ref", "--verify", "--quiet", tag_ref],
+        check=False,
+    )
+    if exists.returncode:
+        raise RuntimeError(f"release tag ref {tag_ref!r} does not exist")
+
+    tag_commit = _git("rev-parse", "--verify", f"{tag_ref}^{{commit}}")
+    head_commit = _git("rev-parse", "--verify", "HEAD^{commit}")
+    if tag_commit != head_commit:
+        raise RuntimeError(
+            f"release tag {release_tag!r} resolves to {tag_commit}, not HEAD {head_commit}"
+        )
+
+
+def main() -> int:
+    try:
+        verify_release_ref()
+    except (KeyError, OSError, RuntimeError, tomllib.TOMLDecodeError) as error:
+        print(f"release ref verification failed: {error}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

{openstatspec-0.1.0 → openstatspec-0.2.0}/.github/workflows/ci.yml

@@ -20,7 +20,7 @@ jobs:
         uses: actions/checkout@v7
         with: &specification-checkout
           repository: OpenStatSpec/specification
-          ref: 6b9d1fc38f2f083c0ac5cf1c64874a6d07b95045
+          ref: fef0dc6f4b17ff7141dad3f49d0524c63efbfed5
           path: openstatspec-specification
       - name: Checkout required SPSS engine
         uses: actions/checkout@v7
@@ -138,3 +138,79 @@ jobs:
     env:
       OPENSTATSPEC_MARIADB_URL: mysql+pymysql://openstatspec:openstatspec@localhost:3306/openstatspec
     steps: *sql-test-steps
+
+  dolt-integration:
+    name: Dolt 2.2.2 integration
+    runs-on: ubuntu-latest
+    env:
+      OPENSTATSPEC_DOLT_URL: mysql+pymysql://openstatspec:openstatspec@127.0.0.1:13308/openstatspec
+    steps:
+      - uses: actions/checkout@v7
+      - name: Checkout OpenStatSpec specification fixtures
+        uses: actions/checkout@v7
+        with: *specification-checkout
+      - name: Checkout required SPSS engine
+        uses: actions/checkout@v7
+        with: *engine-checkout
+      - uses: actions/setup-python@v7
+        with:
+          python-version: "3.12"
+      - name: Initialize Dolt 2.2.2
+        run: |
+          mkdir -p "$RUNNER_TEMP/openstatspec-dolt"
+          docker run --rm \
+            --volume "$RUNNER_TEMP/openstatspec-dolt:/var/lib/dolt" \
+            --workdir /var/lib/dolt \
+            dolthub/dolt:2.2.2 init \
+            --name "OpenStatSpec CI" --email "ci@openstatspec.org"
+          docker run --rm \
+            --volume "$RUNNER_TEMP/openstatspec-dolt:/var/lib/dolt" \
+            --workdir /var/lib/dolt \
+            dolthub/dolt:2.2.2 sql \
+            -q "CREATE DATABASE openstatspec"
+          docker run --rm \
+            --volume "$RUNNER_TEMP/openstatspec-dolt:/var/lib/dolt" \
+            --workdir /var/lib/dolt \
+            dolthub/dolt:2.2.2 sql \
+            -q "CREATE USER 'openstatspec'@'%' IDENTIFIED BY 'openstatspec'; GRANT ALL PRIVILEGES ON openstatspec.* TO 'openstatspec'@'%'"
+          docker run --detach --name openstatspec-dolt \
+            --publish 127.0.0.1:13308:3306 \
+            --volume "$RUNNER_TEMP/openstatspec-dolt:/var/lib/dolt" \
+            --workdir /var/lib/dolt \
+            dolthub/dolt:2.2.2 sql-server -H 0.0.0.0 -l warning
+      - run: python -m pip install --upgrade pip
+      - run: python -m pip install ./openstatspec-pyspssio
+      - run: python -m pip install -e ".[dev,sql]"
+      - name: Wait for Dolt
+        run: |
+          python - <<'PY'
+          import time
+          import pymysql
+
+          for attempt in range(30):
+              try:
+                  connection = pymysql.connect(
+                      host="127.0.0.1",
+                      port=13308,
+                      user="openstatspec",
+                      password="openstatspec",
+                      database="openstatspec",
+                  )
+              except pymysql.MySQLError:
+                  if attempt == 29:
+                      raise
+                  time.sleep(2)
+              else:
+                  connection.close()
+                  break
+          PY
+      - run: python -m pytest -m services
+      - name: Show Dolt logs on failure and stop server
+        if: always()
+        run: |
+          if docker inspect openstatspec-dolt >/dev/null 2>&1; then
+            if [ "${{ job.status }}" != "success" ]; then
+              docker logs openstatspec-dolt
+            fi
+            docker rm --force openstatspec-dolt
+          fi

{openstatspec-0.1.0 → openstatspec-0.2.0}/.github/workflows/release.yml

@@ -7,6 +7,12 @@ on:
   push:
     tags:
       - "v*"
+  workflow_dispatch:
+    inputs:
+      release_ref:
+        description: "Existing release tag to rebuild and publish idempotently"
+        required: true
+        type: string
 
 permissions:
   contents: read
@@ -16,11 +22,32 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v7
+        with:
+          ref: refs/tags/${{ inputs.release_ref || github.ref_name }}
+          fetch-depth: 0
+      - uses: actions/setup-python@v7
+        with:
+          python-version: "3.13"
+      - name: Verify release tag exists and targets this package version
+        env:
+          RELEASE_TAG: ${{ inputs.release_ref || github.ref_name }}
+        run: |
+          set -eu
+          git fetch --force --no-tags origin "refs/tags/${RELEASE_TAG}:refs/tags/${RELEASE_TAG}"
+          python .github/verify_release_ref.py
+      - run: python -m pip install --upgrade pip build
+      - name: Verify required SPSS engine is publicly installable
+        run: >-
+          python -m pip download --only-binary=:all: --no-deps
+          --dest /tmp/openstatspec-engine-release
+          openstatspec-pyspssio==0.5.1.post2
+      - name: Build clean release distributions
+        run: python -m build
       - name: Checkout normative specification fixtures
         uses: actions/checkout@v7
         with:
           repository: OpenStatSpec/specification
-          ref: 6b9d1fc38f2f083c0ac5cf1c64874a6d07b95045
+          ref: fef0dc6f4b17ff7141dad3f49d0524c63efbfed5
           path: openstatspec-specification
       - name: Checkout required SPSS engine
         uses: actions/checkout@v7
@@ -28,19 +55,9 @@ jobs:
           repository: TonisOrmisson/pyspssio
           ref: e069adf33c70bcd9e8e6ee495106479463a84fa2
           path: openstatspec-pyspssio
-      - uses: actions/setup-python@v7
-        with:
-          python-version: "3.13"
-      - run: python -m pip install --upgrade pip build
-      - name: Verify required SPSS engine is publicly installable
-        run: >-
-          python -m pip download --only-binary=:all: --no-deps
-          --dest /tmp/openstatspec-engine-release
-          openstatspec-pyspssio==0.5.1.post2
       - run: python -m pip install ./openstatspec-pyspssio
       - run: python -m pip install -e ".[dev]"
       - run: python -m pytest -m "not services"
-      - run: python -m build
       - run: python -m venv /tmp/openstatspec-release-smoke
       - run: /tmp/openstatspec-release-smoke/bin/python -m pip install ./openstatspec-pyspssio
       - run: /tmp/openstatspec-release-smoke/bin/python -m pip install dist/*.whl
@@ -64,3 +81,5 @@ jobs:
           name: python-distributions
           path: dist/
       - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          skip-existing: true

openstatspec-0.2.0/CHANGELOG.md

@@ -0,0 +1,59 @@
+# Changelog
+
+All notable changes to this reference implementation are documented here.
+
+## 0.2.0 — 2026-07-30
+
+This release extends the reference adapter while keeping imported source
+datasets immutable and all unsupported paths fail-closed.
+
+### Added
+
+- An optional SQLite-only Transformation Workflow for immutable, versioned,
+  parameterized SELECT definitions; materialized derived datasets; lineage and
+  weights; public catalog APIs; and CLI operations.
+- An independent Dolt core SQL profile for exact product version 2.2.2, with
+  service-backed coverage, conservative adapter limits, complete compensating
+  cleanup, and pre-DDL rejection of unsupported values. Transformation Workflow
+  support on Dolt is not claimed.
+
+### Changed
+
+- SAV and ZSAV validation now rewrites legacy compatible names consistently
+  across type-2 records and subtype-13/subtype-14 long-name metadata.
+- Transformation publication, interrupted-run reconciliation, retirement, and
+  physical-removal recovery now preserve auditable state and fail closed on
+  catalog or relation drift.
+- Release automation builds distributions before test-only source checkouts and
+  supports idempotent rebuilds of an existing release tag.
+
+### Specification basis
+
+- CI and release validation are pinned to OpenStatSpec specification release
+  `v1.0.0-rc.1` at exact commit
+  `fef0dc6f4b17ff7141dad3f49d0524c63efbfed5`.
+
+## 0.1.0 — 2026-07-29
+
+First public reference implementation of the OpenStatSpec strict wide-table
+SPSS profile.
+
+### Included
+
+- Import of unencrypted SAV and ZSAV sources into one dedicated wide SQL table
+  and its metadata catalog.
+- Export of supported dataset semantics to SAV and ZSAV.
+- SQLite, PostgreSQL, MySQL, and MariaDB profiles, including service-backed
+  PostgreSQL 17/18, MySQL 8.4/9.7, and MariaDB 11.4/11.8/12.3 CI coverage.
+- Preflight checks for target profile limits, atomic imports, validation, a
+  command-line interface, and machine-readable capability and loss reports.
+
+### Important boundaries
+
+- This release promises semantic round-trip equivalence only for supported
+  features; it never promises byte-identical SAV/ZSAV output.
+- The adapter requires explicit `allow_loss` consent before exporting known
+  unsupported or lossy dictionary semantics.
+- Multiple-response sets, variable alignment, variable sets, and custom
+  attributes have explicit capability diagnostics; see the SAV profile.
+- Encrypted SPSS files are not supported.

{openstatspec-0.1.0 → openstatspec-0.2.0}/PKG-INFO

@@ -1,13 +1,15 @@
 Metadata-Version: 2.4
 Name: openstatspec
-Version: 0.1.0
+Version: 0.2.0
 Summary: Reference adapter for the OpenStatSpec relational contract
 Author: OpenStatSpec
 License: Apache-2.0
 Requires-Python: >=3.11
 Requires-Dist: openstatspec-pyspssio==0.5.1.post2
 Requires-Dist: pandas>=2.2
+Requires-Dist: rfc8785>=0.1.4
 Requires-Dist: sqlalchemy>=2.0
+Requires-Dist: sqlglot>=30.14
 Provides-Extra: dev
 Requires-Dist: pytest>=8; extra == 'dev'
 Provides-Extra: sql
@@ -58,19 +60,38 @@ openstatspec import responses.sav --database-url postgresql+psycopg://... --data
 openstatspec export --database-url postgresql+psycopg://... --dataset-id responses-2026 --output responses-roundtrip.sav
 ```
 
-## 0.1.0 support status
+## Optional database-first SQL workflow
+
+Imported datasets remain immutable source records. The optional SQL
+transformation profile can register versioned, parameterized SQLite SELECT queries,
+materialize results, record lineage and weights, and expose
+derived datasets through a public catalog API. It uses a separate profile
+catalog and never presents SQL output as an imported source dataset.
+Workflow operations support SQLite only in this milestone and fail closed on
+PostgreSQL/MySQL/MariaDB; core import/export database support is unchanged.
+
+See [the SQL transformation workflow](docs/sql-transformation-workflow.md) for
+Python and CLI examples, migration behavior, hashing, atomicity, and the exact
+implemented capability boundary.
+
+## 0.2.0 support status
 
 The adapter requires `openstatspec-pyspssio==0.5.1.post2` as its sole SPSS
 engine. Its import module remains `pyspssio`; the exact source commit is recorded
 in operation metadata. There is no fallback reader or writer. It supports unencrypted SAV and ZSAV import and
 SAV/ZSAV export for the semantics exposed by that engine. SQLite is the local
 reference path.
-PostgreSQL, MySQL, and MariaDB are each covered by separate service-backed CI
-conformance checks. Use these explicit SQLAlchemy URLs:
+PostgreSQL, MySQL, MariaDB, and Dolt are each covered by separate service-backed CI
+conformance checks. Dolt support is an independent core profile pinned to 2.2.2;
+other Dolt versions and unknown MySQL-wire products fail closed. Use these explicit SQLAlchemy URLs:
 
 - SQLite: `sqlite:///dataset.sqlite`
 - PostgreSQL: `postgresql+psycopg://user:password@host/database`
 - MySQL/MariaDB: `mysql+pymysql://user:password@host/database`
+- Dolt 2.2.2: `mysql+pymysql://user:password@host/database` (detected by server identity)
+
+The Dolt core profile supports strict wide-table import, validation, and export;
+the separate Transformation Workflow is unsupported.
 
 Run `openstatspec capabilities` before an integration to inspect the
 machine-readable feature matrix. Export is deliberately strict: if known

{openstatspec-0.1.0 → openstatspec-0.2.0}/README.md

@@ -41,19 +41,38 @@ openstatspec import responses.sav --database-url postgresql+psycopg://... --data
 openstatspec export --database-url postgresql+psycopg://... --dataset-id responses-2026 --output responses-roundtrip.sav
 ```
 
-## 0.1.0 support status
+## Optional database-first SQL workflow
+
+Imported datasets remain immutable source records. The optional SQL
+transformation profile can register versioned, parameterized SQLite SELECT queries,
+materialize results, record lineage and weights, and expose
+derived datasets through a public catalog API. It uses a separate profile
+catalog and never presents SQL output as an imported source dataset.
+Workflow operations support SQLite only in this milestone and fail closed on
+PostgreSQL/MySQL/MariaDB; core import/export database support is unchanged.
+
+See [the SQL transformation workflow](docs/sql-transformation-workflow.md) for
+Python and CLI examples, migration behavior, hashing, atomicity, and the exact
+implemented capability boundary.
+
+## 0.2.0 support status
 
 The adapter requires `openstatspec-pyspssio==0.5.1.post2` as its sole SPSS
 engine. Its import module remains `pyspssio`; the exact source commit is recorded
 in operation metadata. There is no fallback reader or writer. It supports unencrypted SAV and ZSAV import and
 SAV/ZSAV export for the semantics exposed by that engine. SQLite is the local
 reference path.
-PostgreSQL, MySQL, and MariaDB are each covered by separate service-backed CI
-conformance checks. Use these explicit SQLAlchemy URLs:
+PostgreSQL, MySQL, MariaDB, and Dolt are each covered by separate service-backed CI
+conformance checks. Dolt support is an independent core profile pinned to 2.2.2;
+other Dolt versions and unknown MySQL-wire products fail closed. Use these explicit SQLAlchemy URLs:
 
 - SQLite: `sqlite:///dataset.sqlite`
 - PostgreSQL: `postgresql+psycopg://user:password@host/database`
 - MySQL/MariaDB: `mysql+pymysql://user:password@host/database`
+- Dolt 2.2.2: `mysql+pymysql://user:password@host/database` (detected by server identity)
+
+The Dolt core profile supports strict wide-table import, validation, and export;
+the separate Transformation Workflow is unsupported.
 
 Run `openstatspec capabilities` before an integration to inspect the
 machine-readable feature matrix. Export is deliberately strict: if known

openstatspec-0.2.0/docs/architecture.md

@@ -0,0 +1,30 @@
+# Architecture
+
+`openstatspec.core` contains only pure concepts defined by the standard:
+versioning, validation concepts, capability declarations, and loss reports.
+`openstatspec.sql` owns database-target and strict wide-table/catalog work.
+`openstatspec.spss` owns SAV/ZSAV source and export behavior.
+
+The public workflow remains database-connected. A supported import receives an
+SPSS source, a database URL or connection, and a dataset identity. It creates
+one dedicated wide data table plus separate catalog metadata. Export identifies
+the database dataset and writes a supported SPSS file.
+
+No layer may silently substitute EAV, long-form views, JSON, pivots, automatic
+harmonization, truncation, or partial import for the standard's strict mapping.
+
+
+## Optional SQL transformation profile
+
+`openstatspec.sql.workflow` layers a separate, independently identified
+catalog beside the core contract. It owns immutable transformation
+definitions/versions, driver-bound runs, input snapshots, derived relations,
+variables, lineage, weights, safe events, and append-only disposition events.
+Its executable backend is SQLite only: foreign keys, transactional staging,
+scope-aware AST validation, and the SQLite authorizer form one tested security
+and atomicity boundary. Other core database profiles do not imply workflow
+support.
+It may read validated core or published derived datasets but never writes a SQL
+result into the source-faithful core `dataset` relation. Read-only discovery
+is provided by `openstatspec.sql.catalog_api`; historical `*_catalog`
+relations remain private compatibility state.

{openstatspec-0.1.0 → openstatspec-0.2.0}/docs/release-readiness.md

@@ -1,4 +1,4 @@
-# 0.1.0 release readiness
+# 0.2.0 release readiness
 
 This page records the expected release contract, not a publication event.
 Creating a version tag remains a separate maintainer action.
@@ -20,9 +20,12 @@ database contract.
 | PostgreSQL | `postgresql+psycopg://…` | PostgreSQL 17 and 18 service jobs |
 | MySQL | `mysql+pymysql://…` | MySQL 8.4 and 9.7 service jobs |
 | MariaDB | `mysql+pymysql://…` | MariaDB 11.4, 11.8, and 12.3 service jobs |
+| Dolt | `mysql+pymysql://…` | Exact Dolt 2.2.2 service job |
 
 Separate service matrices test the shared MySQL/MariaDB profile contract while
-retaining distinct active-server identities and version claims. This does not
+retaining distinct active-server identities and version claims. Dolt is an
+independent core profile that fails closed unless the product version is exactly
+2.2.2; the optional Transformation Workflow remains SQLite-only. This does not
 claim coverage for every server configuration. The machine-readable capability
 declaration reports both the theoretical profile boundaries and effective
 limits observed from the active connection.
@@ -49,13 +52,16 @@ machine-readable loss report with the export result.
 1. Publish the pinned `openstatspec-pyspssio==0.5.1.post2` engine distribution
    first and confirm that a clean environment can download it from PyPI. The
    main package has no fallback SPSS engine.
-2. Run `python -m pytest`.
+2. Run `python -m pytest -m "not services"`.
 3. Confirm the GitHub Actions matrix is green for PostgreSQL 17/18, MySQL
-   8.4/9.7, and MariaDB 11.4/11.8/12.3.
+   8.4/9.7, MariaDB 11.4/11.8/12.3, and exact Dolt 2.2.2.
 4. Build with `python -m build` and install the generated wheel in a clean
    environment.
 5. Confirm `openstatspec capabilities` reflects the intended support boundary.
-6. Review this document, the README, and CHANGELOG for accurate scope.
+6. Confirm the `v0.2.0` tag matches package version `0.2.0`, and that CI and
+   release fixtures use specification release `v1.0.0-rc.1` at exact commit
+   `fef0dc6f4b17ff7141dad3f49d0524c63efbfed5`.
+7. Review this document, the README, and CHANGELOG for accurate scope.
 
 The tag-triggered release workflow repeats the non-service test suite, builds
 the distributions, and installs the wheel with the exact required SPSS engine

{openstatspec-0.1.0 → openstatspec-0.2.0}/docs/sav-profile.md

@@ -13,12 +13,14 @@ unencrypted .sav and .zsav files, and writes both formats through the same imple
 
 ## Verified SQL profiles
 
-SQLite has a local reference fixture. PostgreSQL, MySQL, and MariaDB have
+SQLite has a local reference fixture. PostgreSQL, MySQL, MariaDB, and Dolt have
 separate service-backed conformance checks in GitHub Actions that import,
-validate, and export the supported fixture. MySQL 8.4 and MariaDB 11.4 exercise
-the shared profile contract; this does not claim coverage for every server
-configuration. An import that exceeds a target's strict single-table column
-limit fails before it creates a dataset.
+validate, and export the supported fixture. Dolt is an independent core profile
+pinned to exact server version 2.2.2 and detected over `mysql+pymysql` by active
+server identity; other Dolt versions and unknown MySQL-wire products fail closed.
+The core Dolt profile does not claim support for the separate Transformation
+Workflow. An import that exceeds a target's strict single-table column or row
+envelope fails before it creates a dataset.
 
 ## Fidelity contract
 
@@ -41,7 +43,7 @@ of this boundary. It records the pinned source commit and installed engine versi
 | Ordered document text | Supported through a strict type-6 dictionary bridge plus the pinned fork | Import stores normalized document rows; export creates a temporary UTF-8 SAV source and has IBM I/O copy the records into SAV or ZSAV without invalidating ZSAV dictionary offsets. |
 | Print and write formats independently | Supported as raw IBM I/O tuples | The adapter stores both tuples separately and writes them without collapsing either value. |
 | Variable sets | Supported through raw IBM I/O | The adapter stores source sets in the extension catalog and writes them through the raw dictionary setter. Invalid target definitions fail before data are written. |
-| Legacy compatible variable names | Supported through the strict dictionary bridge | The adapter stores the short-name mapping and rewrites both the fixed variable record and long-name extension before export. |
+| Legacy compatible variable names | Supported through the strict dictionary bridge | The adapter atomically rewrites and reparses type-2, subtype-13, and VLS subtype-14 names as one consistent SAV/ZSAV dictionary; malformed or duplicate subtype-14 entries fail closed. |
 | Source encoding | UTF-8 by default; legacy code page with an explicit OS locale | The caller supplies legacy_locale and the writer verifies that the emitted encoding matches the stored source encoding. Without it, export fails before output creation. |
 | Custom attributes with one value | Supported | File and variable scalar attributes round trip through the normalized attribute catalog. |
 | Custom-attribute value arrays | Supported through raw IBM I/O | The adapter represents an array as IBM SPSS `Name[1]`, `Name[2]`, … members and reconstructs the ordered array in the normalized catalog. |