xsd-former 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xsd_former-1.0.0/.github/CODEOWNERS +1 -0
- xsd_former-1.0.0/.github/workflows/lint.yml +28 -0
- xsd_former-1.0.0/.github/workflows/release.yml +49 -0
- xsd_former-1.0.0/.github/workflows/tests.yml +39 -0
- xsd_former-1.0.0/.gitignore +135 -0
- xsd_former-1.0.0/.markdownlint.json +13 -0
- xsd_former-1.0.0/.markdownlintignore +1 -0
- xsd_former-1.0.0/.pre-commit-config.yaml +51 -0
- xsd_former-1.0.0/.yamlfmt +10 -0
- xsd_former-1.0.0/CLAUDE.md +74 -0
- xsd_former-1.0.0/CONTEXT.md +115 -0
- xsd_former-1.0.0/LICENSE +21 -0
- xsd_former-1.0.0/PKG-INFO +121 -0
- xsd_former-1.0.0/README.md +89 -0
- xsd_former-1.0.0/clinvar_transforms.yaml +213 -0
- xsd_former-1.0.0/docs/adr/0001-typespec-output-format.md +197 -0
- xsd_former-1.0.0/docs/adr/0002-pydantic-codegen-and-proto-hub.md +249 -0
- xsd_former-1.0.0/docs/releasing.md +30 -0
- xsd_former-1.0.0/docs/style/general.md +30 -0
- xsd_former-1.0.0/docs/style/python.md +305 -0
- xsd_former-1.0.0/examples/clinvar_to_pbtxt.py +89 -0
- xsd_former-1.0.0/examples/pmid_31427284.pbtxt +2260 -0
- xsd_former-1.0.0/examples/pmid_33073003.pbtxt +504 -0
- xsd_former-1.0.0/examples/pubmed_to_pbtxt.py +93 -0
- xsd_former-1.0.0/pubmed_transforms.yaml +34 -0
- xsd_former-1.0.0/pyproject.toml +133 -0
- xsd_former-1.0.0/src/xsdformer/__init__.py +0 -0
- xsd_former-1.0.0/src/xsdformer/build.py +175 -0
- xsd_former-1.0.0/src/xsdformer/dtd/__init__.py +0 -0
- xsd_former-1.0.0/src/xsdformer/dtd/dtd.py +384 -0
- xsd_former-1.0.0/src/xsdformer/generator.py +32 -0
- xsd_former-1.0.0/src/xsdformer/jsonschema/__init__.py +0 -0
- xsd_former-1.0.0/src/xsdformer/jsonschema/generator.py +573 -0
- xsd_former-1.0.0/src/xsdformer/protobuf/__init__.py +0 -0
- xsd_former-1.0.0/src/xsdformer/protobuf/generator.py +154 -0
- xsd_former-1.0.0/src/xsdformer/py/__init__.py +0 -0
- xsd_former-1.0.0/src/xsdformer/py/xml_converter.py +629 -0
- xsd_former-1.0.0/src/xsdformer/py.typed +0 -0
- xsd_former-1.0.0/src/xsdformer/pydantic/__init__.py +0 -0
- xsd_former-1.0.0/src/xsdformer/pydantic/_naming.py +37 -0
- xsd_former-1.0.0/src/xsdformer/pydantic/converter.py +287 -0
- xsd_former-1.0.0/src/xsdformer/pydantic/generator.py +281 -0
- xsd_former-1.0.0/src/xsdformer/tool.py +401 -0
- xsd_former-1.0.0/src/xsdformer/transforms.py +460 -0
- xsd_former-1.0.0/src/xsdformer/typespec/__init__.py +0 -0
- xsd_former-1.0.0/src/xsdformer/typespec/generator.py +349 -0
- xsd_former-1.0.0/src/xsdformer/xsd/__init__.py +0 -0
- xsd_former-1.0.0/src/xsdformer/xsd/text.py +119 -0
- xsd_former-1.0.0/src/xsdformer/xsd/xsd.py +856 -0
- xsd_former-1.0.0/tests/__init__.py +0 -0
- xsd_former-1.0.0/tests/xsdformer/__init__.py +0 -0
- xsd_former-1.0.0/tests/xsdformer/conftest.py +231 -0
- xsd_former-1.0.0/tests/xsdformer/dtd/__init__.py +0 -0
- xsd_former-1.0.0/tests/xsdformer/dtd/test_dtd.py +329 -0
- xsd_former-1.0.0/tests/xsdformer/jsonschema/__init__.py +0 -0
- xsd_former-1.0.0/tests/xsdformer/jsonschema/test_generator.py +493 -0
- xsd_former-1.0.0/tests/xsdformer/protobuf/__init__.py +0 -0
- xsd_former-1.0.0/tests/xsdformer/protobuf/test_generator.py +102 -0
- xsd_former-1.0.0/tests/xsdformer/py/__init__.py +0 -0
- xsd_former-1.0.0/tests/xsdformer/py/test_xml_converter.py +506 -0
- xsd_former-1.0.0/tests/xsdformer/pydantic/__init__.py +0 -0
- xsd_former-1.0.0/tests/xsdformer/pydantic/_equivalence.py +179 -0
- xsd_former-1.0.0/tests/xsdformer/pydantic/records/pmid_31427284.xml +28 -0
- xsd_former-1.0.0/tests/xsdformer/pydantic/records/pmid_33073003.xml +4 -0
- xsd_former-1.0.0/tests/xsdformer/pydantic/test_converter.py +330 -0
- xsd_former-1.0.0/tests/xsdformer/pydantic/test_equivalence.py +152 -0
- xsd_former-1.0.0/tests/xsdformer/pydantic/test_generator.py +442 -0
- xsd_former-1.0.0/tests/xsdformer/pydantic/test_roundtrip.py +75 -0
- xsd_former-1.0.0/tests/xsdformer/test_build.py +212 -0
- xsd_former-1.0.0/tests/xsdformer/test_transforms.py +789 -0
- xsd_former-1.0.0/tests/xsdformer/typespec/__init__.py +0 -0
- xsd_former-1.0.0/tests/xsdformer/typespec/_tsp.py +172 -0
- xsd_former-1.0.0/tests/xsdformer/typespec/schemas/ClinVar_VCV.xsd +2783 -0
- xsd_former-1.0.0/tests/xsdformer/typespec/schemas/pubmed.dtd +475 -0
- xsd_former-1.0.0/tests/xsdformer/typespec/test_generator.py +420 -0
- xsd_former-1.0.0/tests/xsdformer/typespec/test_roundtrip.py +213 -0
- xsd_former-1.0.0/tests/xsdformer/typespec/tsp_project/.gitignore +2 -0
- xsd_former-1.0.0/tests/xsdformer/typespec/tsp_project/package.json +11 -0
- xsd_former-1.0.0/tests/xsdformer/xsd/__init__.py +0 -0
- xsd_former-1.0.0/tests/xsdformer/xsd/test_text.py +64 -0
- xsd_former-1.0.0/tests/xsdformer/xsd/test_xsd.py +442 -0
- xsd_former-1.0.0/uv.lock +1157 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
* @folded
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
name: lint
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
pull_request:
|
|
5
|
+
types: [opened, ready_for_review, synchronize]
|
|
6
|
+
push:
|
|
7
|
+
branches: [main]
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
pre-commit:
|
|
11
|
+
# Runs every hook in .pre-commit-config.yaml — the whole static-check gate.
|
|
12
|
+
# Skip draft PRs.
|
|
13
|
+
if: github.event_name != 'pull_request' || !github.event.pull_request.draft
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
permissions:
|
|
16
|
+
contents: read
|
|
17
|
+
steps:
|
|
18
|
+
- uses: actions/checkout@v6
|
|
19
|
+
- uses: astral-sh/setup-uv@v8.2.0
|
|
20
|
+
with:
|
|
21
|
+
enable-cache: true
|
|
22
|
+
# Pre-sync the lint env so the pyright hook's `uv run` is a no-op.
|
|
23
|
+
- run: uv sync --locked --group lint --python 3.13
|
|
24
|
+
- uses: actions/cache@v6
|
|
25
|
+
with:
|
|
26
|
+
path: ~/.cache/pre-commit
|
|
27
|
+
key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}
|
|
28
|
+
- run: uv run pre-commit run --all-files --show-diff-on-failure
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
name: release
|
|
2
|
+
|
|
3
|
+
# Publishes to PyPI via Trusted Publishing (OIDC) — no API token. The PyPI
|
|
4
|
+
# project's trusted publisher must reference this repo, this workflow filename,
|
|
5
|
+
# and the `pypi` environment. Triggered on a published GitHub Release; the tag
|
|
6
|
+
# must match the package version (guarded below).
|
|
7
|
+
on:
|
|
8
|
+
release:
|
|
9
|
+
types: [published]
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
build:
|
|
13
|
+
runs-on: ubuntu-latest
|
|
14
|
+
permissions:
|
|
15
|
+
contents: read
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v6
|
|
18
|
+
- uses: astral-sh/setup-uv@v8.2.0
|
|
19
|
+
with:
|
|
20
|
+
enable-cache: true
|
|
21
|
+
# Publishing a version is irreversible, so fail loudly if the release tag
|
|
22
|
+
# (vX.Y.Z) doesn't match the version in pyproject.toml.
|
|
23
|
+
- name: Check tag matches package version
|
|
24
|
+
run: |
|
|
25
|
+
tag="${GITHUB_REF_NAME#v}"
|
|
26
|
+
pkg="$(uv version --short)"
|
|
27
|
+
if [ "$tag" != "$pkg" ]; then
|
|
28
|
+
echo "Release tag '$GITHUB_REF_NAME' does not match package version '$pkg'"
|
|
29
|
+
exit 1
|
|
30
|
+
fi
|
|
31
|
+
- run: uv build
|
|
32
|
+
- uses: actions/upload-artifact@v4
|
|
33
|
+
with:
|
|
34
|
+
name: dist
|
|
35
|
+
path: dist/
|
|
36
|
+
|
|
37
|
+
publish:
|
|
38
|
+
needs: build
|
|
39
|
+
runs-on: ubuntu-latest
|
|
40
|
+
# The trusted-publisher and OIDC scope both key off this environment name.
|
|
41
|
+
environment: pypi
|
|
42
|
+
permissions:
|
|
43
|
+
id-token: write # mint the OIDC token PyPI exchanges for an upload token
|
|
44
|
+
steps:
|
|
45
|
+
- uses: actions/download-artifact@v4
|
|
46
|
+
with:
|
|
47
|
+
name: dist
|
|
48
|
+
path: dist/
|
|
49
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
name: tests
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
pull_request:
|
|
5
|
+
types: [opened, ready_for_review, synchronize]
|
|
6
|
+
push:
|
|
7
|
+
branches: [main]
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
pytest:
|
|
11
|
+
# Skip draft PRs.
|
|
12
|
+
if: github.event_name != 'pull_request' || !github.event.pull_request.draft
|
|
13
|
+
runs-on: ubuntu-latest
|
|
14
|
+
strategy:
|
|
15
|
+
fail-fast: false
|
|
16
|
+
matrix:
|
|
17
|
+
python-version: ['3.11', '3.12', '3.13']
|
|
18
|
+
permissions:
|
|
19
|
+
contents: read
|
|
20
|
+
steps:
|
|
21
|
+
- uses: actions/checkout@v6
|
|
22
|
+
- uses: astral-sh/setup-uv@v8.2.0
|
|
23
|
+
with:
|
|
24
|
+
enable-cache: true
|
|
25
|
+
- uses: actions/setup-node@v4
|
|
26
|
+
with:
|
|
27
|
+
# @typespec/compiler >=1.13 requires Node >=22 (imports `glob` from
|
|
28
|
+
# `fs/promises`, added in Node 22). On Node 20 the toolchain crashes at
|
|
29
|
+
# runtime instead of skipping, failing the gated tsp tests.
|
|
30
|
+
node-version: '22'
|
|
31
|
+
# Enables the skipif-gated Node tests: the xsd->tsp->proto round-trip
|
|
32
|
+
# (`_tsp.tsp_available()`) and the pydantic equivalence gate
|
|
33
|
+
# (`_tsp.json_schema_available()`, via `@typespec/json-schema`). Without it
|
|
34
|
+
# both skip.
|
|
35
|
+
- name: Install TypeSpec toolchain
|
|
36
|
+
working-directory: tests/xsdformer/typespec/tsp_project
|
|
37
|
+
run: npm install
|
|
38
|
+
- run: uv sync --locked --group test --python ${{ matrix.python-version }}
|
|
39
|
+
- run: uv run pytest -q
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
# Python codebase default gitignore (github template)
|
|
2
|
+
|
|
3
|
+
# Byte-compiled / optimized / DLL files
|
|
4
|
+
__pycache__/
|
|
5
|
+
*.py[cod]
|
|
6
|
+
*$py.class
|
|
7
|
+
|
|
8
|
+
# local irrelevant files
|
|
9
|
+
.idea
|
|
10
|
+
.DS_Store
|
|
11
|
+
|
|
12
|
+
# C extensions
|
|
13
|
+
*.so
|
|
14
|
+
|
|
15
|
+
# Distribution / packaging
|
|
16
|
+
.Python
|
|
17
|
+
build/
|
|
18
|
+
develop-eggs/
|
|
19
|
+
dist/
|
|
20
|
+
downloads/
|
|
21
|
+
eggs/
|
|
22
|
+
.eggs/
|
|
23
|
+
lib/
|
|
24
|
+
lib64/
|
|
25
|
+
parts/
|
|
26
|
+
sdist/
|
|
27
|
+
var/
|
|
28
|
+
wheels/
|
|
29
|
+
pip-wheel-metadata/
|
|
30
|
+
share/python-wheels/
|
|
31
|
+
*.egg-info/
|
|
32
|
+
.installed.cfg
|
|
33
|
+
*.egg
|
|
34
|
+
MANIFEST
|
|
35
|
+
|
|
36
|
+
# PyInstaller
|
|
37
|
+
# Usually these files are written by a python script from a template
|
|
38
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
39
|
+
*.manifest
|
|
40
|
+
*.spec
|
|
41
|
+
|
|
42
|
+
# Installer logs
|
|
43
|
+
pip-log.txt
|
|
44
|
+
pip-delete-this-directory.txt
|
|
45
|
+
|
|
46
|
+
# Unit test / coverage reports
|
|
47
|
+
htmlcov/
|
|
48
|
+
.tox/
|
|
49
|
+
.nox/
|
|
50
|
+
.coverage
|
|
51
|
+
.coverage.*
|
|
52
|
+
.cache
|
|
53
|
+
nosetests.xml
|
|
54
|
+
coverage.xml
|
|
55
|
+
*.cover
|
|
56
|
+
*.py,cover
|
|
57
|
+
.hypothesis/
|
|
58
|
+
.pytest_cache/
|
|
59
|
+
|
|
60
|
+
# Translations
|
|
61
|
+
*.mo
|
|
62
|
+
*.pot
|
|
63
|
+
|
|
64
|
+
# Django stuff:
|
|
65
|
+
*.log
|
|
66
|
+
local_settings.py
|
|
67
|
+
db.sqlite3
|
|
68
|
+
db.sqlite3-journal
|
|
69
|
+
|
|
70
|
+
# Flask stuff:
|
|
71
|
+
instance/
|
|
72
|
+
.webassets-cache
|
|
73
|
+
|
|
74
|
+
# Scrapy stuff:
|
|
75
|
+
.scrapy
|
|
76
|
+
|
|
77
|
+
# Sphinx documentation
|
|
78
|
+
docs/_build/
|
|
79
|
+
|
|
80
|
+
# PyBuilder
|
|
81
|
+
target/
|
|
82
|
+
|
|
83
|
+
# Jupyter Notebook
|
|
84
|
+
.ipynb_checkpoints
|
|
85
|
+
|
|
86
|
+
# IPython
|
|
87
|
+
profile_default/
|
|
88
|
+
ipython_config.py
|
|
89
|
+
|
|
90
|
+
# pyenv
|
|
91
|
+
.python-version
|
|
92
|
+
|
|
93
|
+
# pipenv
|
|
94
|
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
|
95
|
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
|
96
|
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
|
97
|
+
# install all needed dependencies.
|
|
98
|
+
#Pipfile.lock
|
|
99
|
+
|
|
100
|
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
|
|
101
|
+
__pypackages__/
|
|
102
|
+
|
|
103
|
+
# Celery stuff
|
|
104
|
+
celerybeat-schedule
|
|
105
|
+
celerybeat.pid
|
|
106
|
+
|
|
107
|
+
# SageMath parsed files
|
|
108
|
+
*.sage.py
|
|
109
|
+
|
|
110
|
+
# Environments
|
|
111
|
+
.env
|
|
112
|
+
.venv
|
|
113
|
+
env/
|
|
114
|
+
venv/
|
|
115
|
+
ENV/
|
|
116
|
+
env.bak/
|
|
117
|
+
venv.bak/
|
|
118
|
+
|
|
119
|
+
# Spyder project settings
|
|
120
|
+
.spyderproject
|
|
121
|
+
.spyproject
|
|
122
|
+
|
|
123
|
+
# Rope project settings
|
|
124
|
+
.ropeproject
|
|
125
|
+
|
|
126
|
+
# mkdocs documentation
|
|
127
|
+
/site
|
|
128
|
+
|
|
129
|
+
# mypy
|
|
130
|
+
.mypy_cache/
|
|
131
|
+
.dmypy.json
|
|
132
|
+
dmypy.json
|
|
133
|
+
|
|
134
|
+
# Pyre type checker
|
|
135
|
+
.pyre/
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
// https://github.com/DavidAnson/markdownlint#optionsconfig
|
|
2
|
+
// Disabling some rules here as we find them too restrictive.
|
|
3
|
+
{
|
|
4
|
+
"default": true, // Include all rules by defauls
|
|
5
|
+
"line-length": false, // To allow working with soft wraps in editors
|
|
6
|
+
"no-inline-html": { // Sometimes we need to use <img> html tags in GitHub markdown,
|
|
7
|
+
// as it doesn't allow setting the image size with markdown tags
|
|
8
|
+
"allowed_elements": ["details", "img"]
|
|
9
|
+
},
|
|
10
|
+
"ul-indent": false, // To allow indenting the whole list to distinguish it visually
|
|
11
|
+
"no-multiple-blanks": false // To allow multiple blank lines between a header and
|
|
12
|
+
// the previous paragraph
|
|
13
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
CLAUDE.md
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
repos:
|
|
2
|
+
- repo: https://github.com/pre-commit/pre-commit-hooks
|
|
3
|
+
rev: v6.0.0
|
|
4
|
+
hooks:
|
|
5
|
+
- id: check-yaml
|
|
6
|
+
exclude: '\.*conda/.*'
|
|
7
|
+
- id: end-of-file-fixer
|
|
8
|
+
# Vendored schemas (NLM DTD, ClinVar XSD) and real efetch record fixtures
|
|
9
|
+
# are kept byte-faithful to their upstream form; don't normalize them.
|
|
10
|
+
exclude: '^tests/xsdformer/(typespec/schemas|pydantic/records)/'
|
|
11
|
+
- id: trailing-whitespace
|
|
12
|
+
exclude: '\.txt$|\.tsv$|^tests/xsdformer/(typespec/schemas|pydantic/records)/'
|
|
13
|
+
- id: check-case-conflict
|
|
14
|
+
- id: check-merge-conflict
|
|
15
|
+
- id: detect-private-key
|
|
16
|
+
- id: debug-statements
|
|
17
|
+
- id: check-added-large-files
|
|
18
|
+
|
|
19
|
+
- repo: https://github.com/igorshubovych/markdownlint-cli
|
|
20
|
+
rev: v0.45.0
|
|
21
|
+
hooks:
|
|
22
|
+
- id: markdownlint
|
|
23
|
+
|
|
24
|
+
- repo: https://github.com/populationgenomics/pre-commits
|
|
25
|
+
rev: "e37928f761f17d54aca5cedf93848b40ec7cff26"
|
|
26
|
+
hooks:
|
|
27
|
+
- id: cpg-id-checker
|
|
28
|
+
|
|
29
|
+
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
30
|
+
rev: v0.14.1
|
|
31
|
+
hooks:
|
|
32
|
+
- id: ruff
|
|
33
|
+
- id: ruff-format
|
|
34
|
+
|
|
35
|
+
# pyright needs the project deps to resolve imports, so it runs in the lint env
|
|
36
|
+
# via `uv run`; `--group lint` provisions it on demand. Whole-project (include
|
|
37
|
+
# set in pyproject), so pass_filenames: false; fires on any staged .py.
|
|
38
|
+
- repo: local
|
|
39
|
+
hooks:
|
|
40
|
+
- id: pyright
|
|
41
|
+
name: pyright
|
|
42
|
+
entry: uv run --group lint pyright
|
|
43
|
+
language: system
|
|
44
|
+
files: \.py$
|
|
45
|
+
pass_filenames: false
|
|
46
|
+
|
|
47
|
+
# YAML formatter (ruff can't format YAML).
|
|
48
|
+
- repo: https://github.com/google/yamlfmt
|
|
49
|
+
rev: v0.21.0
|
|
50
|
+
hooks:
|
|
51
|
+
- id: yamlfmt
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# YAML formatter config; the YAML analogue of ruff-format / biome.
|
|
2
|
+
# retain_line_breaks_single: yamlfmt can't selectively strip blank lines
|
|
3
|
+
# between block-sequence items, so the no-blank-line-between-steps layout in
|
|
4
|
+
# .github/workflows is kept by hand — this setting only stops yamlfmt from
|
|
5
|
+
# reintroducing or multiplying blank lines elsewhere.
|
|
6
|
+
formatter:
|
|
7
|
+
type: basic
|
|
8
|
+
retain_line_breaks_single: true
|
|
9
|
+
trim_trailing_whitespace: true
|
|
10
|
+
eof_newline: true
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# xsdformer development notes
|
|
2
|
+
|
|
3
|
+
xsdformer converts an XSD or DTD schema into Protobuf, JSON Schema, TypeSpec,
|
|
4
|
+
Pydantic models, and Python XML-to-protobuf converters. See
|
|
5
|
+
[`CONTEXT.md`](CONTEXT.md) for the domain model and language, and
|
|
6
|
+
[`README.md`](README.md) for usage.
|
|
7
|
+
|
|
8
|
+
## Working norms
|
|
9
|
+
|
|
10
|
+
Operating directives for Claude (and any agent) in this repo; they counteract default
|
|
11
|
+
model dispositions.
|
|
12
|
+
|
|
13
|
+
- **Resist the minimal-diff reflex.** Don't reach for the smallest change that hides the
|
|
14
|
+
symptom (special-casing, papering over root causes). Aim for the correct fix at the
|
|
15
|
+
right complexity level — not the smallest, not gold-plated.
|
|
16
|
+
- **Fail loudly and early.** Raise on a missing expected input or precondition; never fall
|
|
17
|
+
back to a default/placeholder to limp along. A placeholder is an explicit caller input,
|
|
18
|
+
never a code default.
|
|
19
|
+
- **Push back; don't just comply.** When a design, name, or approach seems worse —
|
|
20
|
+
including a shortcut you're asked to take — say so with reasoning, unprompted. The
|
|
21
|
+
author owns the final call.
|
|
22
|
+
- **Offer better alternatives with trade-offs.** When a materially better approach than
|
|
23
|
+
the proposed one exists, present it and the trade-offs — don't just execute the ask.
|
|
24
|
+
- **Investigate before producing.** Read the code and verify constraints first. Don't
|
|
25
|
+
treat a training-pattern convention as load-bearing unchecked; don't speculate about
|
|
26
|
+
what you can read.
|
|
27
|
+
- **Explain non-obvious changes first.** For a change whose rationale isn't self-evident,
|
|
28
|
+
give the why before showing or applying the diff.
|
|
29
|
+
- **Ask when unsure** rather than assume intent.
|
|
30
|
+
- **No intensifiers or emphasis filler.** Drop words and phrases that add emphasis but no
|
|
31
|
+
information — "that's the key", "crucially", "importantly", "the key insight", "it's
|
|
32
|
+
worth noting". State the point plainly. Applies to all prose: chat replies, PR/review
|
|
33
|
+
comments, commit messages, and docs.
|
|
34
|
+
|
|
35
|
+
## Commands
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
uv sync --group test --group lint # install dev dependencies
|
|
39
|
+
uv run pytest # run the test suite
|
|
40
|
+
uv run pytest tests/xsdformer/xsd/test_xsd.py::test_name -x # single test
|
|
41
|
+
uv run pre-commit run --all-files # the full static-check gate (see below)
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Some tests are gated on a TypeSpec (Node) toolchain and skip when it is absent;
|
|
45
|
+
CI installs it. See [`CONTEXT.md`](CONTEXT.md) for what each module does.
|
|
46
|
+
|
|
47
|
+
## Code style
|
|
48
|
+
|
|
49
|
+
@docs/style/general.md
|
|
50
|
+
@docs/style/python.md
|
|
51
|
+
|
|
52
|
+
## Docs
|
|
53
|
+
|
|
54
|
+
The primary audience for docs is a model reading them as context; humans second. Be
|
|
55
|
+
terse: state each decision, mechanism, and rationale once — no rhetorical emphasis, no
|
|
56
|
+
persuasion, no recaps. Every token written is re-paid on every future read. Design
|
|
57
|
+
decisions with lasting consequences get an ADR under [`docs/adr/`](docs/adr).
|
|
58
|
+
|
|
59
|
+
## Committing
|
|
60
|
+
|
|
61
|
+
- **Stage explicit paths**, not `git add -A` / `.`.
|
|
62
|
+
- **Pre-commit is the full static-check gate** (`.pre-commit-config.yaml`): lint, format,
|
|
63
|
+
hygiene, and pyright. CI runs the same hooks via `pre-commit run --all-files`, so the
|
|
64
|
+
two can't drift. Ensure hooks are installed (`pre-commit install`) — if not, install or
|
|
65
|
+
ask the author; never bypass with `--no-verify`.
|
|
66
|
+
- **Correct a pushed branch with a new commit on top**, not amend + force-push. PRs
|
|
67
|
+
squash-merge, so `main` history stays linear regardless and intermediate fixups vanish
|
|
68
|
+
on merge. Reserve force-push for rebasing a branch onto `main`.
|
|
69
|
+
|
|
70
|
+
## CI and review
|
|
71
|
+
|
|
72
|
+
- **Pin third-party GitHub Actions to the latest stable release**: the moving major tag
|
|
73
|
+
(`@v3`) where the action publishes one, else the exact latest version (`@v8.2.0`). Verify
|
|
74
|
+
against the action's releases when adding or bumping one.
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
# xsdformer
|
|
2
|
+
|
|
3
|
+
xsdformer converts an XML schema — an **XSD** or a **DTD** — into code that
|
|
4
|
+
describes and processes the same data in other ecosystems: Protobuf
|
|
5
|
+
definitions, JSON Schema, TypeSpec, Pydantic models, and Python code that
|
|
6
|
+
parses conforming XML into Protobuf messages.
|
|
7
|
+
|
|
8
|
+
It supports enough of XSD/DTD to convert the
|
|
9
|
+
[ClinVar](https://www.ncbi.nlm.nih.gov/clinvar/) and
|
|
10
|
+
[BioC](http://bioc.sourceforge.net/) schemas. Full coverage of the XSD
|
|
11
|
+
specification is a non-goal; the parser raises on constructs it does not model
|
|
12
|
+
rather than emitting a silent approximation.
|
|
13
|
+
|
|
14
|
+
## Pipeline
|
|
15
|
+
|
|
16
|
+
```text
|
|
17
|
+
schema (XSD | DTD) → parse → IR (TypeDefinition graph) → transform → generate → output
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
A front end parses the schema into an intermediate representation (IR). The IR
|
|
21
|
+
is a topologically sorted graph of type definitions, independent of both the
|
|
22
|
+
input schema language and the output target. Each back end (generator) walks
|
|
23
|
+
that IR to emit one output format. Adding an input or output format touches one
|
|
24
|
+
front end or one generator, not the pipeline.
|
|
25
|
+
|
|
26
|
+
## The IR — language
|
|
27
|
+
|
|
28
|
+
The IR lives in [`xsd/xsd.py`](src/xsdformer/xsd/xsd.py). These are the domain
|
|
29
|
+
terms; use them exactly.
|
|
30
|
+
|
|
31
|
+
**TypeDefinition**:
|
|
32
|
+
A named type in the schema. The base for the three concrete kinds a generator
|
|
33
|
+
must handle:
|
|
34
|
+
|
|
35
|
+
- **Message** — a structured type (an XSD complex type): a set of fields.
|
|
36
|
+
- **Enumeration** — a closed set of named values.
|
|
37
|
+
- **MapType** — a type that models a map (key/value) rather than a record.
|
|
38
|
+
|
|
39
|
+
**FieldDefinition**:
|
|
40
|
+
A member of a `Message`. The base for leaf **Field** kinds and **container**
|
|
41
|
+
kinds:
|
|
42
|
+
|
|
43
|
+
- **Field** — a leaf carrying a value. Subtypes: **Elem** (a child element),
|
|
44
|
+
**Attr** (an XML attribute), **ValueElem** (an element's text content).
|
|
45
|
+
- **Seq** — an ordered group of fields.
|
|
46
|
+
- **Choice** — a mutually-exclusive group of fields.
|
|
47
|
+
|
|
48
|
+
**computed_occurs**:
|
|
49
|
+
A field's effective `(min, max)` occurrence after multiplying through every
|
|
50
|
+
enclosing container. A field that is optional inside a repeated `Seq` is
|
|
51
|
+
repeated overall; `computed_occurs` is that resolved multiplicity.
|
|
52
|
+
|
|
53
|
+
**enclosing_type**:
|
|
54
|
+
Links a nested (enclosed) type to the `Message` it is defined inside. A type
|
|
55
|
+
with no `enclosing_type` is top-level.
|
|
56
|
+
|
|
57
|
+
**path**:
|
|
58
|
+
The tuple of names from the root schema type down to a given type — its fully
|
|
59
|
+
qualified position in the type graph.
|
|
60
|
+
|
|
61
|
+
## Front ends (parsers)
|
|
62
|
+
|
|
63
|
+
- [`xsd/xsd.py`](src/xsdformer/xsd/xsd.py) — parses XSD via the `xmlschema`
|
|
64
|
+
library, topologically sorts types by dependency, and produces the IR.
|
|
65
|
+
- [`dtd/dtd.py`](src/xsdformer/dtd/dtd.py) — parses DTD into the same IR.
|
|
66
|
+
- [`transforms.py`](src/xsdformer/transforms.py) — optional IR-level
|
|
67
|
+
simplifications (dropping fields, inlining single-field wrapper types,
|
|
68
|
+
collapsing a type to a plain string) applied before generation, driven by a
|
|
69
|
+
YAML transform spec (see `clinvar_transforms.yaml`, `pubmed_transforms.yaml`).
|
|
70
|
+
|
|
71
|
+
## Back ends (generators)
|
|
72
|
+
|
|
73
|
+
[`generator.py`](src/xsdformer/generator.py) defines the `IGenerator` protocol
|
|
74
|
+
and `generate_with()`, which drives any generator over the IR and skips enclosed
|
|
75
|
+
types at the top level (the enclosing type emits them). Each generator
|
|
76
|
+
implements `IGenerator`:
|
|
77
|
+
|
|
78
|
+
- [`protobuf/generator.py`](src/xsdformer/protobuf/generator.py) — emits
|
|
79
|
+
`.proto` syntax.
|
|
80
|
+
- [`py/xml_converter.py`](src/xsdformer/py/xml_converter.py) — emits Python code
|
|
81
|
+
that converts parsed lxml elements into Protobuf message instances. Runtime
|
|
82
|
+
helpers (e.g. `_xml_bool`, `_consume`) are embedded into the generated output
|
|
83
|
+
via `inspect.getsource`.
|
|
84
|
+
- [`jsonschema/generator.py`](src/xsdformer/jsonschema/generator.py) — compiles
|
|
85
|
+
proto (from the IR or an existing `.proto` file) via `grpc_tools.protoc` into
|
|
86
|
+
a `FileDescriptorSet`, then walks the Protobuf descriptors to emit JSON
|
|
87
|
+
Schema.
|
|
88
|
+
- [`typespec/generator.py`](src/xsdformer/typespec/generator.py) — emits
|
|
89
|
+
TypeSpec (`.tsp`). See [ADR 0001](docs/adr/0001-typespec-output-format.md).
|
|
90
|
+
- [`pydantic/`](src/xsdformer/pydantic) — emits Pydantic models
|
|
91
|
+
(`generator.py`) and a converter (`converter.py`), sharing naming logic in
|
|
92
|
+
`_naming.py`. See [ADR 0002](docs/adr/0002-pydantic-codegen-and-proto-hub.md).
|
|
93
|
+
|
|
94
|
+
## CLI
|
|
95
|
+
|
|
96
|
+
[`tool.py`](src/xsdformer/tool.py) exposes a Click group with three commands:
|
|
97
|
+
|
|
98
|
+
- `xsdformer xsd <file>` — XSD front end; emit any combination of proto, Python
|
|
99
|
+
converter, JSON Schema, TypeSpec, and Pydantic outputs (one `--*-out` flag
|
|
100
|
+
each).
|
|
101
|
+
- `xsdformer dtd <file>` — DTD front end; same output flags.
|
|
102
|
+
- `xsdformer proto <file> <namespace>` — Proto-to-JSON-Schema only, bypassing
|
|
103
|
+
the XSD/DTD front end.
|
|
104
|
+
|
|
105
|
+
## Package builder
|
|
106
|
+
|
|
107
|
+
[`build.py`](src/xsdformer/build.py) assembles a standalone, pip-installable
|
|
108
|
+
Python package from a schema: generated proto (compiled to `_pb2.py`), the XML
|
|
109
|
+
converter, and Pydantic models, wrapped in a generated `pyproject.toml`.
|
|
110
|
+
|
|
111
|
+
## Text utilities
|
|
112
|
+
|
|
113
|
+
[`xsd/text.py`](src/xsdformer/xsd/text.py) — `snake_case`, `pascal_case`, and
|
|
114
|
+
`keep` (wraps a string in `_Exact` to bypass case conversion). All identifier
|
|
115
|
+
naming flows through these.
|
xsd_former-1.0.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Centre for Population Genomics
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: xsd-former
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Generate Protobuf definitions, JSON Schema, and Python XML-to-protobuf converters from XSD and DTD schemas.
|
|
5
|
+
Project-URL: Repository, https://github.com/populationgenomics/xsd-former
|
|
6
|
+
Project-URL: Issues, https://github.com/populationgenomics/xsd-former/issues
|
|
7
|
+
Author: Toby Sargeant
|
|
8
|
+
License: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: bioc,clinvar,code-generation,dtd,json-schema,protobuf,typespec,xml,xsd
|
|
11
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
|
|
21
|
+
Classifier: Topic :: Software Development :: Code Generators
|
|
22
|
+
Classifier: Typing :: Typed
|
|
23
|
+
Requires-Python: >=3.11
|
|
24
|
+
Requires-Dist: click>=8.1
|
|
25
|
+
Requires-Dist: grpcio-tools>=1.60
|
|
26
|
+
Requires-Dist: jinja2>=3.1.6
|
|
27
|
+
Requires-Dist: lxml>=5.0
|
|
28
|
+
Requires-Dist: protobuf>=6.32.1
|
|
29
|
+
Requires-Dist: pyyaml>=6.0.3
|
|
30
|
+
Requires-Dist: xmlschema>=4.0
|
|
31
|
+
Description-Content-Type: text/markdown
|
|
32
|
+
|
|
33
|
+
# XSDFormer
|
|
34
|
+
|
|
35
|
+
XSDFormer deforms [XML schema definitions
|
|
36
|
+
(XSDs)](https://www.w3.org/XML/Schema) into [Protobuf
|
|
37
|
+
definitions](https://protobuf.dev/). It also generates Python code
|
|
38
|
+
to convert parsed XML into the corresponding Protobuf representation.
|
|
39
|
+
|
|
40
|
+
It supports enough of the XSD specification to convert the
|
|
41
|
+
[ClinVar](https://www.ncbi.nlm.nih.gov/clinvar/) and
|
|
42
|
+
[BioC](http://bioc.sourceforge.net/) schemas; full support for all
|
|
43
|
+
XSD features is a non-goal.
|
|
44
|
+
|
|
45
|
+
## Why convert from XSD to Protobuf?
|
|
46
|
+
|
|
47
|
+
While XML and XSDs (or alternatively JSON and JSON schemas) are
|
|
48
|
+
powerful for defining complex, human-readable data structures, they
|
|
49
|
+
have some drawbacks, especially in high-performance applications
|
|
50
|
+
or when dealing with large datasets.
|
|
51
|
+
|
|
52
|
+
### Performance and Size
|
|
53
|
+
|
|
54
|
+
* **Parsing Speed:** XML is a text-based format and can be slow to
|
|
55
|
+
parse. Protobuf is a binary format that is designed for speed and
|
|
56
|
+
efficiency. Converting XML data to Protobuf can result in significantly
|
|
57
|
+
faster parsing times.
|
|
58
|
+
* **Storage Space:** XML is verbose, with opening and closing tags
|
|
59
|
+
that add to the file size. Protobuf's binary format is much more
|
|
60
|
+
compact, leading to smaller file sizes. This is a major advantage
|
|
61
|
+
for storing large datasets or for transmitting data over a network.
|
|
62
|
+
Effectively compressing XML requires a schema-specific dictionary,
|
|
63
|
+
or compressing multiple records together so that the dictionary of
|
|
64
|
+
tag/key names can be reused. However this limits the possibility
|
|
65
|
+
for random access enabled by compressing records individually.
|
|
66
|
+
|
|
67
|
+
### Developer Experience
|
|
68
|
+
|
|
69
|
+
* **Generated Code:** Protobuf compilers generate code in many
|
|
70
|
+
languages, providing a simple and consistent way to work with the
|
|
71
|
+
data.
|
|
72
|
+
* **Type Safety:** The Protobuf schema provides strong typing, which
|
|
73
|
+
can help to prevent bugs. Parsing the wire format requires the
|
|
74
|
+
protobuf definition, meaning that the data is tied to its typed
|
|
75
|
+
representation. Conversely XML representations are only optionally
|
|
76
|
+
validated by a schema, and so by default type information is lost
|
|
77
|
+
during parsing (everything is treated as text).
|
|
78
|
+
|
|
79
|
+
By converting XSDs to Protobuf definitions, `xsd-former` allows
|
|
80
|
+
developers to take advantage of the benefits of Protobuf while still
|
|
81
|
+
working with data that is originally defined in an XML schema.
|
|
82
|
+
|
|
83
|
+
## Output formats
|
|
84
|
+
|
|
85
|
+
From a single XSD (or DTD) source, XSDFormer can emit:
|
|
86
|
+
|
|
87
|
+
* **Protobuf** (`.proto`) — `--proto-out`.
|
|
88
|
+
* **Python XML→protobuf converters** — `--py-out`/`--py-module`.
|
|
89
|
+
* **JSON Schema** — `--json-schema-out` (from the XSD, or directly from
|
|
90
|
+
a `.proto` via the `proto` subcommand).
|
|
91
|
+
* **TypeSpec** (`.tsp`) — `--typespec-out` (see below).
|
|
92
|
+
|
|
93
|
+
## TypeSpec output
|
|
94
|
+
|
|
95
|
+
[TypeSpec](https://typespec.io/) is a compact, authorable schema
|
|
96
|
+
language that fans out to OpenAPI, JSON Schema, and protobuf — and from
|
|
97
|
+
those to pydantic and zod. XSDFormer emits a `.tsp` so the same model
|
|
98
|
+
that backs the stored protobufs can also drive backend pydantic models
|
|
99
|
+
and frontend zod validators, all interconvertible.
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
# Default: clean, readable TypeSpec (string-valued enums, native scalars).
|
|
103
|
+
# Suited to pydantic/zod generation.
|
|
104
|
+
xsdformer xsd schema.xsd --typespec-out schema.tsp
|
|
105
|
+
|
|
106
|
+
# DTD sources work too (e.g. PubMed).
|
|
107
|
+
xsdformer dtd pubmed.dtd --typespec-out pubmed.tsp --proto-package pubmed
|
|
108
|
+
|
|
109
|
+
# proto-compat: adds @typespec/protobuf decorations (@package, @field,
|
|
110
|
+
# integer-valued enums) so `tsp -> proto` can be run as a regression
|
|
111
|
+
# check that it matches the directly generated proto.
|
|
112
|
+
xsdformer xsd schema.xsd --typespec-out schema.tsp --proto-compat
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
The `.tsp` is a derived artifact — regenerated from the XSD alongside
|
|
116
|
+
the proto, never hand-edited. The governing invariant is that
|
|
117
|
+
`xsd → proto` and `xsd → tsp → proto` agree at the wire/semantic level
|
|
118
|
+
(field numbers, field types, enum numbers); cosmetic differences
|
|
119
|
+
(`oneof` grouping, nested-vs-hoisted placement, comments, ordering) are
|
|
120
|
+
tolerated. See [`docs/adr/0001-typespec-output-format.md`](docs/adr/0001-typespec-output-format.md)
|
|
121
|
+
for the full design.
|