xsd-former 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. xsd_former-1.0.0/.github/CODEOWNERS +1 -0
  2. xsd_former-1.0.0/.github/workflows/lint.yml +28 -0
  3. xsd_former-1.0.0/.github/workflows/release.yml +49 -0
  4. xsd_former-1.0.0/.github/workflows/tests.yml +39 -0
  5. xsd_former-1.0.0/.gitignore +135 -0
  6. xsd_former-1.0.0/.markdownlint.json +13 -0
  7. xsd_former-1.0.0/.markdownlintignore +1 -0
  8. xsd_former-1.0.0/.pre-commit-config.yaml +51 -0
  9. xsd_former-1.0.0/.yamlfmt +10 -0
  10. xsd_former-1.0.0/CLAUDE.md +74 -0
  11. xsd_former-1.0.0/CONTEXT.md +115 -0
  12. xsd_former-1.0.0/LICENSE +21 -0
  13. xsd_former-1.0.0/PKG-INFO +121 -0
  14. xsd_former-1.0.0/README.md +89 -0
  15. xsd_former-1.0.0/clinvar_transforms.yaml +213 -0
  16. xsd_former-1.0.0/docs/adr/0001-typespec-output-format.md +197 -0
  17. xsd_former-1.0.0/docs/adr/0002-pydantic-codegen-and-proto-hub.md +249 -0
  18. xsd_former-1.0.0/docs/releasing.md +30 -0
  19. xsd_former-1.0.0/docs/style/general.md +30 -0
  20. xsd_former-1.0.0/docs/style/python.md +305 -0
  21. xsd_former-1.0.0/examples/clinvar_to_pbtxt.py +89 -0
  22. xsd_former-1.0.0/examples/pmid_31427284.pbtxt +2260 -0
  23. xsd_former-1.0.0/examples/pmid_33073003.pbtxt +504 -0
  24. xsd_former-1.0.0/examples/pubmed_to_pbtxt.py +93 -0
  25. xsd_former-1.0.0/pubmed_transforms.yaml +34 -0
  26. xsd_former-1.0.0/pyproject.toml +133 -0
  27. xsd_former-1.0.0/src/xsdformer/__init__.py +0 -0
  28. xsd_former-1.0.0/src/xsdformer/build.py +175 -0
  29. xsd_former-1.0.0/src/xsdformer/dtd/__init__.py +0 -0
  30. xsd_former-1.0.0/src/xsdformer/dtd/dtd.py +384 -0
  31. xsd_former-1.0.0/src/xsdformer/generator.py +32 -0
  32. xsd_former-1.0.0/src/xsdformer/jsonschema/__init__.py +0 -0
  33. xsd_former-1.0.0/src/xsdformer/jsonschema/generator.py +573 -0
  34. xsd_former-1.0.0/src/xsdformer/protobuf/__init__.py +0 -0
  35. xsd_former-1.0.0/src/xsdformer/protobuf/generator.py +154 -0
  36. xsd_former-1.0.0/src/xsdformer/py/__init__.py +0 -0
  37. xsd_former-1.0.0/src/xsdformer/py/xml_converter.py +629 -0
  38. xsd_former-1.0.0/src/xsdformer/py.typed +0 -0
  39. xsd_former-1.0.0/src/xsdformer/pydantic/__init__.py +0 -0
  40. xsd_former-1.0.0/src/xsdformer/pydantic/_naming.py +37 -0
  41. xsd_former-1.0.0/src/xsdformer/pydantic/converter.py +287 -0
  42. xsd_former-1.0.0/src/xsdformer/pydantic/generator.py +281 -0
  43. xsd_former-1.0.0/src/xsdformer/tool.py +401 -0
  44. xsd_former-1.0.0/src/xsdformer/transforms.py +460 -0
  45. xsd_former-1.0.0/src/xsdformer/typespec/__init__.py +0 -0
  46. xsd_former-1.0.0/src/xsdformer/typespec/generator.py +349 -0
  47. xsd_former-1.0.0/src/xsdformer/xsd/__init__.py +0 -0
  48. xsd_former-1.0.0/src/xsdformer/xsd/text.py +119 -0
  49. xsd_former-1.0.0/src/xsdformer/xsd/xsd.py +856 -0
  50. xsd_former-1.0.0/tests/__init__.py +0 -0
  51. xsd_former-1.0.0/tests/xsdformer/__init__.py +0 -0
  52. xsd_former-1.0.0/tests/xsdformer/conftest.py +231 -0
  53. xsd_former-1.0.0/tests/xsdformer/dtd/__init__.py +0 -0
  54. xsd_former-1.0.0/tests/xsdformer/dtd/test_dtd.py +329 -0
  55. xsd_former-1.0.0/tests/xsdformer/jsonschema/__init__.py +0 -0
  56. xsd_former-1.0.0/tests/xsdformer/jsonschema/test_generator.py +493 -0
  57. xsd_former-1.0.0/tests/xsdformer/protobuf/__init__.py +0 -0
  58. xsd_former-1.0.0/tests/xsdformer/protobuf/test_generator.py +102 -0
  59. xsd_former-1.0.0/tests/xsdformer/py/__init__.py +0 -0
  60. xsd_former-1.0.0/tests/xsdformer/py/test_xml_converter.py +506 -0
  61. xsd_former-1.0.0/tests/xsdformer/pydantic/__init__.py +0 -0
  62. xsd_former-1.0.0/tests/xsdformer/pydantic/_equivalence.py +179 -0
  63. xsd_former-1.0.0/tests/xsdformer/pydantic/records/pmid_31427284.xml +28 -0
  64. xsd_former-1.0.0/tests/xsdformer/pydantic/records/pmid_33073003.xml +4 -0
  65. xsd_former-1.0.0/tests/xsdformer/pydantic/test_converter.py +330 -0
  66. xsd_former-1.0.0/tests/xsdformer/pydantic/test_equivalence.py +152 -0
  67. xsd_former-1.0.0/tests/xsdformer/pydantic/test_generator.py +442 -0
  68. xsd_former-1.0.0/tests/xsdformer/pydantic/test_roundtrip.py +75 -0
  69. xsd_former-1.0.0/tests/xsdformer/test_build.py +212 -0
  70. xsd_former-1.0.0/tests/xsdformer/test_transforms.py +789 -0
  71. xsd_former-1.0.0/tests/xsdformer/typespec/__init__.py +0 -0
  72. xsd_former-1.0.0/tests/xsdformer/typespec/_tsp.py +172 -0
  73. xsd_former-1.0.0/tests/xsdformer/typespec/schemas/ClinVar_VCV.xsd +2783 -0
  74. xsd_former-1.0.0/tests/xsdformer/typespec/schemas/pubmed.dtd +475 -0
  75. xsd_former-1.0.0/tests/xsdformer/typespec/test_generator.py +420 -0
  76. xsd_former-1.0.0/tests/xsdformer/typespec/test_roundtrip.py +213 -0
  77. xsd_former-1.0.0/tests/xsdformer/typespec/tsp_project/.gitignore +2 -0
  78. xsd_former-1.0.0/tests/xsdformer/typespec/tsp_project/package.json +11 -0
  79. xsd_former-1.0.0/tests/xsdformer/xsd/__init__.py +0 -0
  80. xsd_former-1.0.0/tests/xsdformer/xsd/test_text.py +64 -0
  81. xsd_former-1.0.0/tests/xsdformer/xsd/test_xsd.py +442 -0
  82. xsd_former-1.0.0/uv.lock +1157 -0
@@ -0,0 +1 @@
1
+ * @folded
@@ -0,0 +1,28 @@
1
+ name: lint
2
+
3
+ on:
4
+ pull_request:
5
+ types: [opened, ready_for_review, synchronize]
6
+ push:
7
+ branches: [main]
8
+
9
+ jobs:
10
+ pre-commit:
11
+ # Runs every hook in .pre-commit-config.yaml — the whole static-check gate.
12
+ # Skip draft PRs.
13
+ if: github.event_name != 'pull_request' || !github.event.pull_request.draft
14
+ runs-on: ubuntu-latest
15
+ permissions:
16
+ contents: read
17
+ steps:
18
+ - uses: actions/checkout@v6
19
+ - uses: astral-sh/setup-uv@v8.2.0
20
+ with:
21
+ enable-cache: true
22
+ # Pre-sync the lint env so the pyright hook's `uv run` is a no-op.
23
+ - run: uv sync --locked --group lint --python 3.13
24
+ - uses: actions/cache@v6
25
+ with:
26
+ path: ~/.cache/pre-commit
27
+ key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}
28
+ - run: uv run pre-commit run --all-files --show-diff-on-failure
@@ -0,0 +1,49 @@
1
+ name: release
2
+
3
+ # Publishes to PyPI via Trusted Publishing (OIDC) — no API token. The PyPI
4
+ # project's trusted publisher must reference this repo, this workflow filename,
5
+ # and the `pypi` environment. Triggered on a published GitHub Release; the tag
6
+ # must match the package version (guarded below).
7
+ on:
8
+ release:
9
+ types: [published]
10
+
11
+ jobs:
12
+ build:
13
+ runs-on: ubuntu-latest
14
+ permissions:
15
+ contents: read
16
+ steps:
17
+ - uses: actions/checkout@v6
18
+ - uses: astral-sh/setup-uv@v8.2.0
19
+ with:
20
+ enable-cache: true
21
+ # Publishing a version is irreversible, so fail loudly if the release tag
22
+ # (vX.Y.Z) doesn't match the version in pyproject.toml.
23
+ - name: Check tag matches package version
24
+ run: |
25
+ tag="${GITHUB_REF_NAME#v}"
26
+ pkg="$(uv version --short)"
27
+ if [ "$tag" != "$pkg" ]; then
28
+ echo "Release tag '$GITHUB_REF_NAME' does not match package version '$pkg'"
29
+ exit 1
30
+ fi
31
+ - run: uv build
32
+ - uses: actions/upload-artifact@v4
33
+ with:
34
+ name: dist
35
+ path: dist/
36
+
37
+ publish:
38
+ needs: build
39
+ runs-on: ubuntu-latest
40
+ # The trusted-publisher and OIDC scope both key off this environment name.
41
+ environment: pypi
42
+ permissions:
43
+ id-token: write # mint the OIDC token PyPI exchanges for an upload token
44
+ steps:
45
+ - uses: actions/download-artifact@v4
46
+ with:
47
+ name: dist
48
+ path: dist/
49
+ - uses: pypa/gh-action-pypi-publish@release/v1
@@ -0,0 +1,39 @@
1
+ name: tests
2
+
3
+ on:
4
+ pull_request:
5
+ types: [opened, ready_for_review, synchronize]
6
+ push:
7
+ branches: [main]
8
+
9
+ jobs:
10
+ pytest:
11
+ # Skip draft PRs.
12
+ if: github.event_name != 'pull_request' || !github.event.pull_request.draft
13
+ runs-on: ubuntu-latest
14
+ strategy:
15
+ fail-fast: false
16
+ matrix:
17
+ python-version: ['3.11', '3.12', '3.13']
18
+ permissions:
19
+ contents: read
20
+ steps:
21
+ - uses: actions/checkout@v6
22
+ - uses: astral-sh/setup-uv@v8.2.0
23
+ with:
24
+ enable-cache: true
25
+ - uses: actions/setup-node@v4
26
+ with:
27
+ # @typespec/compiler >=1.13 requires Node >=22 (imports `glob` from
28
+ # `fs/promises`, added in Node 22). On Node 20 the toolchain crashes at
29
+ # runtime instead of skipping, failing the gated tsp tests.
30
+ node-version: '22'
31
+ # Enables the skipif-gated Node tests: the xsd->tsp->proto round-trip
32
+ # (`_tsp.tsp_available()`) and the pydantic equivalence gate
33
+ # (`_tsp.json_schema_available()`, via `@typespec/json-schema`). Without it
34
+ # both skip.
35
+ - name: Install TypeSpec toolchain
36
+ working-directory: tests/xsdformer/typespec/tsp_project
37
+ run: npm install
38
+ - run: uv sync --locked --group test --python ${{ matrix.python-version }}
39
+ - run: uv run pytest -q
@@ -0,0 +1,135 @@
1
+ # Python codebase default gitignore (github template)
2
+
3
+ # Byte-compiled / optimized / DLL files
4
+ __pycache__/
5
+ *.py[cod]
6
+ *$py.class
7
+
8
+ # local irrelevant files
9
+ .idea
10
+ .DS_Store
11
+
12
+ # C extensions
13
+ *.so
14
+
15
+ # Distribution / packaging
16
+ .Python
17
+ build/
18
+ develop-eggs/
19
+ dist/
20
+ downloads/
21
+ eggs/
22
+ .eggs/
23
+ lib/
24
+ lib64/
25
+ parts/
26
+ sdist/
27
+ var/
28
+ wheels/
29
+ pip-wheel-metadata/
30
+ share/python-wheels/
31
+ *.egg-info/
32
+ .installed.cfg
33
+ *.egg
34
+ MANIFEST
35
+
36
+ # PyInstaller
37
+ # Usually these files are written by a python script from a template
38
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
39
+ *.manifest
40
+ *.spec
41
+
42
+ # Installer logs
43
+ pip-log.txt
44
+ pip-delete-this-directory.txt
45
+
46
+ # Unit test / coverage reports
47
+ htmlcov/
48
+ .tox/
49
+ .nox/
50
+ .coverage
51
+ .coverage.*
52
+ .cache
53
+ nosetests.xml
54
+ coverage.xml
55
+ *.cover
56
+ *.py,cover
57
+ .hypothesis/
58
+ .pytest_cache/
59
+
60
+ # Translations
61
+ *.mo
62
+ *.pot
63
+
64
+ # Django stuff:
65
+ *.log
66
+ local_settings.py
67
+ db.sqlite3
68
+ db.sqlite3-journal
69
+
70
+ # Flask stuff:
71
+ instance/
72
+ .webassets-cache
73
+
74
+ # Scrapy stuff:
75
+ .scrapy
76
+
77
+ # Sphinx documentation
78
+ docs/_build/
79
+
80
+ # PyBuilder
81
+ target/
82
+
83
+ # Jupyter Notebook
84
+ .ipynb_checkpoints
85
+
86
+ # IPython
87
+ profile_default/
88
+ ipython_config.py
89
+
90
+ # pyenv
91
+ .python-version
92
+
93
+ # pipenv
94
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
95
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
96
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
97
+ # install all needed dependencies.
98
+ #Pipfile.lock
99
+
100
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow
101
+ __pypackages__/
102
+
103
+ # Celery stuff
104
+ celerybeat-schedule
105
+ celerybeat.pid
106
+
107
+ # SageMath parsed files
108
+ *.sage.py
109
+
110
+ # Environments
111
+ .env
112
+ .venv
113
+ env/
114
+ venv/
115
+ ENV/
116
+ env.bak/
117
+ venv.bak/
118
+
119
+ # Spyder project settings
120
+ .spyderproject
121
+ .spyproject
122
+
123
+ # Rope project settings
124
+ .ropeproject
125
+
126
+ # mkdocs documentation
127
+ /site
128
+
129
+ # mypy
130
+ .mypy_cache/
131
+ .dmypy.json
132
+ dmypy.json
133
+
134
+ # Pyre type checker
135
+ .pyre/
@@ -0,0 +1,13 @@
1
+ // https://github.com/DavidAnson/markdownlint#optionsconfig
2
+ // Disabling some rules here as we find them too restrictive.
3
+ {
4
+ "default": true, // Include all rules by defauls
5
+ "line-length": false, // To allow working with soft wraps in editors
6
+ "no-inline-html": { // Sometimes we need to use <img> html tags in GitHub markdown,
7
+ // as it doesn't allow setting the image size with markdown tags
8
+ "allowed_elements": ["details", "img"]
9
+ },
10
+ "ul-indent": false, // To allow indenting the whole list to distinguish it visually
11
+ "no-multiple-blanks": false // To allow multiple blank lines between a header and
12
+ // the previous paragraph
13
+ }
@@ -0,0 +1 @@
1
+ CLAUDE.md
@@ -0,0 +1,51 @@
1
+ repos:
2
+ - repo: https://github.com/pre-commit/pre-commit-hooks
3
+ rev: v6.0.0
4
+ hooks:
5
+ - id: check-yaml
6
+ exclude: '\.*conda/.*'
7
+ - id: end-of-file-fixer
8
+ # Vendored schemas (NLM DTD, ClinVar XSD) and real efetch record fixtures
9
+ # are kept byte-faithful to their upstream form; don't normalize them.
10
+ exclude: '^tests/xsdformer/(typespec/schemas|pydantic/records)/'
11
+ - id: trailing-whitespace
12
+ exclude: '\.txt$|\.tsv$|^tests/xsdformer/(typespec/schemas|pydantic/records)/'
13
+ - id: check-case-conflict
14
+ - id: check-merge-conflict
15
+ - id: detect-private-key
16
+ - id: debug-statements
17
+ - id: check-added-large-files
18
+
19
+ - repo: https://github.com/igorshubovych/markdownlint-cli
20
+ rev: v0.45.0
21
+ hooks:
22
+ - id: markdownlint
23
+
24
+ - repo: https://github.com/populationgenomics/pre-commits
25
+ rev: "e37928f761f17d54aca5cedf93848b40ec7cff26"
26
+ hooks:
27
+ - id: cpg-id-checker
28
+
29
+ - repo: https://github.com/astral-sh/ruff-pre-commit
30
+ rev: v0.14.1
31
+ hooks:
32
+ - id: ruff
33
+ - id: ruff-format
34
+
35
+ # pyright needs the project deps to resolve imports, so it runs in the lint env
36
+ # via `uv run`; `--group lint` provisions it on demand. Whole-project (include
37
+ # set in pyproject), so pass_filenames: false; fires on any staged .py.
38
+ - repo: local
39
+ hooks:
40
+ - id: pyright
41
+ name: pyright
42
+ entry: uv run --group lint pyright
43
+ language: system
44
+ files: \.py$
45
+ pass_filenames: false
46
+
47
+ # YAML formatter (ruff can't format YAML).
48
+ - repo: https://github.com/google/yamlfmt
49
+ rev: v0.21.0
50
+ hooks:
51
+ - id: yamlfmt
@@ -0,0 +1,10 @@
1
+ # YAML formatter config; the YAML analogue of ruff-format / biome.
2
+ # retain_line_breaks_single: yamlfmt can't selectively strip blank lines
3
+ # between block-sequence items, so the no-blank-line-between-steps layout in
4
+ # .github/workflows is kept by hand — this setting only stops yamlfmt from
5
+ # reintroducing or multiplying blank lines elsewhere.
6
+ formatter:
7
+ type: basic
8
+ retain_line_breaks_single: true
9
+ trim_trailing_whitespace: true
10
+ eof_newline: true
@@ -0,0 +1,74 @@
1
+ # xsdformer development notes
2
+
3
+ xsdformer converts an XSD or DTD schema into Protobuf, JSON Schema, TypeSpec,
4
+ Pydantic models, and Python XML-to-protobuf converters. See
5
+ [`CONTEXT.md`](CONTEXT.md) for the domain model and language, and
6
+ [`README.md`](README.md) for usage.
7
+
8
+ ## Working norms
9
+
10
+ Operating directives for Claude (and any agent) in this repo; they counteract default
11
+ model dispositions.
12
+
13
+ - **Resist the minimal-diff reflex.** Don't reach for the smallest change that hides the
14
+ symptom (special-casing, papering over root causes). Aim for the correct fix at the
15
+ right complexity level — not the smallest, not gold-plated.
16
+ - **Fail loudly and early.** Raise on a missing expected input or precondition; never fall
17
+ back to a default/placeholder to limp along. A placeholder is an explicit caller input,
18
+ never a code default.
19
+ - **Push back; don't just comply.** When a design, name, or approach seems worse —
20
+ including a shortcut you're asked to take — say so with reasoning, unprompted. The
21
+ author owns the final call.
22
+ - **Offer better alternatives with trade-offs.** When a materially better approach than
23
+ the proposed one exists, present it and the trade-offs — don't just execute the ask.
24
+ - **Investigate before producing.** Read the code and verify constraints first. Don't
25
+ treat a training-pattern convention as load-bearing unchecked; don't speculate about
26
+ what you can read.
27
+ - **Explain non-obvious changes first.** For a change whose rationale isn't self-evident,
28
+ give the why before showing or applying the diff.
29
+ - **Ask when unsure** rather than assume intent.
30
+ - **No intensifiers or emphasis filler.** Drop words and phrases that add emphasis but no
31
+ information — "that's the key", "crucially", "importantly", "the key insight", "it's
32
+ worth noting". State the point plainly. Applies to all prose: chat replies, PR/review
33
+ comments, commit messages, and docs.
34
+
35
+ ## Commands
36
+
37
+ ```bash
38
+ uv sync --group test --group lint # install dev dependencies
39
+ uv run pytest # run the test suite
40
+ uv run pytest tests/xsdformer/xsd/test_xsd.py::test_name -x # single test
41
+ uv run pre-commit run --all-files # the full static-check gate (see below)
42
+ ```
43
+
44
+ Some tests are gated on a TypeSpec (Node) toolchain and skip when it is absent;
45
+ CI installs it. See [`CONTEXT.md`](CONTEXT.md) for what each module does.
46
+
47
+ ## Code style
48
+
49
+ @docs/style/general.md
50
+ @docs/style/python.md
51
+
52
+ ## Docs
53
+
54
+ The primary audience for docs is a model reading them as context; humans second. Be
55
+ terse: state each decision, mechanism, and rationale once — no rhetorical emphasis, no
56
+ persuasion, no recaps. Every token written is re-paid on every future read. Design
57
+ decisions with lasting consequences get an ADR under [`docs/adr/`](docs/adr).
58
+
59
+ ## Committing
60
+
61
+ - **Stage explicit paths**, not `git add -A` / `.`.
62
+ - **Pre-commit is the full static-check gate** (`.pre-commit-config.yaml`): lint, format,
63
+ hygiene, and pyright. CI runs the same hooks via `pre-commit run --all-files`, so the
64
+ two can't drift. Ensure hooks are installed (`pre-commit install`) — if not, install or
65
+ ask the author; never bypass with `--no-verify`.
66
+ - **Correct a pushed branch with a new commit on top**, not amend + force-push. PRs
67
+ squash-merge, so `main` history stays linear regardless and intermediate fixups vanish
68
+ on merge. Reserve force-push for rebasing a branch onto `main`.
69
+
70
+ ## CI and review
71
+
72
+ - **Pin third-party GitHub Actions to the latest stable release**: the moving major tag
73
+ (`@v3`) where the action publishes one, else the exact latest version (`@v8.2.0`). Verify
74
+ against the action's releases when adding or bumping one.
@@ -0,0 +1,115 @@
1
+ # xsdformer
2
+
3
+ xsdformer converts an XML schema — an **XSD** or a **DTD** — into code that
4
+ describes and processes the same data in other ecosystems: Protobuf
5
+ definitions, JSON Schema, TypeSpec, Pydantic models, and Python code that
6
+ parses conforming XML into Protobuf messages.
7
+
8
+ It supports enough of XSD/DTD to convert the
9
+ [ClinVar](https://www.ncbi.nlm.nih.gov/clinvar/) and
10
+ [BioC](http://bioc.sourceforge.net/) schemas. Full coverage of the XSD
11
+ specification is a non-goal; the parser raises on constructs it does not model
12
+ rather than emitting a silent approximation.
13
+
14
+ ## Pipeline
15
+
16
+ ```text
17
+ schema (XSD | DTD) → parse → IR (TypeDefinition graph) → transform → generate → output
18
+ ```
19
+
20
+ A front end parses the schema into an intermediate representation (IR). The IR
21
+ is a topologically sorted graph of type definitions, independent of both the
22
+ input schema language and the output target. Each back end (generator) walks
23
+ that IR to emit one output format. Adding an input or output format touches one
24
+ front end or one generator, not the pipeline.
25
+
26
+ ## The IR — language
27
+
28
+ The IR lives in [`xsd/xsd.py`](src/xsdformer/xsd/xsd.py). These are the domain
29
+ terms; use them exactly.
30
+
31
+ **TypeDefinition**:
32
+ A named type in the schema. The base for the three concrete kinds a generator
33
+ must handle:
34
+
35
+ - **Message** — a structured type (an XSD complex type): a set of fields.
36
+ - **Enumeration** — a closed set of named values.
37
+ - **MapType** — a type that models a map (key/value) rather than a record.
38
+
39
+ **FieldDefinition**:
40
+ A member of a `Message`. The base for leaf **Field** kinds and **container**
41
+ kinds:
42
+
43
+ - **Field** — a leaf carrying a value. Subtypes: **Elem** (a child element),
44
+ **Attr** (an XML attribute), **ValueElem** (an element's text content).
45
+ - **Seq** — an ordered group of fields.
46
+ - **Choice** — a mutually-exclusive group of fields.
47
+
48
+ **computed_occurs**:
49
+ A field's effective `(min, max)` occurrence after multiplying through every
50
+ enclosing container. A field that is optional inside a repeated `Seq` is
51
+ repeated overall; `computed_occurs` is that resolved multiplicity.
52
+
53
+ **enclosing_type**:
54
+ Links a nested (enclosed) type to the `Message` it is defined inside. A type
55
+ with no `enclosing_type` is top-level.
56
+
57
+ **path**:
58
+ The tuple of names from the root schema type down to a given type — its fully
59
+ qualified position in the type graph.
60
+
61
+ ## Front ends (parsers)
62
+
63
+ - [`xsd/xsd.py`](src/xsdformer/xsd/xsd.py) — parses XSD via the `xmlschema`
64
+ library, topologically sorts types by dependency, and produces the IR.
65
+ - [`dtd/dtd.py`](src/xsdformer/dtd/dtd.py) — parses DTD into the same IR.
66
+ - [`transforms.py`](src/xsdformer/transforms.py) — optional IR-level
67
+ simplifications (dropping fields, inlining single-field wrapper types,
68
+ collapsing a type to a plain string) applied before generation, driven by a
69
+ YAML transform spec (see `clinvar_transforms.yaml`, `pubmed_transforms.yaml`).
70
+
71
+ ## Back ends (generators)
72
+
73
+ [`generator.py`](src/xsdformer/generator.py) defines the `IGenerator` protocol
74
+ and `generate_with()`, which drives any generator over the IR and skips enclosed
75
+ types at the top level (the enclosing type emits them). Each generator
76
+ implements `IGenerator`:
77
+
78
+ - [`protobuf/generator.py`](src/xsdformer/protobuf/generator.py) — emits
79
+ `.proto` syntax.
80
+ - [`py/xml_converter.py`](src/xsdformer/py/xml_converter.py) — emits Python code
81
+ that converts parsed lxml elements into Protobuf message instances. Runtime
82
+ helpers (e.g. `_xml_bool`, `_consume`) are embedded into the generated output
83
+ via `inspect.getsource`.
84
+ - [`jsonschema/generator.py`](src/xsdformer/jsonschema/generator.py) — compiles
85
+ proto (from the IR or an existing `.proto` file) via `grpc_tools.protoc` into
86
+ a `FileDescriptorSet`, then walks the Protobuf descriptors to emit JSON
87
+ Schema.
88
+ - [`typespec/generator.py`](src/xsdformer/typespec/generator.py) — emits
89
+ TypeSpec (`.tsp`). See [ADR 0001](docs/adr/0001-typespec-output-format.md).
90
+ - [`pydantic/`](src/xsdformer/pydantic) — emits Pydantic models
91
+ (`generator.py`) and a converter (`converter.py`), sharing naming logic in
92
+ `_naming.py`. See [ADR 0002](docs/adr/0002-pydantic-codegen-and-proto-hub.md).
93
+
94
+ ## CLI
95
+
96
+ [`tool.py`](src/xsdformer/tool.py) exposes a Click group with three commands:
97
+
98
+ - `xsdformer xsd <file>` — XSD front end; emit any combination of proto, Python
99
+ converter, JSON Schema, TypeSpec, and Pydantic outputs (one `--*-out` flag
100
+ each).
101
+ - `xsdformer dtd <file>` — DTD front end; same output flags.
102
+ - `xsdformer proto <file> <namespace>` — Proto-to-JSON-Schema only, bypassing
103
+ the XSD/DTD front end.
104
+
105
+ ## Package builder
106
+
107
+ [`build.py`](src/xsdformer/build.py) assembles a standalone, pip-installable
108
+ Python package from a schema: generated proto (compiled to `_pb2.py`), the XML
109
+ converter, and Pydantic models, wrapped in a generated `pyproject.toml`.
110
+
111
+ ## Text utilities
112
+
113
+ [`xsd/text.py`](src/xsdformer/xsd/text.py) — `snake_case`, `pascal_case`, and
114
+ `keep` (wraps a string in `_Exact` to bypass case conversion). All identifier
115
+ naming flows through these.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 Centre for Population Genomics
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,121 @@
1
+ Metadata-Version: 2.4
2
+ Name: xsd-former
3
+ Version: 1.0.0
4
+ Summary: Generate Protobuf definitions, JSON Schema, and Python XML-to-protobuf converters from XSD and DTD schemas.
5
+ Project-URL: Repository, https://github.com/populationgenomics/xsd-former
6
+ Project-URL: Issues, https://github.com/populationgenomics/xsd-former/issues
7
+ Author: Toby Sargeant
8
+ License: MIT
9
+ License-File: LICENSE
10
+ Keywords: bioc,clinvar,code-generation,dtd,json-schema,protobuf,typespec,xml,xsd
11
+ Classifier: Development Status :: 5 - Production/Stable
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Intended Audience :: Science/Research
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Operating System :: OS Independent
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Programming Language :: Python :: 3.13
20
+ Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
21
+ Classifier: Topic :: Software Development :: Code Generators
22
+ Classifier: Typing :: Typed
23
+ Requires-Python: >=3.11
24
+ Requires-Dist: click>=8.1
25
+ Requires-Dist: grpcio-tools>=1.60
26
+ Requires-Dist: jinja2>=3.1.6
27
+ Requires-Dist: lxml>=5.0
28
+ Requires-Dist: protobuf>=6.32.1
29
+ Requires-Dist: pyyaml>=6.0.3
30
+ Requires-Dist: xmlschema>=4.0
31
+ Description-Content-Type: text/markdown
32
+
33
+ # XSDFormer
34
+
35
+ XSDFormer deforms [XML schema definitions
36
+ (XSDs)](https://www.w3.org/XML/Schema) into [Protobuf
37
+ definitions](https://protobuf.dev/). It also generates Python code
38
+ to convert parsed XML into the corresponding Protobuf representation.
39
+
40
+ It supports enough of the XSD specification to convert the
41
+ [ClinVar](https://www.ncbi.nlm.nih.gov/clinvar/) and
42
+ [BioC](http://bioc.sourceforge.net/) schemas; full support for all
43
+ XSD features is a non-goal.
44
+
45
+ ## Why convert from XSD to Protobuf?
46
+
47
+ While XML and XSDs (or alternatively JSON and JSON schemas) are
48
+ powerful for defining complex, human-readable data structures, they
49
+ have some drawbacks, especially in high-performance applications
50
+ or when dealing with large datasets.
51
+
52
+ ### Performance and Size
53
+
54
+ * **Parsing Speed:** XML is a text-based format and can be slow to
55
+ parse. Protobuf is a binary format that is designed for speed and
56
+ efficiency. Converting XML data to Protobuf can result in significantly
57
+ faster parsing times.
58
+ * **Storage Space:** XML is verbose, with opening and closing tags
59
+ that add to the file size. Protobuf's binary format is much more
60
+ compact, leading to smaller file sizes. This is a major advantage
61
+ for storing large datasets or for transmitting data over a network.
62
+ Effectively compressing XML requires a schema-specific dictionary,
63
+ or compressing multiple records together so that the dictionary of
64
+ tag/key names can be reused. However this limits the possibility
65
+ for random access enabled by compressing records individually.
66
+
67
+ ### Developer Experience
68
+
69
+ * **Generated Code:** Protobuf compilers generate code in many
70
+ languages, providing a simple and consistent way to work with the
71
+ data.
72
+ * **Type Safety:** The Protobuf schema provides strong typing, which
73
+ can help to prevent bugs. Parsing the wire format requires the
74
+ protobuf definition, meaning that the data is tied to its typed
75
+ representation. Conversely XML representations are only optionally
76
+ validated by a schema, and so by default type information is lost
77
+ during parsing (everything is treated as text).
78
+
79
+ By converting XSDs to Protobuf definitions, `xsd-former` allows
80
+ developers to take advantage of the benefits of Protobuf while still
81
+ working with data that is originally defined in an XML schema.
82
+
83
+ ## Output formats
84
+
85
+ From a single XSD (or DTD) source, XSDFormer can emit:
86
+
87
+ * **Protobuf** (`.proto`) — `--proto-out`.
88
+ * **Python XML→protobuf converters** — `--py-out`/`--py-module`.
89
+ * **JSON Schema** — `--json-schema-out` (from the XSD, or directly from
90
+ a `.proto` via the `proto` subcommand).
91
+ * **TypeSpec** (`.tsp`) — `--typespec-out` (see below).
92
+
93
+ ## TypeSpec output
94
+
95
+ [TypeSpec](https://typespec.io/) is a compact, authorable schema
96
+ language that fans out to OpenAPI, JSON Schema, and protobuf — and from
97
+ those to pydantic and zod. XSDFormer emits a `.tsp` so the same model
98
+ that backs the stored protobufs can also drive backend pydantic models
99
+ and frontend zod validators, all interconvertible.
100
+
101
+ ```bash
102
+ # Default: clean, readable TypeSpec (string-valued enums, native scalars).
103
+ # Suited to pydantic/zod generation.
104
+ xsdformer xsd schema.xsd --typespec-out schema.tsp
105
+
106
+ # DTD sources work too (e.g. PubMed).
107
+ xsdformer dtd pubmed.dtd --typespec-out pubmed.tsp --proto-package pubmed
108
+
109
+ # proto-compat: adds @typespec/protobuf decorations (@package, @field,
110
+ # integer-valued enums) so `tsp -> proto` can be run as a regression
111
+ # check that it matches the directly generated proto.
112
+ xsdformer xsd schema.xsd --typespec-out schema.tsp --proto-compat
113
+ ```
114
+
115
+ The `.tsp` is a derived artifact — regenerated from the XSD alongside
116
+ the proto, never hand-edited. The governing invariant is that
117
+ `xsd → proto` and `xsd → tsp → proto` agree at the wire/semantic level
118
+ (field numbers, field types, enum numbers); cosmetic differences
119
+ (`oneof` grouping, nested-vs-hoisted placement, comments, ordering) are
120
+ tolerated. See [`docs/adr/0001-typespec-output-format.md`](docs/adr/0001-typespec-output-format.md)
121
+ for the full design.