doclang 0.7.2__tar.gz → 0.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {doclang-0.7.2 → doclang-0.8.0}/CHANGELOG.md +26 -0
- {doclang-0.7.2/doclang.egg-info → doclang-0.8.0}/PKG-INFO +32 -1
- {doclang-0.7.2 → doclang-0.8.0}/README.md +31 -0
- {doclang-0.7.2 → doclang-0.8.0}/doclang/README.md +80 -0
- {doclang-0.7.2 → doclang-0.8.0}/doclang/__init__.py +8 -1
- {doclang-0.7.2 → doclang-0.8.0}/doclang/_packaging.py +98 -28
- doclang-0.8.0/doclang/_schemas.py +195 -0
- {doclang-0.7.2 → doclang-0.8.0}/doclang/backends/saxonche.py +4 -3
- doclang-0.8.0/doclang/cli.py +499 -0
- doclang-0.8.0/doclang/doclang.sch +539 -0
- doclang-0.8.0/doclang/doclang.xsd +859 -0
- doclang-0.8.0/doclang/migration/__init__.py +5 -0
- doclang-0.8.0/doclang/migration/_api.py +191 -0
- doclang-0.8.0/doclang/migration/_plan.py +64 -0
- doclang-0.8.0/doclang/migration/_steps/__init__.py +5 -0
- doclang-0.8.0/doclang/migration/_steps/next.py +16 -0
- doclang-0.8.0/doclang/migration/_steps/v0_8_to_0_7.py +53 -0
- doclang-0.8.0/doclang/migration/_tree.py +69 -0
- {doclang-0.7.2 → doclang-0.8.0}/doclang/packaging.py +15 -1
- {doclang-0.7.2/doclang → doclang-0.8.0/doclang/schemas/0.7}/doclang.xsd +1 -1
- doclang-0.8.0/doclang/schemas/0.8/doclang.sch +539 -0
- doclang-0.8.0/doclang/schemas/0.8/doclang.xsd +859 -0
- doclang-0.8.0/doclang/tokenization.py +172 -0
- doclang-0.8.0/doclang/utils.py +127 -0
- doclang-0.8.0/doclang/validation.py +183 -0
- doclang-0.8.0/doclang/xsd_validation.py +78 -0
- {doclang-0.7.2 → doclang-0.8.0/doclang.egg-info}/PKG-INFO +32 -1
- {doclang-0.7.2 → doclang-0.8.0}/doclang.egg-info/SOURCES.txt +13 -1
- {doclang-0.7.2 → doclang-0.8.0}/pyproject.toml +7 -4
- doclang-0.7.2/doclang/_schemas.py +0 -23
- doclang-0.7.2/doclang/cli.py +0 -291
- doclang-0.7.2/doclang/utils.py +0 -66
- doclang-0.7.2/doclang/validation.py +0 -104
- doclang-0.7.2/doclang/xsd_validation.py +0 -51
- {doclang-0.7.2 → doclang-0.8.0}/LICENSE +0 -0
- {doclang-0.7.2 → doclang-0.8.0}/MANIFEST.in +0 -0
- {doclang-0.7.2 → doclang-0.8.0}/doclang/backends/__init__.py +0 -0
- {doclang-0.7.2 → doclang-0.8.0}/doclang/backends/_svrl.py +0 -0
- {doclang-0.7.2/doclang → doclang-0.8.0/doclang/schemas/0.7}/doclang.sch +0 -0
- {doclang-0.7.2 → doclang-0.8.0}/doclang/schematron.py +0 -0
- {doclang-0.7.2 → doclang-0.8.0}/doclang/version.py +0 -0
- {doclang-0.7.2 → doclang-0.8.0}/doclang.egg-info/dependency_links.txt +0 -0
- {doclang-0.7.2 → doclang-0.8.0}/doclang.egg-info/entry_points.txt +0 -0
- {doclang-0.7.2 → doclang-0.8.0}/doclang.egg-info/requires.txt +0 -0
- {doclang-0.7.2 → doclang-0.8.0}/doclang.egg-info/top_level.txt +0 -0
- {doclang-0.7.2 → doclang-0.8.0}/setup.cfg +0 -0
|
@@ -1,3 +1,29 @@
|
|
|
1
|
+
## [v0.8.0](https://github.com/doclang-project/doclang/releases/tag/v0.8.0)
|
|
2
|
+
|
|
3
|
+
* add version migration (#196)
|
|
4
|
+
* enable version-specific validation (#195)
|
|
5
|
+
* **BREAKING CHANGE**: XSD no longer restricts `version` attribute to one value nor does it give it a default; selection of version to validate against now defined in spec and implemented by reference toolkit.
|
|
6
|
+
* reformat appendix into annexes (#187)
|
|
7
|
+
* add conformance clause, move versioning rules into it, define roles (#193)
|
|
8
|
+
* fix `<src>` reference entry for `<frame>`, `<cover>` and `<audio>` (#199)
|
|
9
|
+
* define subject matter in scope section (#192)
|
|
10
|
+
* replace mermaid governance workflow diagram with static file (#194)
|
|
11
|
+
* remove section on version 0.x behavior (#191)
|
|
12
|
+
* reorganize terms and definitions, disambiguate "document" (#184)
|
|
13
|
+
* fix normative references and bibliography clauses (#183)
|
|
14
|
+
* move motivation details out of spec (#180)
|
|
15
|
+
* move list of contributors from spec to README (#179)
|
|
16
|
+
* specify spec as authoritative source in case of discrepancies
|
|
17
|
+
* fix commented out section so nothing gets left behind (#190)
|
|
18
|
+
* allow multiple captions (#176)
|
|
19
|
+
* add `<track>` for time-aligned media transcripts (#175)
|
|
20
|
+
* expose special token vocabulary (#174)
|
|
21
|
+
|
|
22
|
+
## [v0.7.3](https://github.com/doclang-project/doclang/releases/tag/v0.7.3)
|
|
23
|
+
|
|
24
|
+
* refuse symbolic links when packing pages and assets (#171)
|
|
25
|
+
* reject DTD and entity references in DocLang validation (#170)
|
|
26
|
+
|
|
1
27
|
## [v0.7.2](https://github.com/doclang-project/doclang/releases/tag/v0.7.2)
|
|
2
28
|
|
|
3
29
|
* expand lxml version support (#158)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: doclang
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.8.0
|
|
4
4
|
Summary: DocLang reference toolkit
|
|
5
5
|
Author: DocLang Project
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -49,12 +49,16 @@ Dynamic: license-file
|
|
|
49
49
|
|
|
50
50
|
This repository is the home of the normative specification and the reference toolkit for DocLang. If you build with LLMs and VLMs on real-world content, this is where the standard lives.
|
|
51
51
|
|
|
52
|
+
For additional background information see [MOTIVATION.md](https://github.com/doclang-project/doclang/blob/main/MOTIVATION.md).
|
|
53
|
+
|
|
52
54
|
## Specification
|
|
53
55
|
|
|
54
56
|
The source of the specification is available in [spec.md](https://github.com/doclang-project/doclang/blob/main/spec.md)
|
|
55
57
|
and exports to different formats can be found in the [exports/](https://github.com/doclang-project/doclang/tree/main/exports)
|
|
56
58
|
directory.
|
|
57
59
|
|
|
60
|
+
In case of discrepancies, the authoritative version of the specification is the original Markdown document.
|
|
61
|
+
|
|
58
62
|
## Reference Toolkit
|
|
59
63
|
|
|
60
64
|
The commands below illustrate basic scenarios. For advanced installation and usage options
|
|
@@ -101,6 +105,33 @@ To work on this repository — setup, tests, reference generation, releases —
|
|
|
101
105
|
|
|
102
106
|
DocLang is developed in the open and supported by the [LF AI & Data Foundation](https://lfaidata.foundation/projects/). Learn more about the project at [doclang-project](https://github.com/doclang-project).
|
|
103
107
|
|
|
108
|
+
## Acknowledgements
|
|
109
|
+
|
|
110
|
+
The following people have contributed to the DocLang specification.
|
|
111
|
+
|
|
112
|
+
| Name | Company |
|
|
113
|
+
|------|---------|
|
|
114
|
+
| Peter Staar | IBM |
|
|
115
|
+
| Panos Vagenas | IBM |
|
|
116
|
+
| Maksym Lysak | IBM |
|
|
117
|
+
| Nikolaos Livathinos | IBM |
|
|
118
|
+
| Christoph Auer | IBM |
|
|
119
|
+
| Michele Dolfi | IBM |
|
|
120
|
+
| Said Gürbüz | IBM |
|
|
121
|
+
| Santosh Borse | IBM |
|
|
122
|
+
| Yousaf Shah | IBM |
|
|
123
|
+
| Maroun Touma | IBM |
|
|
124
|
+
| Christopher Giblin | IBM |
|
|
125
|
+
| Marlene Wolfgruber | ABBYY |
|
|
126
|
+
| Maxime Vermeir | ABBYY |
|
|
127
|
+
| Morgan Logue | ABBYY |
|
|
128
|
+
| Andrew Pery | ABBYY |
|
|
129
|
+
| Jehlum Vitasta Pandit | RedHat |
|
|
130
|
+
| Ali Maredia | RedHat |
|
|
131
|
+
| Micaela Kaplan | HumanSignal |
|
|
132
|
+
| Ryan Chesler | NVIDIA |
|
|
133
|
+
| Riccardo Maggioni | Forgis |
|
|
134
|
+
|
|
104
135
|
## License
|
|
105
136
|
|
|
106
137
|
DocLang is licensed under the Apache License 2.0. See [LICENSE](https://github.com/doclang-project/doclang/blob/main/LICENSE) for details.
|
|
@@ -18,12 +18,16 @@
|
|
|
18
18
|
|
|
19
19
|
This repository is the home of the normative specification and the reference toolkit for DocLang. If you build with LLMs and VLMs on real-world content, this is where the standard lives.
|
|
20
20
|
|
|
21
|
+
For additional background information see [MOTIVATION.md](https://github.com/doclang-project/doclang/blob/main/MOTIVATION.md).
|
|
22
|
+
|
|
21
23
|
## Specification
|
|
22
24
|
|
|
23
25
|
The source of the specification is available in [spec.md](https://github.com/doclang-project/doclang/blob/main/spec.md)
|
|
24
26
|
and exports to different formats can be found in the [exports/](https://github.com/doclang-project/doclang/tree/main/exports)
|
|
25
27
|
directory.
|
|
26
28
|
|
|
29
|
+
In case of discrepancies, the authoritative version of the specification is the original Markdown document.
|
|
30
|
+
|
|
27
31
|
## Reference Toolkit
|
|
28
32
|
|
|
29
33
|
The commands below illustrate basic scenarios. For advanced installation and usage options
|
|
@@ -70,6 +74,33 @@ To work on this repository — setup, tests, reference generation, releases —
|
|
|
70
74
|
|
|
71
75
|
DocLang is developed in the open and supported by the [LF AI & Data Foundation](https://lfaidata.foundation/projects/). Learn more about the project at [doclang-project](https://github.com/doclang-project).
|
|
72
76
|
|
|
77
|
+
## Acknowledgements
|
|
78
|
+
|
|
79
|
+
The following people have contributed to the DocLang specification.
|
|
80
|
+
|
|
81
|
+
| Name | Company |
|
|
82
|
+
|------|---------|
|
|
83
|
+
| Peter Staar | IBM |
|
|
84
|
+
| Panos Vagenas | IBM |
|
|
85
|
+
| Maksym Lysak | IBM |
|
|
86
|
+
| Nikolaos Livathinos | IBM |
|
|
87
|
+
| Christoph Auer | IBM |
|
|
88
|
+
| Michele Dolfi | IBM |
|
|
89
|
+
| Said Gürbüz | IBM |
|
|
90
|
+
| Santosh Borse | IBM |
|
|
91
|
+
| Yousaf Shah | IBM |
|
|
92
|
+
| Maroun Touma | IBM |
|
|
93
|
+
| Christopher Giblin | IBM |
|
|
94
|
+
| Marlene Wolfgruber | ABBYY |
|
|
95
|
+
| Maxime Vermeir | ABBYY |
|
|
96
|
+
| Morgan Logue | ABBYY |
|
|
97
|
+
| Andrew Pery | ABBYY |
|
|
98
|
+
| Jehlum Vitasta Pandit | RedHat |
|
|
99
|
+
| Ali Maredia | RedHat |
|
|
100
|
+
| Micaela Kaplan | HumanSignal |
|
|
101
|
+
| Ryan Chesler | NVIDIA |
|
|
102
|
+
| Riccardo Maggioni | Forgis |
|
|
103
|
+
|
|
73
104
|
## License
|
|
74
105
|
|
|
75
106
|
DocLang is licensed under the Apache License 2.0. See [LICENSE](https://github.com/doclang-project/doclang/blob/main/LICENSE) for details.
|
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
Official Python toolkit for working with DocLang — CLI commands and library APIs.
|
|
4
4
|
|
|
5
|
+
This toolkit and associated schemas are meant to enable validation of DocLang documents in line with the DocLang specification. In case of discrepancies, the authoritative source is the specification.
|
|
6
|
+
|
|
5
7
|
## Installation
|
|
6
8
|
|
|
7
9
|
Recommended — full validation (XSD + Schematron):
|
|
@@ -29,6 +31,25 @@ On a minimal install, pass `xsd_only=True` for XSD-only validation, or supply a
|
|
|
29
31
|
doclang validate my_document.dclg
|
|
30
32
|
```
|
|
31
33
|
|
|
34
|
+
The schema is chosen as follows:
|
|
35
|
+
|
|
36
|
+
1. `--spec-version X.Y` (Python: `spec_version="X.Y"`) uses the bundled schema of that version, overriding the
|
|
37
|
+
document's own. It is an error if there is none.
|
|
38
|
+
2. Otherwise the spec version in the `version` attribute of the document's root element is used.
|
|
39
|
+
3. Otherwise the latest schema (the development head) is used, with a warning. This covers documents that
|
|
40
|
+
declare no version, and (with a warning naming the version) documents whose declared version has no bundled
|
|
41
|
+
schema. Pass `--strict-declared-version` (Python: `strict_declared_version=True`) to fail in the latter case.
|
|
42
|
+
|
|
43
|
+
The latest released schema of each spec minor version is bundled in `doclang/schemas/<MAJOR.MINOR>/`.
|
|
44
|
+
The schema directly in `doclang/` is the development head.
|
|
45
|
+
|
|
46
|
+
> **Note on 0.7:** the 0.7 XSD only accepts `0.7` in the root `version` attribute (later versions only
|
|
47
|
+
> require the `MAJOR.MINOR` format). So that a document declaring another version (e.g. `0.8`) can be
|
|
48
|
+
> checked against the 0.7 rules, the toolkit by default replaces the declared `version` with `0.7` in
|
|
49
|
+
> memory for XSD validation (the file is not modified), and says so in a note. The declared version itself
|
|
50
|
+
> is then not checked; all other 0.7 rules are still checked. Use `--strict-xsd-version-attr` (Python:
|
|
51
|
+
> `strict_xsd_version_attr=True`) to apply the XSD's version check as is, in which case such a document fails.
|
|
52
|
+
|
|
32
53
|
#### More validation scenarios
|
|
33
54
|
|
|
34
55
|
```bash
|
|
@@ -44,6 +65,12 @@ doclang validate my_document.dclg --schematron-only
|
|
|
44
65
|
# JSON output
|
|
45
66
|
doclang validate my_document.dclg --format json
|
|
46
67
|
|
|
68
|
+
# Validate against a specific spec version (overrides the document's `version`)
|
|
69
|
+
doclang validate my_document.dclg --spec-version 0.7
|
|
70
|
+
|
|
71
|
+
# ... and apply the XSD's own `version` attribute check as is
|
|
72
|
+
doclang validate my_document.dclg --spec-version 0.7 --strict-xsd-version-attr
|
|
73
|
+
|
|
47
74
|
# Quiet mode (exit code only)
|
|
48
75
|
doclang validate my_document.dclg --quiet
|
|
49
76
|
|
|
@@ -68,6 +95,35 @@ doclang pack markup.dclg --assets payload/
|
|
|
68
95
|
doclang pack markup.dclg --validate
|
|
69
96
|
```
|
|
70
97
|
|
|
98
|
+
### Migration
|
|
99
|
+
|
|
100
|
+
Migrate a document to another spec version. Currently only older versions are supported (documents are forward
|
|
101
|
+
compatible, so a newer target needs no migration):
|
|
102
|
+
|
|
103
|
+
```bash
|
|
104
|
+
doclang migrate document.dclg --to 0.7
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
The migrated document goes to stdout (or `-o FILE`); what was changed goes to stderr. The migration is best effort:
|
|
108
|
+
content that has an equivalent in the target version is mapped (e.g. additional captions become `<text>`), the rest
|
|
109
|
+
is dropped (e.g. `<track>`), and every change is reported with its location.
|
|
110
|
+
|
|
111
|
+
#### More migration scenarios
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
doclang migrate document.dclg --to 0.7 -o document_0.7.dclg
|
|
115
|
+
doclang migrate document.dclg --to 0.7 --strict # fail instead of changing anything
|
|
116
|
+
doclang migrate document.dclg --to 0.7 --format json # versions, changes and the document as JSON
|
|
117
|
+
doclang migrate document.dclg --to 0.7 --input-validation xsd --output-validation xsd # without Schematron
|
|
118
|
+
doclang migrate document.dclg --to 0.7 --input-validation none # skip validating the input
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
By default the document is validated before the migration (against its own version) and the result afterwards
|
|
122
|
+
(against the target version), in full, XSD and Schematron. `--input-validation` and `--output-validation` take
|
|
123
|
+
`all`, `xsd`, `schematron` or `none`; `xsd` and `schematron` mean the same as `--xsd-only` and `--schematron-only`
|
|
124
|
+
of `doclang validate`. As there, Schematron needs a backend (see [Installation](#installation)): without one,
|
|
125
|
+
`doclang migrate` fails and tells you so, and you opt out explicitly, e.g. with the `xsd` scope for both.
|
|
126
|
+
|
|
71
127
|
## Python API
|
|
72
128
|
|
|
73
129
|
### Validation
|
|
@@ -141,6 +197,30 @@ path = pack(
|
|
|
141
197
|
print(f"Created {path}")
|
|
142
198
|
```
|
|
143
199
|
|
|
200
|
+
### Migration
|
|
201
|
+
|
|
202
|
+
```python
|
|
203
|
+
from doclang import migrate, MigrationError
|
|
204
|
+
|
|
205
|
+
try:
|
|
206
|
+
result = migrate("document.dclg", to="0.7")
|
|
207
|
+
except MigrationError as exc:
|
|
208
|
+
print(exc)
|
|
209
|
+
else:
|
|
210
|
+
print(result.xml) # the migrated document
|
|
211
|
+
for change in result.changes: # {"kind": "mapped" | "dropped", "location": ..., "line": ..., "message": ...}
|
|
212
|
+
print(change["kind"], change["location"])
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
`migrate(..., strict=True)` raises `MigrationError` (carrying `changes`) if anything would be changed. The source
|
|
216
|
+
version is the document's `version` attribute; without one the document is treated as the latest version, with a
|
|
217
|
+
warning.
|
|
218
|
+
|
|
219
|
+
The input is validated before and the result after the migration; `input_validation` and `output_validation`
|
|
220
|
+
(`"all"` by default, or `"xsd"`, `"schematron"`, `"none"`) select what runs. As with `validate()`, a missing
|
|
221
|
+
Schematron backend raises `SchematronBackendNotFound`. An invalid input raises `MigrationError`; so does an invalid
|
|
222
|
+
result, which indicates a bug in a migration step (or an invalid input that was not validated).
|
|
223
|
+
|
|
144
224
|
## Validation Rules
|
|
145
225
|
|
|
146
226
|
### XSD Validation (doclang.xsd)
|
|
@@ -1,15 +1,22 @@
|
|
|
1
1
|
"""DocLang reference toolkit."""
|
|
2
2
|
|
|
3
|
+
from doclang.migration import MigrationError, MigrationResult, migrate
|
|
3
4
|
from doclang.packaging import PackagingError, pack
|
|
4
5
|
from doclang.schematron import SchematronBackendNotFound, SchematronValidator, SchematronViolation
|
|
5
|
-
from doclang.
|
|
6
|
+
from doclang.tokenization import get_special_tokens
|
|
7
|
+
from doclang.validation import UnsupportedVersionError, ValidationError, validate
|
|
6
8
|
|
|
7
9
|
__all__ = [
|
|
10
|
+
"MigrationError",
|
|
11
|
+
"MigrationResult",
|
|
8
12
|
"PackagingError",
|
|
9
13
|
"SchematronBackendNotFound",
|
|
10
14
|
"SchematronValidator",
|
|
11
15
|
"SchematronViolation",
|
|
16
|
+
"UnsupportedVersionError",
|
|
12
17
|
"ValidationError",
|
|
18
|
+
"get_special_tokens",
|
|
19
|
+
"migrate",
|
|
13
20
|
"pack",
|
|
14
21
|
"validate",
|
|
15
22
|
]
|
|
@@ -17,6 +17,17 @@ _CONTENT_TYPES_XML = """\
|
|
|
17
17
|
<Default Extension="jpg" ContentType="image/jpeg"/>
|
|
18
18
|
<Default Extension="jpeg" ContentType="image/jpeg"/>
|
|
19
19
|
<Default Extension="webp" ContentType="image/webp"/>
|
|
20
|
+
<Default Extension="mp3" ContentType="audio/mpeg"/>
|
|
21
|
+
<Default Extension="m4a" ContentType="audio/mp4"/>
|
|
22
|
+
<Default Extension="aac" ContentType="audio/aac"/>
|
|
23
|
+
<Default Extension="ogg" ContentType="audio/ogg"/>
|
|
24
|
+
<Default Extension="opus" ContentType="audio/ogg"/>
|
|
25
|
+
<Default Extension="wav" ContentType="audio/wav"/>
|
|
26
|
+
<Default Extension="flac" ContentType="audio/flac"/>
|
|
27
|
+
<Default Extension="mp4" ContentType="video/mp4"/>
|
|
28
|
+
<Default Extension="webm" ContentType="video/webm"/>
|
|
29
|
+
<Default Extension="mkv" ContentType="video/x-matroska"/>
|
|
30
|
+
<Default Extension="mov" ContentType="video/quicktime"/>
|
|
20
31
|
<Override PartName="/document.xml" ContentType="application/vnd.doclang.document+xml"/>
|
|
21
32
|
</Types>
|
|
22
33
|
"""
|
|
@@ -37,6 +48,9 @@ PagesInput = Union[
|
|
|
37
48
|
Mapping[int, Union[str, Path]],
|
|
38
49
|
]
|
|
39
50
|
|
|
51
|
+
# Whole-track media (audio/, video/) uses the same 1-based positional model as pages/.
|
|
52
|
+
MediaInput = PagesInput
|
|
53
|
+
|
|
40
54
|
AssetsInput = Union[
|
|
41
55
|
str,
|
|
42
56
|
Path,
|
|
@@ -58,6 +72,11 @@ def _require_directory(path: Path, *, label: str) -> None:
|
|
|
58
72
|
raise PackagingError(f"{label} not found or not a directory: {path}")
|
|
59
73
|
|
|
60
74
|
|
|
75
|
+
def _reject_symlink(path: Path, *, label: str) -> None:
|
|
76
|
+
if path.is_symlink():
|
|
77
|
+
raise PackagingError(f"{label} must not be a symbolic link: {path}")
|
|
78
|
+
|
|
79
|
+
|
|
61
80
|
def _validate_archive_relative_path(path: str, *, label: str) -> None:
|
|
62
81
|
if not path or path.startswith("/") or "\\" in path:
|
|
63
82
|
raise PackagingError(f"Invalid {label} path: {path!r}")
|
|
@@ -66,12 +85,22 @@ def _validate_archive_relative_path(path: str, *, label: str) -> None:
|
|
|
66
85
|
raise PackagingError(f"Invalid {label} path: {path!r}")
|
|
67
86
|
|
|
68
87
|
|
|
69
|
-
def
|
|
88
|
+
def _copy_file(source: Path, destination: Path, *, label: str) -> None:
|
|
89
|
+
_reject_symlink(source, label=label)
|
|
90
|
+
_require_file(source, label=label)
|
|
91
|
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
92
|
+
shutil.copy2(source, destination)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _copy_tree_into(source: Path, destination: Path, *, label: str) -> None:
|
|
96
|
+
_reject_symlink(source, label=label)
|
|
97
|
+
_require_directory(source, label=label)
|
|
70
98
|
destination.mkdir(parents=True, exist_ok=True)
|
|
71
99
|
for item in source.iterdir():
|
|
72
100
|
target = destination / item.name
|
|
101
|
+
_reject_symlink(item, label=label)
|
|
73
102
|
if item.is_dir():
|
|
74
|
-
|
|
103
|
+
_copy_tree_into(item, target, label=label)
|
|
75
104
|
else:
|
|
76
105
|
shutil.copy2(item, target)
|
|
77
106
|
|
|
@@ -81,31 +110,71 @@ def _place_document(stage: Path, document: Path) -> None:
|
|
|
81
110
|
shutil.copy2(document, stage / "document.xml")
|
|
82
111
|
|
|
83
112
|
|
|
84
|
-
def
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
113
|
+
def _place_indexed(
|
|
114
|
+
stage: Path,
|
|
115
|
+
spec: PagesInput,
|
|
116
|
+
*,
|
|
117
|
+
subdir: str,
|
|
118
|
+
file_label: str,
|
|
119
|
+
dir_label: str,
|
|
120
|
+
number_label: str,
|
|
121
|
+
) -> None:
|
|
122
|
+
"""Place 1-based positionally-indexed files (``{N}.{ext}``) under ``stage/subdir``.
|
|
123
|
+
|
|
124
|
+
Shared by ``pages/`` (page images) and ``audio/`` / ``video/`` (whole-track media),
|
|
125
|
+
which all key a file to an integer position in the document.
|
|
126
|
+
"""
|
|
127
|
+
target_dir = stage / subdir
|
|
128
|
+
if isinstance(spec, Mapping):
|
|
129
|
+
for number, source in spec.items():
|
|
130
|
+
if not isinstance(number, int) or number < 1:
|
|
131
|
+
raise PackagingError(f"{number_label} must be positive integers, got {number!r}")
|
|
90
132
|
source_path = Path(source)
|
|
91
|
-
|
|
92
|
-
destination =
|
|
93
|
-
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
94
|
-
shutil.copy2(source_path, destination)
|
|
133
|
+
destination = target_dir / f"{number}{source_path.suffix}"
|
|
134
|
+
_copy_file(source_path, destination, label=file_label)
|
|
95
135
|
return
|
|
96
136
|
|
|
97
|
-
if isinstance(
|
|
98
|
-
|
|
99
|
-
_require_directory(source_dir, label="Pages directory")
|
|
100
|
-
_copy_tree_into(source_dir, pages_dir)
|
|
137
|
+
if isinstance(spec, str | Path):
|
|
138
|
+
_copy_tree_into(Path(spec), target_dir, label=dir_label)
|
|
101
139
|
return
|
|
102
140
|
|
|
103
|
-
for index, source in enumerate(
|
|
141
|
+
for index, source in enumerate(spec, start=1):
|
|
104
142
|
source_path = Path(source)
|
|
105
|
-
|
|
106
|
-
destination =
|
|
107
|
-
|
|
108
|
-
|
|
143
|
+
destination = target_dir / f"{index}{source_path.suffix}"
|
|
144
|
+
_copy_file(source_path, destination, label=file_label)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _place_pages(stage: Path, pages: PagesInput) -> None:
|
|
148
|
+
_place_indexed(
|
|
149
|
+
stage,
|
|
150
|
+
pages,
|
|
151
|
+
subdir="pages",
|
|
152
|
+
file_label="Page file",
|
|
153
|
+
dir_label="Pages directory",
|
|
154
|
+
number_label="Page numbers",
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _place_audio(stage: Path, audio: MediaInput) -> None:
|
|
159
|
+
_place_indexed(
|
|
160
|
+
stage,
|
|
161
|
+
audio,
|
|
162
|
+
subdir="audio",
|
|
163
|
+
file_label="Audio file",
|
|
164
|
+
dir_label="Audio directory",
|
|
165
|
+
number_label="Audio track numbers",
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def _place_video(stage: Path, video: MediaInput) -> None:
|
|
170
|
+
_place_indexed(
|
|
171
|
+
stage,
|
|
172
|
+
video,
|
|
173
|
+
subdir="video",
|
|
174
|
+
file_label="Video file",
|
|
175
|
+
dir_label="Video directory",
|
|
176
|
+
number_label="Video track numbers",
|
|
177
|
+
)
|
|
109
178
|
|
|
110
179
|
|
|
111
180
|
def _place_assets(stage: Path, assets: AssetsInput) -> None:
|
|
@@ -113,16 +182,11 @@ def _place_assets(stage: Path, assets: AssetsInput) -> None:
|
|
|
113
182
|
if isinstance(assets, Mapping):
|
|
114
183
|
for archive_path, source in assets.items():
|
|
115
184
|
_validate_archive_relative_path(archive_path, label="asset")
|
|
116
|
-
source_path = Path(source)
|
|
117
|
-
_require_file(source_path, label="Asset file")
|
|
118
185
|
destination = assets_dir / archive_path
|
|
119
|
-
|
|
120
|
-
shutil.copy2(source_path, destination)
|
|
186
|
+
_copy_file(Path(source), destination, label="Asset file")
|
|
121
187
|
return
|
|
122
188
|
|
|
123
|
-
|
|
124
|
-
_require_directory(source_dir, label="Assets directory")
|
|
125
|
-
_copy_tree_into(source_dir, assets_dir)
|
|
189
|
+
_copy_tree_into(Path(assets), assets_dir, label="Assets directory")
|
|
126
190
|
|
|
127
191
|
|
|
128
192
|
def _write_opc_metadata(stage: Path) -> None:
|
|
@@ -160,6 +224,8 @@ def _pack(
|
|
|
160
224
|
output: Union[str, Path, None] = None,
|
|
161
225
|
pages: PagesInput | None = None,
|
|
162
226
|
assets: AssetsInput | None = None,
|
|
227
|
+
audio: MediaInput | None = None,
|
|
228
|
+
video: MediaInput | None = None,
|
|
163
229
|
validate: bool = False,
|
|
164
230
|
) -> Path:
|
|
165
231
|
document_path = Path(document)
|
|
@@ -172,6 +238,10 @@ def _pack(
|
|
|
172
238
|
_place_pages(stage, pages)
|
|
173
239
|
if assets is not None:
|
|
174
240
|
_place_assets(stage, assets)
|
|
241
|
+
if audio is not None:
|
|
242
|
+
_place_audio(stage, audio)
|
|
243
|
+
if video is not None:
|
|
244
|
+
_place_video(stage, video)
|
|
175
245
|
_write_opc_metadata(stage)
|
|
176
246
|
|
|
177
247
|
if validate:
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
"""Bundled DocLang schema paths and spec-version selection (internal).
|
|
2
|
+
|
|
3
|
+
``schemas/<MAJOR.MINOR>/`` holds the latest released schemas of each spec minor version.
|
|
4
|
+
The schemas directly in the package directory are the development head: they are used when no
|
|
5
|
+
version is specified, and as a fallback for a declared version that has no ``schemas/`` folder.
|
|
6
|
+
At release, the head is copied to ``schemas/<MAJOR.MINOR>/`` (see ``utils/prepare_release.py``).
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import re
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
from lxml import etree
|
|
16
|
+
|
|
17
|
+
_SCHEMA_DIR = Path(__file__).resolve().parent
|
|
18
|
+
_VERSION_RE = re.compile(r"^(\d+)\.(\d+)$")
|
|
19
|
+
_SCHEMA_VERSION_RE = re.compile(r'<xs:schema[^>]*\sversion="(\d+)\.(\d+)\.\d+"', re.DOTALL)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
_XS_NS = "http://www.w3.org/2001/XMLSchema"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class UnsupportedVersionError(ValueError):
|
|
26
|
+
"""Raised when there are no schemas for a requested or declared DocLang spec version."""
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class SchemaSelection:
|
|
31
|
+
"""The schemas chosen for a document."""
|
|
32
|
+
|
|
33
|
+
version: str # version of the chosen schemas (for the head: its label)
|
|
34
|
+
xsd: Path
|
|
35
|
+
sch: Path
|
|
36
|
+
declared: bool # False when no version was requested or declared (head used)
|
|
37
|
+
head_fallback_for: str | None = None # declared version without schemas, for which the head was used
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _bundled_xsd_path() -> Path:
|
|
41
|
+
path = _SCHEMA_DIR / "doclang.xsd"
|
|
42
|
+
if not path.exists():
|
|
43
|
+
raise FileNotFoundError(f"Bundled XSD schema not found: {path}")
|
|
44
|
+
return path
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _bundled_sch_path() -> Path:
|
|
48
|
+
path = _SCHEMA_DIR / "doclang.sch"
|
|
49
|
+
if not path.exists():
|
|
50
|
+
raise FileNotFoundError(f"Bundled Schematron schema not found: {path}")
|
|
51
|
+
return path
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _bundled_schema_paths() -> tuple[Path, Path]:
|
|
55
|
+
return _bundled_xsd_path(), _bundled_sch_path()
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _version_key(version: str) -> tuple[int, int]:
|
|
59
|
+
match = _VERSION_RE.match(version)
|
|
60
|
+
assert match is not None
|
|
61
|
+
return int(match.group(1)), int(match.group(2))
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _schema_version(xsd: Path) -> str:
|
|
65
|
+
"""The ``MAJOR.MINOR`` spec version of an XSD, from its ``xs:schema`` version attribute."""
|
|
66
|
+
match = _SCHEMA_VERSION_RE.search(xsd.read_text(encoding="utf-8"))
|
|
67
|
+
if match is None:
|
|
68
|
+
raise FileNotFoundError(f"No schema version found in {xsd}")
|
|
69
|
+
return f"{match.group(1)}.{match.group(2)}"
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _current_version() -> str:
|
|
73
|
+
"""The spec version of the top-level (latest) schemas."""
|
|
74
|
+
return _schema_version(_bundled_xsd_path())
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _versioned_dir(version: str) -> Path:
|
|
78
|
+
return _SCHEMA_DIR / "schemas" / version
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _has_versioned_schemas(version: str) -> bool:
|
|
82
|
+
folder = _versioned_dir(version)
|
|
83
|
+
return (folder / "doclang.xsd").exists() and (folder / "doclang.sch").exists()
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _available_versions() -> list[str]:
|
|
87
|
+
"""Versions with bundled ``schemas/`` folders, oldest first."""
|
|
88
|
+
versioned_dir = _SCHEMA_DIR / "schemas"
|
|
89
|
+
if not versioned_dir.is_dir():
|
|
90
|
+
return []
|
|
91
|
+
versions = [e.name for e in versioned_dir.iterdir() if _VERSION_RE.match(e.name) and _has_versioned_schemas(e.name)]
|
|
92
|
+
return sorted(versions, key=_version_key)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _resolve_in_folders(requested: str) -> str | None:
|
|
96
|
+
"""Map a requested ``MAJOR.MINOR`` to a version with bundled schemas, or None.
|
|
97
|
+
|
|
98
|
+
An exact match wins. From major version 1, an instance is compatible with any later minor
|
|
99
|
+
of the same major, so the nearest newer minor with schemas is used.
|
|
100
|
+
"""
|
|
101
|
+
if not _VERSION_RE.match(requested):
|
|
102
|
+
return None
|
|
103
|
+
available = _available_versions()
|
|
104
|
+
if requested in available:
|
|
105
|
+
return requested
|
|
106
|
+
major, minor = _version_key(requested)
|
|
107
|
+
if major >= 1:
|
|
108
|
+
for candidate in available:
|
|
109
|
+
if _version_key(candidate)[0] == major and _version_key(candidate)[1] > minor:
|
|
110
|
+
return candidate
|
|
111
|
+
return None
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _versioned_paths(version: str) -> tuple[Path, Path]:
|
|
115
|
+
folder = _versioned_dir(version)
|
|
116
|
+
return folder / "doclang.xsd", folder / "doclang.sch"
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _head_description() -> str:
|
|
120
|
+
"""How to refer to the head in messages: its version if identical to the released schema of that
|
|
121
|
+
version, else "unreleased development version" (the head's label is then not reliable)."""
|
|
122
|
+
label = _current_version()
|
|
123
|
+
if _has_versioned_schemas(label):
|
|
124
|
+
released_xsd, released_sch = _versioned_paths(label)
|
|
125
|
+
head_xsd, head_sch = _bundled_schema_paths()
|
|
126
|
+
if released_xsd.read_bytes() == head_xsd.read_bytes() and released_sch.read_bytes() == head_sch.read_bytes():
|
|
127
|
+
return label
|
|
128
|
+
return "unreleased development version"
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _enumerated_versions(xsd: Path) -> set[str]:
|
|
132
|
+
"""Versions the XSD enumerates for the root ``version`` attribute (empty if only format-constrained)."""
|
|
133
|
+
parser = etree.XMLParser(resolve_entities=False, no_network=True)
|
|
134
|
+
tree = etree.parse(str(xsd), parser)
|
|
135
|
+
values = tree.xpath("//xs:attribute[@name='version']//xs:enumeration/@value", namespaces={"xs": _XS_NS})
|
|
136
|
+
return {str(v) for v in values} # type: ignore[union-attr]
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _declared_version(xml_file: str | Path) -> str | None:
|
|
140
|
+
"""The ``version`` attribute of the root element, or None if absent or unreadable."""
|
|
141
|
+
try:
|
|
142
|
+
with open(xml_file, "rb") as f:
|
|
143
|
+
_, root = next(
|
|
144
|
+
etree.iterparse(f, events=("start",), resolve_entities=False, no_network=True, load_dtd=False)
|
|
145
|
+
)
|
|
146
|
+
return root.get("version")
|
|
147
|
+
except Exception:
|
|
148
|
+
return None
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _select_schemas(
|
|
152
|
+
xml_file: str | Path,
|
|
153
|
+
spec_version: str | None = None,
|
|
154
|
+
*,
|
|
155
|
+
strict_declared_version: bool = False,
|
|
156
|
+
) -> SchemaSelection:
|
|
157
|
+
"""Pick schemas for a document.
|
|
158
|
+
|
|
159
|
+
1. An explicit ``spec_version`` uses the bundled schemas of that version (else UnsupportedVersionError).
|
|
160
|
+
2. Otherwise the document's declared ``version`` uses the bundled schemas of that version.
|
|
161
|
+
3. Otherwise the development head is used. For a declared version without schemas this is a
|
|
162
|
+
fallback, unless ``strict_declared_version`` is set (then UnsupportedVersionError).
|
|
163
|
+
"""
|
|
164
|
+
if spec_version is not None:
|
|
165
|
+
version = _resolve_in_folders(spec_version)
|
|
166
|
+
if version is None:
|
|
167
|
+
available = ", ".join(_available_versions()) or "none"
|
|
168
|
+
raise UnsupportedVersionError(
|
|
169
|
+
f"No schema for DocLang spec version '{spec_version}'; available versions: {available}"
|
|
170
|
+
)
|
|
171
|
+
xsd, sch = _versioned_paths(version)
|
|
172
|
+
return SchemaSelection(version=version, xsd=xsd, sch=sch, declared=True)
|
|
173
|
+
|
|
174
|
+
declared_version = _declared_version(xml_file)
|
|
175
|
+
head_xsd, head_sch = _bundled_schema_paths()
|
|
176
|
+
if declared_version is None:
|
|
177
|
+
return SchemaSelection(version=_current_version(), xsd=head_xsd, sch=head_sch, declared=False)
|
|
178
|
+
|
|
179
|
+
version = _resolve_in_folders(declared_version)
|
|
180
|
+
if version is not None:
|
|
181
|
+
xsd, sch = _versioned_paths(version)
|
|
182
|
+
return SchemaSelection(version=version, xsd=xsd, sch=sch, declared=True)
|
|
183
|
+
|
|
184
|
+
if strict_declared_version:
|
|
185
|
+
available = ", ".join(_available_versions()) or "none"
|
|
186
|
+
raise UnsupportedVersionError(
|
|
187
|
+
f"No schema for declared DocLang version '{declared_version}'; available versions: {available}"
|
|
188
|
+
)
|
|
189
|
+
return SchemaSelection(
|
|
190
|
+
version=_current_version(),
|
|
191
|
+
xsd=head_xsd,
|
|
192
|
+
sch=head_sch,
|
|
193
|
+
declared=True,
|
|
194
|
+
head_fallback_for=declared_version,
|
|
195
|
+
)
|