doclang 0.7.2__tar.gz → 0.8.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. {doclang-0.7.2 → doclang-0.8.0}/CHANGELOG.md +26 -0
  2. {doclang-0.7.2/doclang.egg-info → doclang-0.8.0}/PKG-INFO +32 -1
  3. {doclang-0.7.2 → doclang-0.8.0}/README.md +31 -0
  4. {doclang-0.7.2 → doclang-0.8.0}/doclang/README.md +80 -0
  5. {doclang-0.7.2 → doclang-0.8.0}/doclang/__init__.py +8 -1
  6. {doclang-0.7.2 → doclang-0.8.0}/doclang/_packaging.py +98 -28
  7. doclang-0.8.0/doclang/_schemas.py +195 -0
  8. {doclang-0.7.2 → doclang-0.8.0}/doclang/backends/saxonche.py +4 -3
  9. doclang-0.8.0/doclang/cli.py +499 -0
  10. doclang-0.8.0/doclang/doclang.sch +539 -0
  11. doclang-0.8.0/doclang/doclang.xsd +859 -0
  12. doclang-0.8.0/doclang/migration/__init__.py +5 -0
  13. doclang-0.8.0/doclang/migration/_api.py +191 -0
  14. doclang-0.8.0/doclang/migration/_plan.py +64 -0
  15. doclang-0.8.0/doclang/migration/_steps/__init__.py +5 -0
  16. doclang-0.8.0/doclang/migration/_steps/next.py +16 -0
  17. doclang-0.8.0/doclang/migration/_steps/v0_8_to_0_7.py +53 -0
  18. doclang-0.8.0/doclang/migration/_tree.py +69 -0
  19. {doclang-0.7.2 → doclang-0.8.0}/doclang/packaging.py +15 -1
  20. {doclang-0.7.2/doclang → doclang-0.8.0/doclang/schemas/0.7}/doclang.xsd +1 -1
  21. doclang-0.8.0/doclang/schemas/0.8/doclang.sch +539 -0
  22. doclang-0.8.0/doclang/schemas/0.8/doclang.xsd +859 -0
  23. doclang-0.8.0/doclang/tokenization.py +172 -0
  24. doclang-0.8.0/doclang/utils.py +127 -0
  25. doclang-0.8.0/doclang/validation.py +183 -0
  26. doclang-0.8.0/doclang/xsd_validation.py +78 -0
  27. {doclang-0.7.2 → doclang-0.8.0/doclang.egg-info}/PKG-INFO +32 -1
  28. {doclang-0.7.2 → doclang-0.8.0}/doclang.egg-info/SOURCES.txt +13 -1
  29. {doclang-0.7.2 → doclang-0.8.0}/pyproject.toml +7 -4
  30. doclang-0.7.2/doclang/_schemas.py +0 -23
  31. doclang-0.7.2/doclang/cli.py +0 -291
  32. doclang-0.7.2/doclang/utils.py +0 -66
  33. doclang-0.7.2/doclang/validation.py +0 -104
  34. doclang-0.7.2/doclang/xsd_validation.py +0 -51
  35. {doclang-0.7.2 → doclang-0.8.0}/LICENSE +0 -0
  36. {doclang-0.7.2 → doclang-0.8.0}/MANIFEST.in +0 -0
  37. {doclang-0.7.2 → doclang-0.8.0}/doclang/backends/__init__.py +0 -0
  38. {doclang-0.7.2 → doclang-0.8.0}/doclang/backends/_svrl.py +0 -0
  39. {doclang-0.7.2/doclang → doclang-0.8.0/doclang/schemas/0.7}/doclang.sch +0 -0
  40. {doclang-0.7.2 → doclang-0.8.0}/doclang/schematron.py +0 -0
  41. {doclang-0.7.2 → doclang-0.8.0}/doclang/version.py +0 -0
  42. {doclang-0.7.2 → doclang-0.8.0}/doclang.egg-info/dependency_links.txt +0 -0
  43. {doclang-0.7.2 → doclang-0.8.0}/doclang.egg-info/entry_points.txt +0 -0
  44. {doclang-0.7.2 → doclang-0.8.0}/doclang.egg-info/requires.txt +0 -0
  45. {doclang-0.7.2 → doclang-0.8.0}/doclang.egg-info/top_level.txt +0 -0
  46. {doclang-0.7.2 → doclang-0.8.0}/setup.cfg +0 -0
@@ -1,3 +1,29 @@
1
+ ## [v0.8.0](https://github.com/doclang-project/doclang/releases/tag/v0.8.0)
2
+
3
+ * add version migration (#196)
4
+ * enable version-specific validation (#195)
5
+ * **BREAKING CHANGE**: XSD no longer restricts `version` attribute to one value nor does it give it a default; selection of version to validate against now defined in spec and implemented by reference toolkit.
6
+ * reformat appendix into annexes (#187)
7
+ * add conformance clause, move versioning rules into it, define roles (#193)
8
+ * fix `<src>` reference entry for `<frame>`, `<cover>` and `<audio>` (#199)
9
+ * define subject matter in scope section (#192)
10
+ * replace mermaid governance workflow diagram with static file (#194)
11
+ * remove section on version 0.x behavior (#191)
12
+ * reorganize terms and definitions, disambiguate "document" (#184)
13
+ * fix normative references and bibliography clauses (#183)
14
+ * move motivation details out of spec (#180)
15
+ * move list of contributors from spec to README (#179)
16
+ * specify spec as authoritative source in case of discrepancies
17
+ * fix commented out section so nothing gets left behind (#190)
18
+ * allow multiple captions (#176)
19
+ * add `<track>` for time-aligned media transcripts (#175)
20
+ * expose special token vocabulary (#174)
21
+
22
+ ## [v0.7.3](https://github.com/doclang-project/doclang/releases/tag/v0.7.3)
23
+
24
+ * refuse symbolic links when packing pages and assets (#171)
25
+ * reject DTD and entity references in DocLang validation (#170)
26
+
1
27
  ## [v0.7.2](https://github.com/doclang-project/doclang/releases/tag/v0.7.2)
2
28
 
3
29
  * expand lxml version support (#158)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: doclang
3
- Version: 0.7.2
3
+ Version: 0.8.0
4
4
  Summary: DocLang reference toolkit
5
5
  Author: DocLang Project
6
6
  License-Expression: Apache-2.0
@@ -49,12 +49,16 @@ Dynamic: license-file
49
49
 
50
50
  This repository is the home of the normative specification and the reference toolkit for DocLang. If you build with LLMs and VLMs on real-world content, this is where the standard lives.
51
51
 
52
+ For additional background information see [MOTIVATION.md](https://github.com/doclang-project/doclang/blob/main/MOTIVATION.md).
53
+
52
54
  ## Specification
53
55
 
54
56
  The source of the specification is available in [spec.md](https://github.com/doclang-project/doclang/blob/main/spec.md)
55
57
  and exports to different formats can be found in the [exports/](https://github.com/doclang-project/doclang/tree/main/exports)
56
58
  directory.
57
59
 
60
+ In case of discrepancies, the authoritative version of the specification is the original Markdown document.
61
+
58
62
  ## Reference Toolkit
59
63
 
60
64
  The commands below illustrate basic scenarios. For advanced installation and usage options
@@ -101,6 +105,33 @@ To work on this repository — setup, tests, reference generation, releases —
101
105
 
102
106
  DocLang is developed in the open and supported by the [LF AI & Data Foundation](https://lfaidata.foundation/projects/). Learn more about the project at [doclang-project](https://github.com/doclang-project).
103
107
 
108
+ ## Acknowledgements
109
+
110
+ The following people have contributed to the DocLang specification.
111
+
112
+ | Name | Company |
113
+ |------|---------|
114
+ | Peter Staar | IBM |
115
+ | Panos Vagenas | IBM |
116
+ | Maksym Lysak | IBM |
117
+ | Nikolaos Livathinos | IBM |
118
+ | Christoph Auer | IBM |
119
+ | Michele Dolfi | IBM |
120
+ | Said Gürbüz | IBM |
121
+ | Santosh Borse | IBM |
122
+ | Yousaf Shah | IBM |
123
+ | Maroun Touma | IBM |
124
+ | Christopher Giblin | IBM |
125
+ | Marlene Wolfgruber | ABBYY |
126
+ | Maxime Vermeir | ABBYY |
127
+ | Morgan Logue | ABBYY |
128
+ | Andrew Pery | ABBYY |
129
+ | Jehlum Vitasta Pandit | RedHat |
130
+ | Ali Maredia | RedHat |
131
+ | Micaela Kaplan | HumanSignal |
132
+ | Ryan Chesler | NVIDIA |
133
+ | Riccardo Maggioni | Forgis |
134
+
104
135
  ## License
105
136
 
106
137
  DocLang is licensed under the Apache License 2.0. See [LICENSE](https://github.com/doclang-project/doclang/blob/main/LICENSE) for details.
@@ -18,12 +18,16 @@
18
18
 
19
19
  This repository is the home of the normative specification and the reference toolkit for DocLang. If you build with LLMs and VLMs on real-world content, this is where the standard lives.
20
20
 
21
+ For additional background information see [MOTIVATION.md](https://github.com/doclang-project/doclang/blob/main/MOTIVATION.md).
22
+
21
23
  ## Specification
22
24
 
23
25
  The source of the specification is available in [spec.md](https://github.com/doclang-project/doclang/blob/main/spec.md)
24
26
  and exports to different formats can be found in the [exports/](https://github.com/doclang-project/doclang/tree/main/exports)
25
27
  directory.
26
28
 
29
+ In case of discrepancies, the authoritative version of the specification is the original Markdown document.
30
+
27
31
  ## Reference Toolkit
28
32
 
29
33
  The commands below illustrate basic scenarios. For advanced installation and usage options
@@ -70,6 +74,33 @@ To work on this repository — setup, tests, reference generation, releases —
70
74
 
71
75
  DocLang is developed in the open and supported by the [LF AI & Data Foundation](https://lfaidata.foundation/projects/). Learn more about the project at [doclang-project](https://github.com/doclang-project).
72
76
 
77
+ ## Acknowledgements
78
+
79
+ The following people have contributed to the DocLang specification.
80
+
81
+ | Name | Company |
82
+ |------|---------|
83
+ | Peter Staar | IBM |
84
+ | Panos Vagenas | IBM |
85
+ | Maksym Lysak | IBM |
86
+ | Nikolaos Livathinos | IBM |
87
+ | Christoph Auer | IBM |
88
+ | Michele Dolfi | IBM |
89
+ | Said Gürbüz | IBM |
90
+ | Santosh Borse | IBM |
91
+ | Yousaf Shah | IBM |
92
+ | Maroun Touma | IBM |
93
+ | Christopher Giblin | IBM |
94
+ | Marlene Wolfgruber | ABBYY |
95
+ | Maxime Vermeir | ABBYY |
96
+ | Morgan Logue | ABBYY |
97
+ | Andrew Pery | ABBYY |
98
+ | Jehlum Vitasta Pandit | RedHat |
99
+ | Ali Maredia | RedHat |
100
+ | Micaela Kaplan | HumanSignal |
101
+ | Ryan Chesler | NVIDIA |
102
+ | Riccardo Maggioni | Forgis |
103
+
73
104
  ## License
74
105
 
75
106
  DocLang is licensed under the Apache License 2.0. See [LICENSE](https://github.com/doclang-project/doclang/blob/main/LICENSE) for details.
@@ -2,6 +2,8 @@
2
2
 
3
3
  Official Python toolkit for working with DocLang — CLI commands and library APIs.
4
4
 
5
+ This toolkit and associated schemas are meant to enable validation of DocLang documents in line with the DocLang specification. In case of discrepancies, the authoritative source is the specification.
6
+
5
7
  ## Installation
6
8
 
7
9
  Recommended — full validation (XSD + Schematron):
@@ -29,6 +31,25 @@ On a minimal install, pass `xsd_only=True` for XSD-only validation, or supply a
29
31
  doclang validate my_document.dclg
30
32
  ```
31
33
 
34
+ The schema is chosen as follows:
35
+
36
+ 1. `--spec-version X.Y` (Python: `spec_version="X.Y"`) uses the bundled schema of that version, overriding the
37
+ document's own. It is an error if there is none.
38
+ 2. Otherwise the spec version in the `version` attribute of the document's root element is used.
39
+ 3. Otherwise the latest schema (the development head) is used, with a warning. This covers documents that
40
+ declare no version, and (with a warning naming the version) documents whose declared version has no bundled
41
+ schema. Pass `--strict-declared-version` (Python: `strict_declared_version=True`) to fail in the latter case.
42
+
43
+ The latest released schema of each spec minor version is bundled in `doclang/schemas/<MAJOR.MINOR>/`.
44
+ The schema directly in `doclang/` is the development head.
45
+
46
+ > **Note on 0.7:** the 0.7 XSD only accepts `0.7` in the root `version` attribute (later versions only
47
+ > require the `MAJOR.MINOR` format). So that a document declaring another version (e.g. `0.8`) can be
48
+ > checked against the 0.7 rules, the toolkit by default replaces the declared `version` with `0.7` in
49
+ > memory for XSD validation (the file is not modified), and says so in a note. The declared version itself
50
+ > is then not checked; all other 0.7 rules are still checked. Use `--strict-xsd-version-attr` (Python:
51
+ > `strict_xsd_version_attr=True`) to apply the XSD's version check as is, in which case such a document fails.
52
+
32
53
  #### More validation scenarios
33
54
 
34
55
  ```bash
@@ -44,6 +65,12 @@ doclang validate my_document.dclg --schematron-only
44
65
  # JSON output
45
66
  doclang validate my_document.dclg --format json
46
67
 
68
+ # Validate against a specific spec version (overrides the document's `version`)
69
+ doclang validate my_document.dclg --spec-version 0.7
70
+
71
+ # ... and apply the XSD's own `version` attribute check as is
72
+ doclang validate my_document.dclg --spec-version 0.7 --strict-xsd-version-attr
73
+
47
74
  # Quiet mode (exit code only)
48
75
  doclang validate my_document.dclg --quiet
49
76
 
@@ -68,6 +95,35 @@ doclang pack markup.dclg --assets payload/
68
95
  doclang pack markup.dclg --validate
69
96
  ```
70
97
 
98
+ ### Migration
99
+
100
+ Migrate a document to another spec version. Currently only older versions are supported (documents are forward
101
+ compatible, so a newer target needs no migration):
102
+
103
+ ```bash
104
+ doclang migrate document.dclg --to 0.7
105
+ ```
106
+
107
+ The migrated document goes to stdout (or `-o FILE`); what was changed goes to stderr. The migration is best effort:
108
+ content that has an equivalent in the target version is mapped (e.g. additional captions become `<text>`), the rest
109
+ is dropped (e.g. `<track>`), and every change is reported with its location.
110
+
111
+ #### More migration scenarios
112
+
113
+ ```bash
114
+ doclang migrate document.dclg --to 0.7 -o document_0.7.dclg
115
+ doclang migrate document.dclg --to 0.7 --strict # fail instead of changing anything
116
+ doclang migrate document.dclg --to 0.7 --format json # versions, changes and the document as JSON
117
+ doclang migrate document.dclg --to 0.7 --input-validation xsd --output-validation xsd # without Schematron
118
+ doclang migrate document.dclg --to 0.7 --input-validation none # skip validating the input
119
+ ```
120
+
121
+ By default the document is validated before the migration (against its own version) and the result afterwards
122
+ (against the target version), in full, XSD and Schematron. `--input-validation` and `--output-validation` take
123
+ `all`, `xsd`, `schematron` or `none`; `xsd` and `schematron` mean the same as `--xsd-only` and `--schematron-only`
124
+ of `doclang validate`. As there, Schematron needs a backend (see [Installation](#installation)): without one,
125
+ `doclang migrate` fails and tells you so, and you opt out explicitly, e.g. with the `xsd` scope for both.
126
+
71
127
  ## Python API
72
128
 
73
129
  ### Validation
@@ -141,6 +197,30 @@ path = pack(
141
197
  print(f"Created {path}")
142
198
  ```
143
199
 
200
+ ### Migration
201
+
202
+ ```python
203
+ from doclang import migrate, MigrationError
204
+
205
+ try:
206
+ result = migrate("document.dclg", to="0.7")
207
+ except MigrationError as exc:
208
+ print(exc)
209
+ else:
210
+ print(result.xml) # the migrated document
211
+ for change in result.changes: # {"kind": "mapped" | "dropped", "location": ..., "line": ..., "message": ...}
212
+ print(change["kind"], change["location"])
213
+ ```
214
+
215
+ `migrate(..., strict=True)` raises `MigrationError` (carrying `changes`) if anything would be changed. The source
216
+ version is the document's `version` attribute; without one the document is treated as the latest version, with a
217
+ warning.
218
+
219
+ The input is validated before and the result after the migration; `input_validation` and `output_validation`
220
+ (`"all"` by default, or `"xsd"`, `"schematron"`, `"none"`) select what runs. As with `validate()`, a missing
221
+ Schematron backend raises `SchematronBackendNotFound`. An invalid input raises `MigrationError`; so does an invalid
222
+ result, which indicates a bug in a migration step (or an invalid input that was not validated).
223
+
144
224
  ## Validation Rules
145
225
 
146
226
  ### XSD Validation (doclang.xsd)
@@ -1,15 +1,22 @@
1
1
  """DocLang reference toolkit."""
2
2
 
3
+ from doclang.migration import MigrationError, MigrationResult, migrate
3
4
  from doclang.packaging import PackagingError, pack
4
5
  from doclang.schematron import SchematronBackendNotFound, SchematronValidator, SchematronViolation
5
- from doclang.validation import ValidationError, validate
6
+ from doclang.tokenization import get_special_tokens
7
+ from doclang.validation import UnsupportedVersionError, ValidationError, validate
6
8
 
7
9
  __all__ = [
10
+ "MigrationError",
11
+ "MigrationResult",
8
12
  "PackagingError",
9
13
  "SchematronBackendNotFound",
10
14
  "SchematronValidator",
11
15
  "SchematronViolation",
16
+ "UnsupportedVersionError",
12
17
  "ValidationError",
18
+ "get_special_tokens",
19
+ "migrate",
13
20
  "pack",
14
21
  "validate",
15
22
  ]
@@ -17,6 +17,17 @@ _CONTENT_TYPES_XML = """\
17
17
  <Default Extension="jpg" ContentType="image/jpeg"/>
18
18
  <Default Extension="jpeg" ContentType="image/jpeg"/>
19
19
  <Default Extension="webp" ContentType="image/webp"/>
20
+ <Default Extension="mp3" ContentType="audio/mpeg"/>
21
+ <Default Extension="m4a" ContentType="audio/mp4"/>
22
+ <Default Extension="aac" ContentType="audio/aac"/>
23
+ <Default Extension="ogg" ContentType="audio/ogg"/>
24
+ <Default Extension="opus" ContentType="audio/ogg"/>
25
+ <Default Extension="wav" ContentType="audio/wav"/>
26
+ <Default Extension="flac" ContentType="audio/flac"/>
27
+ <Default Extension="mp4" ContentType="video/mp4"/>
28
+ <Default Extension="webm" ContentType="video/webm"/>
29
+ <Default Extension="mkv" ContentType="video/x-matroska"/>
30
+ <Default Extension="mov" ContentType="video/quicktime"/>
20
31
  <Override PartName="/document.xml" ContentType="application/vnd.doclang.document+xml"/>
21
32
  </Types>
22
33
  """
@@ -37,6 +48,9 @@ PagesInput = Union[
37
48
  Mapping[int, Union[str, Path]],
38
49
  ]
39
50
 
51
+ # Whole-track media (audio/, video/) uses the same 1-based positional model as pages/.
52
+ MediaInput = PagesInput
53
+
40
54
  AssetsInput = Union[
41
55
  str,
42
56
  Path,
@@ -58,6 +72,11 @@ def _require_directory(path: Path, *, label: str) -> None:
58
72
  raise PackagingError(f"{label} not found or not a directory: {path}")
59
73
 
60
74
 
75
+ def _reject_symlink(path: Path, *, label: str) -> None:
76
+ if path.is_symlink():
77
+ raise PackagingError(f"{label} must not be a symbolic link: {path}")
78
+
79
+
61
80
  def _validate_archive_relative_path(path: str, *, label: str) -> None:
62
81
  if not path or path.startswith("/") or "\\" in path:
63
82
  raise PackagingError(f"Invalid {label} path: {path!r}")
@@ -66,12 +85,22 @@ def _validate_archive_relative_path(path: str, *, label: str) -> None:
66
85
  raise PackagingError(f"Invalid {label} path: {path!r}")
67
86
 
68
87
 
69
- def _copy_tree_into(source: Path, destination: Path) -> None:
88
+ def _copy_file(source: Path, destination: Path, *, label: str) -> None:
89
+ _reject_symlink(source, label=label)
90
+ _require_file(source, label=label)
91
+ destination.parent.mkdir(parents=True, exist_ok=True)
92
+ shutil.copy2(source, destination)
93
+
94
+
95
+ def _copy_tree_into(source: Path, destination: Path, *, label: str) -> None:
96
+ _reject_symlink(source, label=label)
97
+ _require_directory(source, label=label)
70
98
  destination.mkdir(parents=True, exist_ok=True)
71
99
  for item in source.iterdir():
72
100
  target = destination / item.name
101
+ _reject_symlink(item, label=label)
73
102
  if item.is_dir():
74
- shutil.copytree(item, target, dirs_exist_ok=True)
103
+ _copy_tree_into(item, target, label=label)
75
104
  else:
76
105
  shutil.copy2(item, target)
77
106
 
@@ -81,31 +110,71 @@ def _place_document(stage: Path, document: Path) -> None:
81
110
  shutil.copy2(document, stage / "document.xml")
82
111
 
83
112
 
84
- def _place_pages(stage: Path, pages: PagesInput) -> None:
85
- pages_dir = stage / "pages"
86
- if isinstance(pages, Mapping):
87
- for page_number, source in pages.items():
88
- if not isinstance(page_number, int) or page_number < 1:
89
- raise PackagingError(f"Page numbers must be positive integers, got {page_number!r}")
113
+ def _place_indexed(
114
+ stage: Path,
115
+ spec: PagesInput,
116
+ *,
117
+ subdir: str,
118
+ file_label: str,
119
+ dir_label: str,
120
+ number_label: str,
121
+ ) -> None:
122
+ """Place 1-based positionally-indexed files (``{N}.{ext}``) under ``stage/subdir``.
123
+
124
+ Shared by ``pages/`` (page images) and ``audio/`` / ``video/`` (whole-track media),
125
+ which all key a file to an integer position in the document.
126
+ """
127
+ target_dir = stage / subdir
128
+ if isinstance(spec, Mapping):
129
+ for number, source in spec.items():
130
+ if not isinstance(number, int) or number < 1:
131
+ raise PackagingError(f"{number_label} must be positive integers, got {number!r}")
90
132
  source_path = Path(source)
91
- _require_file(source_path, label="Page file")
92
- destination = pages_dir / f"{page_number}{source_path.suffix}"
93
- destination.parent.mkdir(parents=True, exist_ok=True)
94
- shutil.copy2(source_path, destination)
133
+ destination = target_dir / f"{number}{source_path.suffix}"
134
+ _copy_file(source_path, destination, label=file_label)
95
135
  return
96
136
 
97
- if isinstance(pages, str | Path):
98
- source_dir = Path(pages)
99
- _require_directory(source_dir, label="Pages directory")
100
- _copy_tree_into(source_dir, pages_dir)
137
+ if isinstance(spec, str | Path):
138
+ _copy_tree_into(Path(spec), target_dir, label=dir_label)
101
139
  return
102
140
 
103
- for index, source in enumerate(pages, start=1):
141
+ for index, source in enumerate(spec, start=1):
104
142
  source_path = Path(source)
105
- _require_file(source_path, label="Page file")
106
- destination = pages_dir / f"{index}{source_path.suffix}"
107
- destination.parent.mkdir(parents=True, exist_ok=True)
108
- shutil.copy2(source_path, destination)
143
+ destination = target_dir / f"{index}{source_path.suffix}"
144
+ _copy_file(source_path, destination, label=file_label)
145
+
146
+
147
+ def _place_pages(stage: Path, pages: PagesInput) -> None:
148
+ _place_indexed(
149
+ stage,
150
+ pages,
151
+ subdir="pages",
152
+ file_label="Page file",
153
+ dir_label="Pages directory",
154
+ number_label="Page numbers",
155
+ )
156
+
157
+
158
+ def _place_audio(stage: Path, audio: MediaInput) -> None:
159
+ _place_indexed(
160
+ stage,
161
+ audio,
162
+ subdir="audio",
163
+ file_label="Audio file",
164
+ dir_label="Audio directory",
165
+ number_label="Audio track numbers",
166
+ )
167
+
168
+
169
+ def _place_video(stage: Path, video: MediaInput) -> None:
170
+ _place_indexed(
171
+ stage,
172
+ video,
173
+ subdir="video",
174
+ file_label="Video file",
175
+ dir_label="Video directory",
176
+ number_label="Video track numbers",
177
+ )
109
178
 
110
179
 
111
180
  def _place_assets(stage: Path, assets: AssetsInput) -> None:
@@ -113,16 +182,11 @@ def _place_assets(stage: Path, assets: AssetsInput) -> None:
113
182
  if isinstance(assets, Mapping):
114
183
  for archive_path, source in assets.items():
115
184
  _validate_archive_relative_path(archive_path, label="asset")
116
- source_path = Path(source)
117
- _require_file(source_path, label="Asset file")
118
185
  destination = assets_dir / archive_path
119
- destination.parent.mkdir(parents=True, exist_ok=True)
120
- shutil.copy2(source_path, destination)
186
+ _copy_file(Path(source), destination, label="Asset file")
121
187
  return
122
188
 
123
- source_dir = Path(assets)
124
- _require_directory(source_dir, label="Assets directory")
125
- _copy_tree_into(source_dir, assets_dir)
189
+ _copy_tree_into(Path(assets), assets_dir, label="Assets directory")
126
190
 
127
191
 
128
192
  def _write_opc_metadata(stage: Path) -> None:
@@ -160,6 +224,8 @@ def _pack(
160
224
  output: Union[str, Path, None] = None,
161
225
  pages: PagesInput | None = None,
162
226
  assets: AssetsInput | None = None,
227
+ audio: MediaInput | None = None,
228
+ video: MediaInput | None = None,
163
229
  validate: bool = False,
164
230
  ) -> Path:
165
231
  document_path = Path(document)
@@ -172,6 +238,10 @@ def _pack(
172
238
  _place_pages(stage, pages)
173
239
  if assets is not None:
174
240
  _place_assets(stage, assets)
241
+ if audio is not None:
242
+ _place_audio(stage, audio)
243
+ if video is not None:
244
+ _place_video(stage, video)
175
245
  _write_opc_metadata(stage)
176
246
 
177
247
  if validate:
@@ -0,0 +1,195 @@
1
+ """Bundled DocLang schema paths and spec-version selection (internal).
2
+
3
+ ``schemas/<MAJOR.MINOR>/`` holds the latest released schemas of each spec minor version.
4
+ The schemas directly in the package directory are the development head: they are used when no
5
+ version is specified, and as a fallback for a declared version that has no ``schemas/`` folder.
6
+ At release, the head is copied to ``schemas/<MAJOR.MINOR>/`` (see ``utils/prepare_release.py``).
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import re
12
+ from dataclasses import dataclass
13
+ from pathlib import Path
14
+
15
+ from lxml import etree
16
+
17
+ _SCHEMA_DIR = Path(__file__).resolve().parent
18
+ _VERSION_RE = re.compile(r"^(\d+)\.(\d+)$")
19
+ _SCHEMA_VERSION_RE = re.compile(r'<xs:schema[^>]*\sversion="(\d+)\.(\d+)\.\d+"', re.DOTALL)
20
+
21
+
22
+ _XS_NS = "http://www.w3.org/2001/XMLSchema"
23
+
24
+
25
+ class UnsupportedVersionError(ValueError):
26
+ """Raised when there are no schemas for a requested or declared DocLang spec version."""
27
+
28
+
29
+ @dataclass(frozen=True)
30
+ class SchemaSelection:
31
+ """The schemas chosen for a document."""
32
+
33
+ version: str # version of the chosen schemas (for the head: its label)
34
+ xsd: Path
35
+ sch: Path
36
+ declared: bool # False when no version was requested or declared (head used)
37
+ head_fallback_for: str | None = None # declared version without schemas, for which the head was used
38
+
39
+
40
+ def _bundled_xsd_path() -> Path:
41
+ path = _SCHEMA_DIR / "doclang.xsd"
42
+ if not path.exists():
43
+ raise FileNotFoundError(f"Bundled XSD schema not found: {path}")
44
+ return path
45
+
46
+
47
+ def _bundled_sch_path() -> Path:
48
+ path = _SCHEMA_DIR / "doclang.sch"
49
+ if not path.exists():
50
+ raise FileNotFoundError(f"Bundled Schematron schema not found: {path}")
51
+ return path
52
+
53
+
54
+ def _bundled_schema_paths() -> tuple[Path, Path]:
55
+ return _bundled_xsd_path(), _bundled_sch_path()
56
+
57
+
58
+ def _version_key(version: str) -> tuple[int, int]:
59
+ match = _VERSION_RE.match(version)
60
+ assert match is not None
61
+ return int(match.group(1)), int(match.group(2))
62
+
63
+
64
+ def _schema_version(xsd: Path) -> str:
65
+ """The ``MAJOR.MINOR`` spec version of an XSD, from its ``xs:schema`` version attribute."""
66
+ match = _SCHEMA_VERSION_RE.search(xsd.read_text(encoding="utf-8"))
67
+ if match is None:
68
+ raise FileNotFoundError(f"No schema version found in {xsd}")
69
+ return f"{match.group(1)}.{match.group(2)}"
70
+
71
+
72
+ def _current_version() -> str:
73
+ """The spec version of the top-level (latest) schemas."""
74
+ return _schema_version(_bundled_xsd_path())
75
+
76
+
77
+ def _versioned_dir(version: str) -> Path:
78
+ return _SCHEMA_DIR / "schemas" / version
79
+
80
+
81
+ def _has_versioned_schemas(version: str) -> bool:
82
+ folder = _versioned_dir(version)
83
+ return (folder / "doclang.xsd").exists() and (folder / "doclang.sch").exists()
84
+
85
+
86
+ def _available_versions() -> list[str]:
87
+ """Versions with bundled ``schemas/`` folders, oldest first."""
88
+ versioned_dir = _SCHEMA_DIR / "schemas"
89
+ if not versioned_dir.is_dir():
90
+ return []
91
+ versions = [e.name for e in versioned_dir.iterdir() if _VERSION_RE.match(e.name) and _has_versioned_schemas(e.name)]
92
+ return sorted(versions, key=_version_key)
93
+
94
+
95
+ def _resolve_in_folders(requested: str) -> str | None:
96
+ """Map a requested ``MAJOR.MINOR`` to a version with bundled schemas, or None.
97
+
98
+ An exact match wins. From major version 1, an instance is compatible with any later minor
99
+ of the same major, so the nearest newer minor with schemas is used.
100
+ """
101
+ if not _VERSION_RE.match(requested):
102
+ return None
103
+ available = _available_versions()
104
+ if requested in available:
105
+ return requested
106
+ major, minor = _version_key(requested)
107
+ if major >= 1:
108
+ for candidate in available:
109
+ if _version_key(candidate)[0] == major and _version_key(candidate)[1] > minor:
110
+ return candidate
111
+ return None
112
+
113
+
114
+ def _versioned_paths(version: str) -> tuple[Path, Path]:
115
+ folder = _versioned_dir(version)
116
+ return folder / "doclang.xsd", folder / "doclang.sch"
117
+
118
+
119
+ def _head_description() -> str:
120
+ """How to refer to the head in messages: its version if identical to the released schema of that
121
+ version, else "unreleased development version" (the head's label is then not reliable)."""
122
+ label = _current_version()
123
+ if _has_versioned_schemas(label):
124
+ released_xsd, released_sch = _versioned_paths(label)
125
+ head_xsd, head_sch = _bundled_schema_paths()
126
+ if released_xsd.read_bytes() == head_xsd.read_bytes() and released_sch.read_bytes() == head_sch.read_bytes():
127
+ return label
128
+ return "unreleased development version"
129
+
130
+
131
+ def _enumerated_versions(xsd: Path) -> set[str]:
132
+ """Versions the XSD enumerates for the root ``version`` attribute (empty if only format-constrained)."""
133
+ parser = etree.XMLParser(resolve_entities=False, no_network=True)
134
+ tree = etree.parse(str(xsd), parser)
135
+ values = tree.xpath("//xs:attribute[@name='version']//xs:enumeration/@value", namespaces={"xs": _XS_NS})
136
+ return {str(v) for v in values} # type: ignore[union-attr]
137
+
138
+
139
+ def _declared_version(xml_file: str | Path) -> str | None:
140
+ """The ``version`` attribute of the root element, or None if absent or unreadable."""
141
+ try:
142
+ with open(xml_file, "rb") as f:
143
+ _, root = next(
144
+ etree.iterparse(f, events=("start",), resolve_entities=False, no_network=True, load_dtd=False)
145
+ )
146
+ return root.get("version")
147
+ except Exception:
148
+ return None
149
+
150
+
151
+ def _select_schemas(
152
+ xml_file: str | Path,
153
+ spec_version: str | None = None,
154
+ *,
155
+ strict_declared_version: bool = False,
156
+ ) -> SchemaSelection:
157
+ """Pick schemas for a document.
158
+
159
+ 1. An explicit ``spec_version`` uses the bundled schemas of that version (else UnsupportedVersionError).
160
+ 2. Otherwise the document's declared ``version`` uses the bundled schemas of that version.
161
+ 3. Otherwise the development head is used. For a declared version without schemas this is a
162
+ fallback, unless ``strict_declared_version`` is set (then UnsupportedVersionError).
163
+ """
164
+ if spec_version is not None:
165
+ version = _resolve_in_folders(spec_version)
166
+ if version is None:
167
+ available = ", ".join(_available_versions()) or "none"
168
+ raise UnsupportedVersionError(
169
+ f"No schema for DocLang spec version '{spec_version}'; available versions: {available}"
170
+ )
171
+ xsd, sch = _versioned_paths(version)
172
+ return SchemaSelection(version=version, xsd=xsd, sch=sch, declared=True)
173
+
174
+ declared_version = _declared_version(xml_file)
175
+ head_xsd, head_sch = _bundled_schema_paths()
176
+ if declared_version is None:
177
+ return SchemaSelection(version=_current_version(), xsd=head_xsd, sch=head_sch, declared=False)
178
+
179
+ version = _resolve_in_folders(declared_version)
180
+ if version is not None:
181
+ xsd, sch = _versioned_paths(version)
182
+ return SchemaSelection(version=version, xsd=xsd, sch=sch, declared=True)
183
+
184
+ if strict_declared_version:
185
+ available = ", ".join(_available_versions()) or "none"
186
+ raise UnsupportedVersionError(
187
+ f"No schema for declared DocLang version '{declared_version}'; available versions: {available}"
188
+ )
189
+ return SchemaSelection(
190
+ version=_current_version(),
191
+ xsd=head_xsd,
192
+ sch=head_sch,
193
+ declared=True,
194
+ head_fallback_for=declared_version,
195
+ )