mdxml 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mdxml/__init__.py +39 -0
- mdxml/__main__.py +5 -0
- mdxml/_cli.py +96 -0
- mdxml/_convert.py +151 -0
- mdxml/_dict.py +42 -0
- mdxml/_errors.py +39 -0
- mdxml/_grammar.py +259 -0
- mdxml/_header.py +92 -0
- mdxml/_lookup.py +54 -0
- mdxml/_protocols.py +135 -0
- mdxml/_read.py +75 -0
- mdxml/_reader.py +191 -0
- mdxml/_schema.py +131 -0
- mdxml/_source.py +192 -0
- mdxml/_variables.py +426 -0
- mdxml/fmi2/__init__.py +73 -0
- mdxml/fmi2/_annotations.py +34 -0
- mdxml/fmi2/_header.py +525 -0
- mdxml/fmi2/_model.py +73 -0
- mdxml/fmi2/_structure.py +100 -0
- mdxml/fmi2/_variables.py +323 -0
- mdxml/fmi3/__init__.py +125 -0
- mdxml/fmi3/_annotations.py +43 -0
- mdxml/fmi3/_header.py +635 -0
- mdxml/fmi3/_model.py +59 -0
- mdxml/fmi3/_structure.py +103 -0
- mdxml/fmi3/_variables.py +603 -0
- mdxml/py.typed +0 -0
- mdxml-0.1.0.dist-info/METADATA +202 -0
- mdxml-0.1.0.dist-info/RECORD +33 -0
- mdxml-0.1.0.dist-info/WHEEL +4 -0
- mdxml-0.1.0.dist-info/entry_points.txt +3 -0
- mdxml-0.1.0.dist-info/licenses/LICENSE +21 -0
mdxml/_header.py
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
"""Reading the Model Header: parse up to ``<ModelVariables>`` and stop."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Callable
|
|
6
|
+
|
|
7
|
+
from mdxml import _reader
|
|
8
|
+
from mdxml._errors import UnsupportedFmiVersion
|
|
9
|
+
from mdxml._grammar import Grammar, TreeHandler
|
|
10
|
+
from mdxml._schema import Frame
|
|
11
|
+
from mdxml._source import Source, open_source
|
|
12
|
+
from mdxml.fmi2._header import HEADER_GRAMMAR as FMI2_HEADER_GRAMMAR
|
|
13
|
+
from mdxml.fmi2._header import HEADER_GRAMMAR_WITH_ANNOTATIONS as FMI2_HEADER_GRAMMAR_WITH_ANNOTATIONS
|
|
14
|
+
from mdxml.fmi2._header import ModelHeader as Fmi2ModelHeader
|
|
15
|
+
from mdxml.fmi3._header import HEADER_GRAMMAR as FMI3_HEADER_GRAMMAR
|
|
16
|
+
from mdxml.fmi3._header import ModelHeader as Fmi3ModelHeader
|
|
17
|
+
|
|
18
|
+
ROOT = "fmiModelDescription"
|
|
19
|
+
|
|
20
|
+
AnyModelHeader = Fmi2ModelHeader | Fmi3ModelHeader
|
|
21
|
+
"""The Model Header of either FMI Version."""
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def header_grammar_chooser(*, annotations: bool) -> Callable[[Frame], Grammar[AnyModelHeader]]:
|
|
25
|
+
"""Return a function picking the Model Header grammar for the root element's FMI Version."""
|
|
26
|
+
fmi2_grammar = FMI2_HEADER_GRAMMAR_WITH_ANNOTATIONS if annotations else FMI2_HEADER_GRAMMAR
|
|
27
|
+
|
|
28
|
+
def choose(root: Frame) -> Grammar[AnyModelHeader]:
|
|
29
|
+
return _choose_header_grammar(root, fmi2_grammar)
|
|
30
|
+
|
|
31
|
+
return choose
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _choose_header_grammar(root: Frame, fmi2_grammar: Grammar[Fmi2ModelHeader]) -> Grammar[AnyModelHeader]:
|
|
35
|
+
if root.tag != ROOT:
|
|
36
|
+
raise root.error(f"root element is <{root.tag}>, expected <{ROOT}>")
|
|
37
|
+
fmi_version = root.attrs.get("fmiVersion")
|
|
38
|
+
if fmi_version is None:
|
|
39
|
+
raise root.error(f"<{ROOT}> has no fmiVersion attribute")
|
|
40
|
+
major = fmi_version.strip().split(".", 1)[0]
|
|
41
|
+
grammar: Grammar[AnyModelHeader]
|
|
42
|
+
if major == "2":
|
|
43
|
+
grammar = fmi2_grammar
|
|
44
|
+
elif major == "3":
|
|
45
|
+
grammar = FMI3_HEADER_GRAMMAR
|
|
46
|
+
else:
|
|
47
|
+
raise UnsupportedFmiVersion(fmi_version)
|
|
48
|
+
return grammar
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def read_header(
|
|
52
|
+
src: Source,
|
|
53
|
+
*,
|
|
54
|
+
member: str | None = None,
|
|
55
|
+
strict: bool = False,
|
|
56
|
+
annotations: bool = False,
|
|
57
|
+
max_xml_bytes: int = _reader.DEFAULT_MAX_XML_BYTES,
|
|
58
|
+
) -> AnyModelHeader:
|
|
59
|
+
"""Read the Model Header of an FMU.
|
|
60
|
+
|
|
61
|
+
Returns an :class:`mdxml.fmi2.ModelHeader` or :class:`mdxml.fmi3.ModelHeader`,
|
|
62
|
+
depending on the FMI Version declared in the Model Description.
|
|
63
|
+
|
|
64
|
+
``src`` is a path to a ``.fmu`` zip, an unzipped FMU directory or a
|
|
65
|
+
``modelDescription.xml`` file; a binary file-like object; or bytes. Zips
|
|
66
|
+
are recognised by their content, not their extension. ``member`` overrides
|
|
67
|
+
where the Model Description lives inside a zip or directory.
|
|
68
|
+
|
|
69
|
+
Parsing stops at ``<ModelVariables>``, so the cost doesn't depend on how
|
|
70
|
+
many Model Variables the FMU has.
|
|
71
|
+
|
|
72
|
+
By default parsing is lenient (ADR 0003): unknown elements and attributes
|
|
73
|
+
are ignored, booleans may be written in any case, out-of-range enum values
|
|
74
|
+
stay plain strings, and optional values that can't be read are left at
|
|
75
|
+
their default. With ``strict=True`` all of those raise
|
|
76
|
+
:class:`~mdxml.ParseError`, as do missing required attributes.
|
|
77
|
+
|
|
78
|
+
With ``annotations=True``, an FMI 2 header's ``vendor_annotations`` holds
|
|
79
|
+
its Tool Annotations as raw XML; otherwise they're skipped and left as
|
|
80
|
+
``None``. (FMI 3's model-level annotations come after the Model Structure,
|
|
81
|
+
so only :func:`mdxml.read` reaches them.)
|
|
82
|
+
|
|
83
|
+
Hostile input raises :class:`~mdxml.UnsafeInput` in both modes: any
|
|
84
|
+
``DOCTYPE`` or entity declaration, or XML larger than ``max_xml_bytes``
|
|
85
|
+
(256 MB by default; for a zip, the decompressed size counts).
|
|
86
|
+
"""
|
|
87
|
+
_reader.check_max_xml_bytes(max_xml_bytes)
|
|
88
|
+
position: list[_reader.Position] = []
|
|
89
|
+
handler = TreeHandler(header_grammar_chooser(annotations=annotations), position, strict=strict)
|
|
90
|
+
with open_source(src, member, max_xml_bytes) as stream:
|
|
91
|
+
_reader.run(stream, handler, position, max_xml_bytes=max_xml_bytes)
|
|
92
|
+
return handler.finish()
|
mdxml/_lookup.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""Tuples of named definitions that can also be looked up by name."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Protocol, SupportsIndex, TypeVar, overload
|
|
6
|
+
|
|
7
|
+
T_co = TypeVar("T_co", bound="HasName", covariant=True)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class HasName(Protocol):
|
|
11
|
+
"""Anything with a ``name``, like a unit or type definition."""
|
|
12
|
+
|
|
13
|
+
@property
|
|
14
|
+
def name(self) -> str: ...
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class Named(tuple[T_co, ...]):
|
|
18
|
+
"""A tuple of definitions in document order that also accepts a name as the key.
|
|
19
|
+
|
|
20
|
+
``units["m/s"]`` returns the first definition with that name and raises
|
|
21
|
+
:class:`KeyError` when there's none. Integer indexes and slices work as for
|
|
22
|
+
any tuple, and a ``Named`` compares equal to a plain tuple with the same items.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
__slots__ = ()
|
|
26
|
+
|
|
27
|
+
@overload
|
|
28
|
+
def __getitem__(self, key: SupportsIndex) -> T_co: ...
|
|
29
|
+
@overload
|
|
30
|
+
def __getitem__(self, key: slice) -> tuple[T_co, ...]: ...
|
|
31
|
+
@overload
|
|
32
|
+
def __getitem__(self, key: str) -> T_co: ...
|
|
33
|
+
|
|
34
|
+
def __getitem__(self, key: SupportsIndex | slice | str) -> T_co | tuple[T_co, ...]:
|
|
35
|
+
"""Return the item at an index, the items in a slice, or the first item with a name."""
|
|
36
|
+
if isinstance(key, str):
|
|
37
|
+
for item in self:
|
|
38
|
+
if item.name == key:
|
|
39
|
+
return item
|
|
40
|
+
raise KeyError(key)
|
|
41
|
+
return super().__getitem__(key)
|
|
42
|
+
|
|
43
|
+
def __contains__(self, key: object) -> bool:
|
|
44
|
+
"""Check for an item, or for a definition with that name."""
|
|
45
|
+
if isinstance(key, str):
|
|
46
|
+
return any(item.name == key for item in self)
|
|
47
|
+
return super().__contains__(key)
|
|
48
|
+
|
|
49
|
+
def get(self, name: str) -> T_co | None:
|
|
50
|
+
"""Return the first definition with ``name``, or ``None``."""
|
|
51
|
+
for item in self:
|
|
52
|
+
if item.name == name:
|
|
53
|
+
return item
|
|
54
|
+
return None
|
mdxml/_protocols.py
ADDED
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
"""Version-independent views of a Model Description.
|
|
2
|
+
|
|
3
|
+
These :class:`typing.Protocol` classes describe what FMI 2.0 and FMI 3.0
|
|
4
|
+
objects have in common, so code that doesn't care about the FMI Version can be
|
|
5
|
+
written (and type-checked) once. The concrete classes in :mod:`mdxml.fmi2` and
|
|
6
|
+
:mod:`mdxml.fmi3` satisfy them structurally; they don't inherit from them or
|
|
7
|
+
from each other.
|
|
8
|
+
|
|
9
|
+
Fixed-choice values (``causality``, ``variability``) are typed as ``str``:
|
|
10
|
+
each version's enums are ``str`` subclasses, and values outside the allowed
|
|
11
|
+
set stay plain strings in lenient mode.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from datetime import datetime
|
|
17
|
+
from typing import Protocol, runtime_checkable
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@runtime_checkable
|
|
21
|
+
class InterfaceType(Protocol):
|
|
22
|
+
"""What FMI 2.0 and FMI 3.0 Model Exchange and Co-Simulation Interface Types share."""
|
|
23
|
+
|
|
24
|
+
@property
|
|
25
|
+
def model_identifier(self) -> str:
|
|
26
|
+
"""Short name of the FMU's binaries and functions."""
|
|
27
|
+
...
|
|
28
|
+
|
|
29
|
+
@property
|
|
30
|
+
def needs_execution_tool(self) -> bool:
|
|
31
|
+
"""Whether the FMU needs its execution tool installed."""
|
|
32
|
+
...
|
|
33
|
+
|
|
34
|
+
@property
|
|
35
|
+
def can_be_instantiated_only_once_per_process(self) -> bool:
|
|
36
|
+
"""Whether only one instance per process is allowed."""
|
|
37
|
+
...
|
|
38
|
+
|
|
39
|
+
@property
|
|
40
|
+
def can_get_and_set_fmu_state(self) -> bool:
|
|
41
|
+
"""Whether the FMU state can be saved and restored."""
|
|
42
|
+
...
|
|
43
|
+
|
|
44
|
+
@property
|
|
45
|
+
def can_serialize_fmu_state(self) -> bool:
|
|
46
|
+
"""Whether the FMU state can be serialized."""
|
|
47
|
+
...
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@runtime_checkable
|
|
51
|
+
class ModelHeader(Protocol):
|
|
52
|
+
"""What FMI 2.0 and FMI 3.0 Model Headers share.
|
|
53
|
+
|
|
54
|
+
FMI 2.0's ``guid`` is available as ``instantiation_token``, its FMI 3.0 name.
|
|
55
|
+
"""
|
|
56
|
+
|
|
57
|
+
@property
|
|
58
|
+
def fmi_version(self) -> str:
|
|
59
|
+
"""The ``fmiVersion`` attribute, as written."""
|
|
60
|
+
...
|
|
61
|
+
|
|
62
|
+
@property
|
|
63
|
+
def model_name(self) -> str:
|
|
64
|
+
"""The ``modelName`` attribute."""
|
|
65
|
+
...
|
|
66
|
+
|
|
67
|
+
@property
|
|
68
|
+
def instantiation_token(self) -> str:
|
|
69
|
+
"""FMI 3.0's ``instantiationToken``, or FMI 2.0's ``guid``."""
|
|
70
|
+
...
|
|
71
|
+
|
|
72
|
+
@property
|
|
73
|
+
def description(self) -> str | None:
|
|
74
|
+
"""The model's description, if any."""
|
|
75
|
+
...
|
|
76
|
+
|
|
77
|
+
@property
|
|
78
|
+
def generation_tool(self) -> str | None:
|
|
79
|
+
"""The tool that generated the FMU, if given."""
|
|
80
|
+
...
|
|
81
|
+
|
|
82
|
+
@property
|
|
83
|
+
def generation_date_and_time(self) -> datetime | None:
|
|
84
|
+
"""When the FMU was generated, if given."""
|
|
85
|
+
...
|
|
86
|
+
|
|
87
|
+
@property
|
|
88
|
+
def model_exchange(self) -> InterfaceType | None:
|
|
89
|
+
"""The Model Exchange Interface Type, if the FMU supports it."""
|
|
90
|
+
...
|
|
91
|
+
|
|
92
|
+
@property
|
|
93
|
+
def co_simulation(self) -> InterfaceType | None:
|
|
94
|
+
"""The Co-Simulation Interface Type, if the FMU supports it."""
|
|
95
|
+
...
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
@runtime_checkable
|
|
99
|
+
class ModelVariable(Protocol):
|
|
100
|
+
"""What FMI 2.0 and FMI 3.0 Model Variables share."""
|
|
101
|
+
|
|
102
|
+
@property
|
|
103
|
+
def name(self) -> str:
|
|
104
|
+
"""The variable's name."""
|
|
105
|
+
...
|
|
106
|
+
|
|
107
|
+
@property
|
|
108
|
+
def value_reference(self) -> int:
|
|
109
|
+
"""The variable's value reference."""
|
|
110
|
+
...
|
|
111
|
+
|
|
112
|
+
@property
|
|
113
|
+
def causality(self) -> str:
|
|
114
|
+
"""The variable's causality, e.g. ``"output"``."""
|
|
115
|
+
...
|
|
116
|
+
|
|
117
|
+
@property
|
|
118
|
+
def variability(self) -> str:
|
|
119
|
+
"""The variable's variability, e.g. ``"continuous"``."""
|
|
120
|
+
...
|
|
121
|
+
|
|
122
|
+
@property
|
|
123
|
+
def description(self) -> str | None:
|
|
124
|
+
"""The variable's description, if any."""
|
|
125
|
+
...
|
|
126
|
+
|
|
127
|
+
@property
|
|
128
|
+
def index(self) -> int:
|
|
129
|
+
"""The Variable Index: 1-based position in ``ModelVariables``."""
|
|
130
|
+
...
|
|
131
|
+
|
|
132
|
+
@property
|
|
133
|
+
def type_name(self) -> str:
|
|
134
|
+
"""The variable's kind: ``"Real"`` in FMI 2.0, ``"Float64"`` in FMI 3.0, and so on."""
|
|
135
|
+
...
|
mdxml/_read.py
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"""Reading a whole Model Description into memory."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import cast
|
|
6
|
+
|
|
7
|
+
from mdxml import _reader
|
|
8
|
+
from mdxml._errors import ParseError
|
|
9
|
+
from mdxml._source import Source
|
|
10
|
+
from mdxml._variables import VariableFilter, VariableIterator
|
|
11
|
+
from mdxml.fmi2._header import ModelHeader as Fmi2ModelHeader
|
|
12
|
+
from mdxml.fmi2._model import ModelDescription as Fmi2ModelDescription
|
|
13
|
+
from mdxml.fmi2._structure import ModelStructure as Fmi2ModelStructure
|
|
14
|
+
from mdxml.fmi2._variables import Variable as Fmi2Variable
|
|
15
|
+
from mdxml.fmi3._header import ModelHeader as Fmi3ModelHeader
|
|
16
|
+
from mdxml.fmi3._model import ModelDescription as Fmi3ModelDescription
|
|
17
|
+
from mdxml.fmi3._structure import ModelStructure as Fmi3ModelStructure
|
|
18
|
+
from mdxml.fmi3._variables import Variable as Fmi3Variable
|
|
19
|
+
|
|
20
|
+
AnyModelDescription = Fmi2ModelDescription | Fmi3ModelDescription
|
|
21
|
+
"""The Model Description of either FMI Version."""
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def read(
|
|
25
|
+
src: Source,
|
|
26
|
+
*,
|
|
27
|
+
member: str | None = None,
|
|
28
|
+
strict: bool = False,
|
|
29
|
+
annotations: bool = False,
|
|
30
|
+
max_xml_bytes: int = _reader.DEFAULT_MAX_XML_BYTES,
|
|
31
|
+
) -> AnyModelDescription:
|
|
32
|
+
"""Read an FMU's whole Model Description: Model Header, Model Variables and Model Structure.
|
|
33
|
+
|
|
34
|
+
``src``, ``member``, ``strict`` and ``max_xml_bytes`` work as for
|
|
35
|
+
:func:`mdxml.read_header`. The document is read in one streaming pass and
|
|
36
|
+
only the parsed objects are kept, never the XML.
|
|
37
|
+
|
|
38
|
+
Annotations are skipped unless ``annotations=True``. Then the FMI 2
|
|
39
|
+
header's ``vendor_annotations``, FMI 3's model-level ``annotations`` and
|
|
40
|
+
every variable's ``annotations`` hold their content as raw XML.
|
|
41
|
+
|
|
42
|
+
Returns an :class:`mdxml.fmi2.ModelDescription` or
|
|
43
|
+
:class:`mdxml.fmi3.ModelDescription`, depending on the FMI Version.
|
|
44
|
+
"""
|
|
45
|
+
_reader.check_max_xml_bytes(max_xml_bytes)
|
|
46
|
+
no_filter = VariableFilter(causality=None, variability=None, name_prefix=None)
|
|
47
|
+
iterator = VariableIterator(
|
|
48
|
+
src,
|
|
49
|
+
member=member,
|
|
50
|
+
strict=strict,
|
|
51
|
+
max_xml_bytes=max_xml_bytes,
|
|
52
|
+
variable_filter=no_filter,
|
|
53
|
+
read_structure=True,
|
|
54
|
+
annotations=annotations,
|
|
55
|
+
)
|
|
56
|
+
with iterator:
|
|
57
|
+
variables = tuple(iterator)
|
|
58
|
+
header = iterator.header
|
|
59
|
+
structure = iterator.model_structure
|
|
60
|
+
if structure is None and strict:
|
|
61
|
+
raise ParseError("<fmiModelDescription> has no <ModelStructure> element")
|
|
62
|
+
|
|
63
|
+
if isinstance(header, Fmi2ModelHeader):
|
|
64
|
+
return Fmi2ModelDescription(
|
|
65
|
+
header=header,
|
|
66
|
+
variables=cast("tuple[Fmi2Variable, ...]", variables),
|
|
67
|
+
model_structure=structure or Fmi2ModelStructure(),
|
|
68
|
+
)
|
|
69
|
+
assert isinstance(header, Fmi3ModelHeader) # noqa: S101 - read_header only returns these two
|
|
70
|
+
return Fmi3ModelDescription(
|
|
71
|
+
header=header,
|
|
72
|
+
variables=cast("tuple[Fmi3Variable, ...]", variables),
|
|
73
|
+
model_structure=structure or Fmi3ModelStructure(),
|
|
74
|
+
annotations=(iterator.model_annotations or ()) if annotations else None,
|
|
75
|
+
)
|
mdxml/_reader.py
ADDED
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
"""Streaming XML reading on top of expat.
|
|
2
|
+
|
|
3
|
+
A reader drives a :class:`Handler` with start/end element events and stops
|
|
4
|
+
feeding expat as soon as the handler says it's done, so nothing after the
|
|
5
|
+
part a caller asked for is ever read (see ADR 0001).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from collections.abc import Generator
|
|
11
|
+
from typing import Protocol
|
|
12
|
+
from xml.parsers import expat
|
|
13
|
+
|
|
14
|
+
from mdxml._errors import ParseError, UnsafeInput
|
|
15
|
+
from mdxml._source import Readable
|
|
16
|
+
|
|
17
|
+
CHUNK_SIZE = 64 * 1024
|
|
18
|
+
DEFAULT_MAX_XML_BYTES = 256 * 1024 * 1024
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class Stop(Exception):
|
|
22
|
+
"""Raised by a handler to end parsing early. Never escapes this module."""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class Handler(Protocol):
|
|
26
|
+
def start(self, name: str, attrs: dict[str, str]) -> None: ...
|
|
27
|
+
def end(self, name: str) -> None: ...
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class Position(Protocol):
|
|
31
|
+
"""Where the parser currently is, for error messages and for keeping raw XML."""
|
|
32
|
+
|
|
33
|
+
@property
|
|
34
|
+
def line(self) -> int: ...
|
|
35
|
+
@property
|
|
36
|
+
def column(self) -> int: ...
|
|
37
|
+
@property
|
|
38
|
+
def byte_index(self) -> int:
|
|
39
|
+
"""Absolute byte offset of the current event in the document."""
|
|
40
|
+
...
|
|
41
|
+
|
|
42
|
+
def keep_from(self, index: int) -> None:
|
|
43
|
+
"""Start keeping the document's bytes from ``index`` on, so they can be sliced later."""
|
|
44
|
+
...
|
|
45
|
+
|
|
46
|
+
def kept(self, start: int, end: int) -> bytes:
|
|
47
|
+
"""Return the document's bytes ``[start, end)``, which must lie in what's being kept."""
|
|
48
|
+
...
|
|
49
|
+
|
|
50
|
+
def release(self) -> None:
|
|
51
|
+
"""Stop keeping bytes."""
|
|
52
|
+
...
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class _ExpatPosition:
|
|
56
|
+
"""The live parser position, plus the raw bytes still needed for :meth:`kept`.
|
|
57
|
+
|
|
58
|
+
Before each chunk is fed, the bytes after expat's last complete token are
|
|
59
|
+
carried over (expat reports a start tag only once it's whole, by which time
|
|
60
|
+
it may have begun in the previous chunk). While :meth:`keep_from` is active,
|
|
61
|
+
everything from that index on is kept as well.
|
|
62
|
+
"""
|
|
63
|
+
|
|
64
|
+
__slots__ = ("_buffer", "_buffer_start", "_keep_from", "_parser")
|
|
65
|
+
|
|
66
|
+
def __init__(self, parser: expat.XMLParserType) -> None:
|
|
67
|
+
self._parser = parser
|
|
68
|
+
self._buffer = bytearray()
|
|
69
|
+
self._buffer_start = 0
|
|
70
|
+
"""Absolute offset of ``_buffer[0]`` in the document."""
|
|
71
|
+
self._keep_from: int | None = None
|
|
72
|
+
|
|
73
|
+
@property
|
|
74
|
+
def line(self) -> int:
|
|
75
|
+
return self._parser.CurrentLineNumber
|
|
76
|
+
|
|
77
|
+
@property
|
|
78
|
+
def column(self) -> int:
|
|
79
|
+
return self._parser.CurrentColumnNumber + 1
|
|
80
|
+
|
|
81
|
+
@property
|
|
82
|
+
def byte_index(self) -> int:
|
|
83
|
+
return self._parser.CurrentByteIndex
|
|
84
|
+
|
|
85
|
+
def next_chunk(self, chunk: bytes) -> None:
|
|
86
|
+
"""Drop the bytes no longer needed, then append the chunk about to be fed."""
|
|
87
|
+
# Outside a callback, expat's byte index points just past its last complete token.
|
|
88
|
+
cut = max(self._parser.CurrentByteIndex, self._buffer_start)
|
|
89
|
+
if self._keep_from is not None:
|
|
90
|
+
cut = min(cut, self._keep_from)
|
|
91
|
+
if cut > self._buffer_start:
|
|
92
|
+
del self._buffer[: cut - self._buffer_start]
|
|
93
|
+
self._buffer_start = cut
|
|
94
|
+
self._buffer += chunk
|
|
95
|
+
|
|
96
|
+
def keep_from(self, index: int) -> None:
|
|
97
|
+
self._keep_from = index
|
|
98
|
+
|
|
99
|
+
def kept(self, start: int, end: int) -> bytes:
|
|
100
|
+
return bytes(self._buffer[start - self._buffer_start : end - self._buffer_start])
|
|
101
|
+
|
|
102
|
+
def release(self) -> None:
|
|
103
|
+
self._keep_from = None
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def check_max_xml_bytes(max_xml_bytes: object) -> int:
|
|
107
|
+
"""Validate a caller's ``max_xml_bytes``; it arrives from untyped callers too."""
|
|
108
|
+
if isinstance(max_xml_bytes, bool) or not isinstance(max_xml_bytes, int) or max_xml_bytes <= 0:
|
|
109
|
+
raise ValueError(f"max_xml_bytes must be a positive int, got {max_xml_bytes!r}")
|
|
110
|
+
return max_xml_bytes
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _refuse_doctype(doctype_name: str, *_: object) -> None:
|
|
114
|
+
raise UnsafeInput(f"refusing a DOCTYPE declaration (<!DOCTYPE {doctype_name}>); Model Descriptions never need one")
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _refuse_entity(entity_name: str, *_: object) -> None:
|
|
118
|
+
raise UnsafeInput(f"refusing entity declaration {entity_name!r}")
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _refuse_external_entity(*_: object) -> int:
|
|
122
|
+
raise UnsafeInput("refusing to resolve an external entity")
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _create_parser() -> expat.XMLParserType:
|
|
126
|
+
"""Create an expat parser that refuses DOCTYPEs and entities instead of processing them."""
|
|
127
|
+
parser = expat.ParserCreate()
|
|
128
|
+
# Entity declarations can only appear inside a DOCTYPE, so refusing the DOCTYPE at its first
|
|
129
|
+
# token blocks entity bombs and XXE before any declaration is read. The other handlers are
|
|
130
|
+
# a second line of defence.
|
|
131
|
+
parser.StartDoctypeDeclHandler = _refuse_doctype
|
|
132
|
+
parser.EntityDeclHandler = _refuse_entity
|
|
133
|
+
parser.ExternalEntityRefHandler = _refuse_external_entity
|
|
134
|
+
parser.SetParamEntityParsing(expat.XML_PARAM_ENTITY_PARSING_NEVER)
|
|
135
|
+
return parser
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def feed(
|
|
139
|
+
stream: Readable,
|
|
140
|
+
handler: Handler,
|
|
141
|
+
position_sink: list[Position],
|
|
142
|
+
*,
|
|
143
|
+
max_xml_bytes: int = DEFAULT_MAX_XML_BYTES,
|
|
144
|
+
) -> Generator[None, None, None]:
|
|
145
|
+
"""Parse ``stream`` with ``handler`` one chunk at a time, yielding after each chunk.
|
|
146
|
+
|
|
147
|
+
This lets a caller hand out what the handler built from each chunk before
|
|
148
|
+
reading more, so memory stays bounded by the chunk size. The generator
|
|
149
|
+
ends when the document ends or the handler raises :class:`Stop`.
|
|
150
|
+
|
|
151
|
+
``position_sink`` receives the live parser position before parsing starts,
|
|
152
|
+
so the handler can attach line and column to its own errors. At most
|
|
153
|
+
``max_xml_bytes`` bytes are read; reaching more raises :class:`UnsafeInput`.
|
|
154
|
+
"""
|
|
155
|
+
parser = _create_parser()
|
|
156
|
+
parser.StartElementHandler = handler.start
|
|
157
|
+
parser.EndElementHandler = handler.end
|
|
158
|
+
position = _ExpatPosition(parser)
|
|
159
|
+
position_sink.append(position)
|
|
160
|
+
total = 0
|
|
161
|
+
try:
|
|
162
|
+
# Ask for one byte more than the budget allows, so going over is detected
|
|
163
|
+
# without ever reading (or decompressing) much past the limit.
|
|
164
|
+
while chunk := stream.read(min(CHUNK_SIZE, max_xml_bytes - total + 1)):
|
|
165
|
+
total += len(chunk)
|
|
166
|
+
if total > max_xml_bytes:
|
|
167
|
+
raise UnsafeInput(f"Model Description is larger than max_xml_bytes={max_xml_bytes}")
|
|
168
|
+
position.next_chunk(chunk)
|
|
169
|
+
parser.Parse(chunk, False) # noqa: FBT003 - expat only accepts isfinal positionally
|
|
170
|
+
yield
|
|
171
|
+
parser.Parse(b"", True) # noqa: FBT003
|
|
172
|
+
except Stop:
|
|
173
|
+
return
|
|
174
|
+
except expat.ExpatError as exc:
|
|
175
|
+
raise ParseError(
|
|
176
|
+
expat.ErrorString(exc.code) if exc.code else str(exc),
|
|
177
|
+
line=exc.lineno,
|
|
178
|
+
column=exc.offset + 1,
|
|
179
|
+
) from None
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def run(
|
|
183
|
+
stream: Readable,
|
|
184
|
+
handler: Handler,
|
|
185
|
+
position_sink: list[Position],
|
|
186
|
+
*,
|
|
187
|
+
max_xml_bytes: int = DEFAULT_MAX_XML_BYTES,
|
|
188
|
+
) -> None:
|
|
189
|
+
"""Parse ``stream`` with ``handler`` until the document ends or the handler raises :class:`Stop`."""
|
|
190
|
+
for _ in feed(stream, handler, position_sink, max_xml_bytes=max_xml_bytes):
|
|
191
|
+
pass
|
mdxml/_schema.py
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"""Building parsed objects from elements, following the XSD.
|
|
2
|
+
|
|
3
|
+
Each element type has an attribute table (:class:`Attr` entries keyed by XML
|
|
4
|
+
name) that says which field each attribute fills, how to convert it and
|
|
5
|
+
whether it's required. :func:`attributes` applies a table to one element,
|
|
6
|
+
following ADR 0003:
|
|
7
|
+
|
|
8
|
+
- unknown attributes are ignored, or raise with ``strict=True``;
|
|
9
|
+
- a missing optional attribute leaves the field at its XSD default;
|
|
10
|
+
- a missing required attribute raises with ``strict=True``; in lenient mode it
|
|
11
|
+
takes the table's fallback if there is one (e.g. ``""`` for text), else raises;
|
|
12
|
+
- a value that can't be converted raises if the attribute is required or
|
|
13
|
+
``strict=True``; otherwise it's dropped and the field keeps its default.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
from collections.abc import Mapping
|
|
19
|
+
from dataclasses import dataclass
|
|
20
|
+
from types import MappingProxyType
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
from mdxml._convert import Converter, to_bool, to_double, to_text
|
|
24
|
+
from mdxml._errors import ParseError
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class _NoFallback:
|
|
28
|
+
__slots__ = ()
|
|
29
|
+
|
|
30
|
+
def __repr__(self) -> str:
|
|
31
|
+
return "NO_FALLBACK"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
NO_FALLBACK: Any = _NoFallback()
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass(frozen=True, slots=True)
|
|
38
|
+
class Attr:
|
|
39
|
+
"""How one XSD attribute maps onto a dataclass field."""
|
|
40
|
+
|
|
41
|
+
xml: str
|
|
42
|
+
field: str
|
|
43
|
+
convert: Converter[object]
|
|
44
|
+
required: bool = False
|
|
45
|
+
fallback: object = NO_FALLBACK
|
|
46
|
+
"""Value used in lenient mode when a required attribute is missing."""
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
AttrTable = Mapping[str, Attr]
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def table(*attrs: Attr) -> AttrTable:
|
|
53
|
+
"""Build a read-only attribute table keyed by XML name."""
|
|
54
|
+
return MappingProxyType({attr.xml: attr for attr in attrs})
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def required_text(xml: str, field: str) -> Attr:
|
|
58
|
+
"""Return a required text attribute; lenient mode uses ``""`` when it's missing."""
|
|
59
|
+
return Attr(xml, field, to_text, required=True, fallback="")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def text(xml: str, field: str) -> Attr:
|
|
63
|
+
"""Return an optional text attribute."""
|
|
64
|
+
return Attr(xml, field, to_text)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def flag(xml: str, field: str) -> Attr:
|
|
68
|
+
"""Return an optional ``xs:boolean`` attribute."""
|
|
69
|
+
return Attr(xml, field, to_bool)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def double(xml: str, field: str) -> Attr:
|
|
73
|
+
"""Return an optional ``xs:double`` (or ``xs:float``) attribute."""
|
|
74
|
+
return Attr(xml, field, to_double)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
EMPTY = table()
|
|
78
|
+
"""The attribute table of an element without attributes."""
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class Frame:
|
|
82
|
+
"""One open element while its subtree is being read."""
|
|
83
|
+
|
|
84
|
+
__slots__ = ("attrs", "children", "column", "line", "tag")
|
|
85
|
+
|
|
86
|
+
def __init__(self, tag: str, attrs: dict[str, str], line: int, column: int) -> None:
|
|
87
|
+
self.tag = tag
|
|
88
|
+
self.attrs = attrs
|
|
89
|
+
self.line = line
|
|
90
|
+
self.column = column
|
|
91
|
+
self.children: list[tuple[str, object]] = []
|
|
92
|
+
|
|
93
|
+
def error(self, message: str) -> ParseError:
|
|
94
|
+
"""Build a :class:`ParseError` pointing at this element."""
|
|
95
|
+
return ParseError(message, line=self.line, column=self.column)
|
|
96
|
+
|
|
97
|
+
def all(self, tag: str) -> list[Any]:
|
|
98
|
+
"""Return the built children with this tag, in document order."""
|
|
99
|
+
return [obj for child_tag, obj in self.children if child_tag == tag]
|
|
100
|
+
|
|
101
|
+
def one(self, tag: str, *, strict: bool) -> Any:
|
|
102
|
+
"""Return the single built child with this tag, or ``None``.
|
|
103
|
+
|
|
104
|
+
More than one raises with ``strict=True``; lenient mode keeps the first.
|
|
105
|
+
"""
|
|
106
|
+
found = self.all(tag)
|
|
107
|
+
if len(found) > 1 and strict:
|
|
108
|
+
raise self.error(f"<{self.tag}> has {len(found)} <{tag}> elements, expected at most one")
|
|
109
|
+
return found[0] if found else None
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def attributes(frame: Frame, spec: AttrTable, *, strict: bool) -> dict[str, Any]:
|
|
113
|
+
"""Convert ``frame``'s attributes into dataclass keyword arguments using ``spec``."""
|
|
114
|
+
fields: dict[str, Any] = {}
|
|
115
|
+
for xml_name, raw in frame.attrs.items():
|
|
116
|
+
attr = spec.get(xml_name)
|
|
117
|
+
if attr is None:
|
|
118
|
+
if strict:
|
|
119
|
+
raise frame.error(f"unknown attribute {xml_name!r} on <{frame.tag}>")
|
|
120
|
+
continue
|
|
121
|
+
try:
|
|
122
|
+
fields[attr.field] = attr.convert(raw, strict=strict)
|
|
123
|
+
except ValueError as exc:
|
|
124
|
+
if strict or attr.required:
|
|
125
|
+
raise frame.error(f"invalid {xml_name}={raw!r} on <{frame.tag}>: {exc}") from None
|
|
126
|
+
for attr in spec.values():
|
|
127
|
+
if attr.required and attr.field not in fields:
|
|
128
|
+
if strict or attr.fallback is NO_FALLBACK:
|
|
129
|
+
raise frame.error(f"<{frame.tag}> is missing required attribute {attr.xml!r}")
|
|
130
|
+
fields[attr.field] = attr.fallback
|
|
131
|
+
return fields
|