wodson 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wodson/__init__.py +3 -0
- wodson/_cli.py +36 -0
- wodson/atfx/__init__.py +7 -0
- wodson/atfx/_atfx_store.py +142 -0
- wodson/atfx/_base_model.py +155 -0
- wodson/atfx/_binary_reader.py +311 -0
- wodson/atfx/_cli.py +121 -0
- wodson/atfx/_data_read.py +613 -0
- wodson/atfx/_db.py +221 -0
- wodson/atfx/_instance_parser.py +527 -0
- wodson/atfx/_model_builder.py +212 -0
- wodson/atfx/_naming.py +13 -0
- wodson/atfx/_server.py +308 -0
- wodson/atfx/_session.py +293 -0
- wodson/atfx/_xml_utils.py +40 -0
- wodson/atfx/base_model/ODSBaseModel_asam37.protobuf.json +5422 -0
- wodson/py.typed +0 -0
- wodson-1.0.0.dist-info/METADATA +108 -0
- wodson-1.0.0.dist-info/RECORD +22 -0
- wodson-1.0.0.dist-info/WHEEL +4 -0
- wodson-1.0.0.dist-info/entry_points.txt +3 -0
- wodson-1.0.0.dist-info/licenses/LICENSE +201 -0
wodson/__init__.py
ADDED
wodson/_cli.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""Top-level command-line interface for wodson.
|
|
2
|
+
|
|
3
|
+
Usage::
|
|
4
|
+
|
|
5
|
+
uv run wodson atfx serve
|
|
6
|
+
uv run wodson atfx serve --file path/to/file.atfx
|
|
7
|
+
uv run wodson atfx serve --file path/to/file.atfx --host 0.0.0.0 --port 8080
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import argparse
|
|
13
|
+
|
|
14
|
+
import wodson.atfx._cli as _atfx_cli
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _build_parser() -> argparse.ArgumentParser:
|
|
18
|
+
parser = argparse.ArgumentParser(
|
|
19
|
+
prog="wodson",
|
|
20
|
+
description="wodson ASAM ODS command-line tools.",
|
|
21
|
+
)
|
|
22
|
+
sub = parser.add_subparsers(dest="module", required=True)
|
|
23
|
+
_atfx_cli.register_atfx_subparser(sub)
|
|
24
|
+
return parser
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def main() -> None:
|
|
28
|
+
"""Entry point for the ``wodson`` CLI command."""
|
|
29
|
+
parser = _build_parser()
|
|
30
|
+
args = parser.parse_args()
|
|
31
|
+
if args.module == "atfx":
|
|
32
|
+
_atfx_cli.dispatch(args)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
if __name__ == "__main__":
|
|
36
|
+
main()
|
wodson/atfx/__init__.py
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""wodson.atfx — ATFX reader with in-memory SQLite backend and ODS data-read API."""
|
|
2
|
+
|
|
3
|
+
from ._atfx_store import AtfxStore
|
|
4
|
+
from ._server import CONTEXT_VAR_ATFX_FILE, AtfxServer
|
|
5
|
+
from ._session import AtfxSession
|
|
6
|
+
|
|
7
|
+
__all__ = ["AtfxServer", "AtfxSession", "AtfxStore", "CONTEXT_VAR_ATFX_FILE"]
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
"""AtfxStore: main public class for loading and querying ATFX files."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
import sqlite3
|
|
7
|
+
import xml.etree.ElementTree as ET
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
import odsbox.proto.ods_pb2 as ods
|
|
11
|
+
|
|
12
|
+
from ._base_model import load_base_model
|
|
13
|
+
from ._data_read import data_read
|
|
14
|
+
from ._db import create_schema, load_instances
|
|
15
|
+
from ._instance_parser import parse_instances, resolve_external_component_refs
|
|
16
|
+
from ._model_builder import build_model, detect_ods_version
|
|
17
|
+
from ._xml_utils import _find, _findall, _text
|
|
18
|
+
|
|
19
|
+
_log = logging.getLogger(__name__)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class AtfxStore:
|
|
23
|
+
"""Load an ATFX file into an in-memory SQLite database and provide ODS API access.
|
|
24
|
+
|
|
25
|
+
Usage::
|
|
26
|
+
|
|
27
|
+
store = AtfxStore("path/to/file.atfx")
|
|
28
|
+
model = store.model()
|
|
29
|
+
result = store.data_read(select_statement)
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
def __init__(self, file_path: str | Path, base_model_path: Path | None = None) -> None:
|
|
33
|
+
"""Initialize the store by parsing and loading the ATFX file.
|
|
34
|
+
|
|
35
|
+
Args:
|
|
36
|
+
file_path: Path to the .atfx file.
|
|
37
|
+
base_model_path: Optional path to the base model JSON. Defaults to shipped version.
|
|
38
|
+
"""
|
|
39
|
+
self._file_path = Path(file_path).resolve()
|
|
40
|
+
self._atfx_dir = self._file_path.parent
|
|
41
|
+
_log.info("Loading ATFX file: %s", self._file_path)
|
|
42
|
+
|
|
43
|
+
# Parse XML
|
|
44
|
+
tree = ET.parse(self._file_path) # noqa: S314
|
|
45
|
+
root = tree.getroot()
|
|
46
|
+
_log.debug("XML parsed successfully")
|
|
47
|
+
|
|
48
|
+
# Extract version metadata from the XML
|
|
49
|
+
self._ods_version: str = detect_ods_version(root)
|
|
50
|
+
self._base_model_version: str = _text(root, "base_model_version")
|
|
51
|
+
_log.debug(
|
|
52
|
+
"ODS version: %s, base model version: %s",
|
|
53
|
+
self._ods_version or "(not detected)",
|
|
54
|
+
self._base_model_version or "(not set)",
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
# Load base model
|
|
58
|
+
self._base_model = load_base_model(base_model_path)
|
|
59
|
+
|
|
60
|
+
# Build application model
|
|
61
|
+
self._model = build_model(root, self._base_model)
|
|
62
|
+
|
|
63
|
+
# Parse file map from <files> section
|
|
64
|
+
self._file_map = self._parse_file_map(root)
|
|
65
|
+
if self._file_map:
|
|
66
|
+
_log.debug("%d external file(s) mapped", len(self._file_map))
|
|
67
|
+
|
|
68
|
+
# Parse instances
|
|
69
|
+
instances = parse_instances(root, self._model)
|
|
70
|
+
|
|
71
|
+
# Resolve AoExternalComponent references (third value-reference pattern)
|
|
72
|
+
resolve_external_component_refs(self._model, instances, self._file_map, self._atfx_dir)
|
|
73
|
+
|
|
74
|
+
# Create in-memory SQLite DB
|
|
75
|
+
self._conn = sqlite3.connect(":memory:", check_same_thread=False)
|
|
76
|
+
create_schema(self._conn, self._model)
|
|
77
|
+
load_instances(self._conn, self._model, instances, self._file_map)
|
|
78
|
+
_log.info(
|
|
79
|
+
"AtfxStore ready: %d entities, %d instance groups",
|
|
80
|
+
len(self._model.entities),
|
|
81
|
+
len(instances),
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
def model(self) -> ods.Model:
|
|
85
|
+
"""Return the application model as an ods.Model protobuf."""
|
|
86
|
+
return self._model
|
|
87
|
+
|
|
88
|
+
def data_read(self, select_statement: ods.SelectStatement) -> ods.DataMatrices:
|
|
89
|
+
"""Execute a SelectStatement and return DataMatrices.
|
|
90
|
+
|
|
91
|
+
This matches the ASAM ODS HTTP API data-read method semantics.
|
|
92
|
+
|
|
93
|
+
Args:
|
|
94
|
+
select_statement: The ODS SelectStatement protobuf.
|
|
95
|
+
|
|
96
|
+
Returns:
|
|
97
|
+
ods.DataMatrices containing the query results.
|
|
98
|
+
"""
|
|
99
|
+
return data_read(self._conn, self._model, select_statement)
|
|
100
|
+
|
|
101
|
+
def context_read(self) -> ods.ContextVariables:
|
|
102
|
+
"""Return context variables for this ATFX session.
|
|
103
|
+
|
|
104
|
+
Populated variables:
|
|
105
|
+
|
|
106
|
+
* ``ASAM-ODS-VERSION`` -- ODS schema version extracted from the XML
|
|
107
|
+
namespace, e.g. ``"6.1.0"``.
|
|
108
|
+
* ``BASE-MODEL-VERSION`` -- value of ``<base_model_version>`` in the
|
|
109
|
+
ATFX file, e.g. ``"asam35"``.
|
|
110
|
+
"""
|
|
111
|
+
ctx = ods.ContextVariables()
|
|
112
|
+
if self._ods_version:
|
|
113
|
+
ctx.variables["ASAM-ODS-VERSION"].string_array.values.append(self._ods_version)
|
|
114
|
+
if self._base_model_version:
|
|
115
|
+
ctx.variables["BASE-MODEL-VERSION"].string_array.values.append(self._base_model_version)
|
|
116
|
+
return ctx
|
|
117
|
+
|
|
118
|
+
def close(self) -> None:
|
|
119
|
+
"""Close the SQLite connection."""
|
|
120
|
+
_log.debug("Closing AtfxStore for %s", self._file_path)
|
|
121
|
+
self._conn.close()
|
|
122
|
+
|
|
123
|
+
def __enter__(self) -> AtfxStore:
|
|
124
|
+
return self
|
|
125
|
+
|
|
126
|
+
def __exit__(self, *_args: object) -> None:
|
|
127
|
+
self.close()
|
|
128
|
+
|
|
129
|
+
def _parse_file_map(self, root: ET.Element) -> dict[str, Path]:
|
|
130
|
+
"""Parse the <files> section to build identifier -> file path mapping."""
|
|
131
|
+
file_map: dict[str, Path] = {}
|
|
132
|
+
files_el = _find(root, "files")
|
|
133
|
+
if files_el is None:
|
|
134
|
+
return file_map
|
|
135
|
+
|
|
136
|
+
for comp_el in _findall(files_el, "component"):
|
|
137
|
+
identifier = _text(comp_el, "identifier")
|
|
138
|
+
filename = _text(comp_el, "filename")
|
|
139
|
+
if identifier and filename:
|
|
140
|
+
file_map[identifier] = self._atfx_dir / filename
|
|
141
|
+
|
|
142
|
+
return file_map
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
"""Load the ASAM ODS base model from the shipped JSON file."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import logging
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
import odsbox.proto.ods_pb2 as ods
|
|
10
|
+
|
|
11
|
+
_log = logging.getLogger(__name__)
|
|
12
|
+
|
|
13
|
+
_BASE_MODEL_PATH = Path(__file__).resolve().parent / "base_model" / "ODSBaseModel_asam37.protobuf.json"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def load_base_model(path: Path | None = None) -> ods.BaseModel:
|
|
17
|
+
"""Load the ODS base model from a JSON file.
|
|
18
|
+
|
|
19
|
+
The JSON structure mirrors the protobuf BaseModel message layout
|
|
20
|
+
but uses a slightly different naming convention, so we manually
|
|
21
|
+
parse rather than using ParseDict.
|
|
22
|
+
"""
|
|
23
|
+
json_path = path if path is not None else _BASE_MODEL_PATH
|
|
24
|
+
_log.debug("Loading base model from %s", json_path)
|
|
25
|
+
with open(json_path, encoding="utf-8") as f:
|
|
26
|
+
data: dict[str, object] = json.load(f)
|
|
27
|
+
|
|
28
|
+
base_model = ods.BaseModel()
|
|
29
|
+
base_model.version = str(data.get("version", ""))
|
|
30
|
+
|
|
31
|
+
# Enumerations
|
|
32
|
+
enums_data = data.get("enumerations")
|
|
33
|
+
if isinstance(enums_data, dict):
|
|
34
|
+
for enum_name, enum_obj in enums_data.items():
|
|
35
|
+
if not isinstance(enum_obj, dict):
|
|
36
|
+
continue
|
|
37
|
+
enum_entry = base_model.enumerations[enum_name]
|
|
38
|
+
enum_entry.name = str(enum_obj.get("name", enum_name))
|
|
39
|
+
items = enum_obj.get("items")
|
|
40
|
+
if isinstance(items, dict):
|
|
41
|
+
for item_name, item_value in items.items():
|
|
42
|
+
enum_entry.items[item_name] = int(item_value)
|
|
43
|
+
|
|
44
|
+
# Entities
|
|
45
|
+
entities_data = data.get("entities")
|
|
46
|
+
if isinstance(entities_data, dict):
|
|
47
|
+
for entity_name, entity_obj in entities_data.items():
|
|
48
|
+
if not isinstance(entity_obj, dict):
|
|
49
|
+
continue
|
|
50
|
+
entity = base_model.entities[entity_name]
|
|
51
|
+
entity.name = str(entity_obj.get("name", entity_name))
|
|
52
|
+
entity.bid = int(entity_obj.get("bid", 0))
|
|
53
|
+
|
|
54
|
+
derivation_str = str(entity_obj.get("derivation", "DV_UNRESTRICTED"))
|
|
55
|
+
derivation_map: dict[str, int] = {
|
|
56
|
+
"DV_UNRESTRICTED": 0,
|
|
57
|
+
"DV_SINGLE": 1,
|
|
58
|
+
"DV_TREE_AT_INSTANCE": 2,
|
|
59
|
+
"DV_TREE_AT_MODEL": 3,
|
|
60
|
+
}
|
|
61
|
+
entity.derivation = derivation_map.get(derivation_str, 0) # type: ignore[assignment]
|
|
62
|
+
|
|
63
|
+
# Attributes
|
|
64
|
+
attrs_data = entity_obj.get("attributes")
|
|
65
|
+
if isinstance(attrs_data, dict):
|
|
66
|
+
for attr_name, attr_obj in attrs_data.items():
|
|
67
|
+
if not isinstance(attr_obj, dict):
|
|
68
|
+
continue
|
|
69
|
+
attr = entity.attributes[attr_name]
|
|
70
|
+
attr.name = str(attr_obj.get("name", attr_name))
|
|
71
|
+
attr.data_type = _parse_datatype( # type: ignore[assignment]
|
|
72
|
+
str(attr_obj.get("dataType", "DT_UNKNOWN"))
|
|
73
|
+
)
|
|
74
|
+
attr.mandatory = bool(attr_obj.get("mandatory", False))
|
|
75
|
+
attr.obligatory = bool(attr_obj.get("obligatory", False))
|
|
76
|
+
attr.autogenerated = bool(attr_obj.get("autogenerated", False))
|
|
77
|
+
attr.enumeration = str(attr_obj.get("enumeration", ""))
|
|
78
|
+
|
|
79
|
+
# Relations
|
|
80
|
+
rels_data = entity_obj.get("relations")
|
|
81
|
+
if isinstance(rels_data, dict):
|
|
82
|
+
for rel_name, rel_obj in rels_data.items():
|
|
83
|
+
if not isinstance(rel_obj, dict):
|
|
84
|
+
continue
|
|
85
|
+
rel = entity.relations[rel_name]
|
|
86
|
+
rel.name = str(rel_obj.get("name", rel_name))
|
|
87
|
+
rel.inverse_name = str(rel_obj.get("inverseName", ""))
|
|
88
|
+
entity_names = rel_obj.get("entityNames")
|
|
89
|
+
if isinstance(entity_names, list) and len(entity_names) > 0:
|
|
90
|
+
rel.entity_names.extend([str(n) for n in entity_names])
|
|
91
|
+
rel.range_min = int(rel_obj.get("rangeMin", 0))
|
|
92
|
+
rel.range_max = int(rel_obj.get("rangeMax", -1))
|
|
93
|
+
rel.mandatory = bool(rel_obj.get("mandatory", False))
|
|
94
|
+
rel.inverse_range_min = int(rel_obj.get("inverseRangeMin", 0))
|
|
95
|
+
rel.inverse_range_max = int(rel_obj.get("inverseRangeMax", -1))
|
|
96
|
+
|
|
97
|
+
relationship_str = str(rel_obj.get("relationship", "RS_INFO_TO"))
|
|
98
|
+
relationship_map: dict[str, int] = {
|
|
99
|
+
"RS_FATHER": 0,
|
|
100
|
+
"RS_CHILD": 1,
|
|
101
|
+
"RS_INFO_TO": 2,
|
|
102
|
+
"RS_INFO_FROM": 3,
|
|
103
|
+
"RS_INFO_REL": 4,
|
|
104
|
+
"RS_SUPERTYPE": 5,
|
|
105
|
+
"RS_SUBTYPE": 6,
|
|
106
|
+
"RS_ALL_REL": 7,
|
|
107
|
+
}
|
|
108
|
+
rel.relationship = relationship_map.get(relationship_str, 2) # type: ignore[assignment]
|
|
109
|
+
|
|
110
|
+
relation_type_str = str(rel_obj.get("relationType", "RT_INFO"))
|
|
111
|
+
relation_type_map: dict[str, int] = {
|
|
112
|
+
"RT_FATHER_CHILD": 0,
|
|
113
|
+
"RT_INFO": 1,
|
|
114
|
+
"RT_INHERITANCE": 2,
|
|
115
|
+
}
|
|
116
|
+
rel.relation_type = relation_type_map.get(relation_type_str, 1) # type: ignore[assignment]
|
|
117
|
+
|
|
118
|
+
return base_model
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _parse_datatype(dt_str: str) -> int:
|
|
122
|
+
"""Map datatype string to ods.DataTypeEnum int value."""
|
|
123
|
+
mapping: dict[str, int] = {
|
|
124
|
+
"DT_UNKNOWN": 0,
|
|
125
|
+
"DT_STRING": 1,
|
|
126
|
+
"DT_SHORT": 2,
|
|
127
|
+
"DT_FLOAT": 3,
|
|
128
|
+
"DT_BOOLEAN": 4,
|
|
129
|
+
"DT_BYTE": 5,
|
|
130
|
+
"DT_LONG": 6,
|
|
131
|
+
"DT_DOUBLE": 7,
|
|
132
|
+
"DT_LONGLONG": 8,
|
|
133
|
+
"DT_DATE": 10,
|
|
134
|
+
"DT_BYTESTR": 11,
|
|
135
|
+
"DT_BLOB": 12,
|
|
136
|
+
"DT_COMPLEX": 13,
|
|
137
|
+
"DT_DCOMPLEX": 14,
|
|
138
|
+
"DS_STRING": 15,
|
|
139
|
+
"DS_SHORT": 16,
|
|
140
|
+
"DS_FLOAT": 17,
|
|
141
|
+
"DS_BOOLEAN": 18,
|
|
142
|
+
"DS_BYTE": 19,
|
|
143
|
+
"DS_LONG": 20,
|
|
144
|
+
"DS_DOUBLE": 21,
|
|
145
|
+
"DS_LONGLONG": 22,
|
|
146
|
+
"DS_COMPLEX": 23,
|
|
147
|
+
"DS_DCOMPLEX": 24,
|
|
148
|
+
"DS_DATE": 26,
|
|
149
|
+
"DS_BYTESTR": 27,
|
|
150
|
+
"DT_EXTERNALREFERENCE": 28,
|
|
151
|
+
"DS_EXTERNALREFERENCE": 29,
|
|
152
|
+
"DT_ENUM": 30,
|
|
153
|
+
"DS_ENUM": 31,
|
|
154
|
+
}
|
|
155
|
+
return mapping.get(dt_str, 0)
|
|
@@ -0,0 +1,311 @@
|
|
|
1
|
+
"""Read external binary .dat files referenced by ATFX local columns."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
import struct
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
import numpy as np
|
|
11
|
+
import odsbox.proto.ods_pb2 as ods
|
|
12
|
+
from numpy.typing import NDArray
|
|
13
|
+
|
|
14
|
+
from ._instance_parser import ExternalComponentRef, TypedValues
|
|
15
|
+
|
|
16
|
+
_log = logging.getLogger(__name__)
|
|
17
|
+
|
|
18
|
+
# Mapping: typespec_enum string -> (numpy dtype, element_size_bytes)
|
|
19
|
+
_TYPESPEC_MAP: dict[str, tuple[str, int]] = {
|
|
20
|
+
"dt_boolean": ("|b1", 1),
|
|
21
|
+
"dt_byte": ("|u1", 1),
|
|
22
|
+
"dt_sbyte": ("|i1", 1),
|
|
23
|
+
"dt_short": ("<i2", 2),
|
|
24
|
+
"dt_long": ("<i4", 4),
|
|
25
|
+
"dt_longlong": ("<i8", 8),
|
|
26
|
+
"ieeefloat4": ("<f4", 4),
|
|
27
|
+
"ieeefloat8": ("<f8", 8),
|
|
28
|
+
"dt_short_beo": (">i2", 2),
|
|
29
|
+
"dt_long_beo": (">i4", 4),
|
|
30
|
+
"dt_longlong_beo": (">i8", 8),
|
|
31
|
+
"ieeefloat4_beo": (">f4", 4),
|
|
32
|
+
"ieeefloat8_beo": (">f8", 8),
|
|
33
|
+
"dt_ushort": ("<u2", 2),
|
|
34
|
+
"dt_ushort_beo": (">u2", 2),
|
|
35
|
+
"dt_ulong": ("<u4", 4),
|
|
36
|
+
"dt_ulong_beo": (">u4", 4),
|
|
37
|
+
"dt_string": ("|S1", 1),
|
|
38
|
+
"dt_string_utf8": ("|S1", 1),
|
|
39
|
+
"dt_bytestr_leo": ("|u1", 1),
|
|
40
|
+
"dt_bytestr_beo": ("|u1", 1),
|
|
41
|
+
# Bit types — raw-byte fallback; use _read_bit_values for actual decoding
|
|
42
|
+
"dt_bit_int": ("|u1", 1),
|
|
43
|
+
"dt_bit_int_beo": ("|u1", 1),
|
|
44
|
+
"dt_bit_uint": ("|u1", 1),
|
|
45
|
+
"dt_bit_uint_beo": ("|u1", 1),
|
|
46
|
+
"dt_bit_ieeefloat": ("|u1", 1),
|
|
47
|
+
"dt_bit_ieeefloat_beo": ("|u1", 1),
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
# Mapping: typespec_enum string -> ODS DataTypeEnum (for DT_UNKNOWN columns)
|
|
51
|
+
_TYPESPEC_TO_ODS_DT: dict[str, int] = {
|
|
52
|
+
"dt_boolean": ods.DataTypeEnum.DT_BOOLEAN,
|
|
53
|
+
"dt_byte": ods.DataTypeEnum.DT_BYTE,
|
|
54
|
+
# dt_sbyte: ASAM ODS has no signed-byte type; promote to DT_SHORT
|
|
55
|
+
"dt_sbyte": ods.DataTypeEnum.DT_SHORT,
|
|
56
|
+
"dt_short": ods.DataTypeEnum.DT_SHORT,
|
|
57
|
+
"dt_long": ods.DataTypeEnum.DT_LONG,
|
|
58
|
+
"dt_longlong": ods.DataTypeEnum.DT_LONGLONG,
|
|
59
|
+
"ieeefloat4": ods.DataTypeEnum.DT_FLOAT,
|
|
60
|
+
"ieeefloat8": ods.DataTypeEnum.DT_DOUBLE,
|
|
61
|
+
"dt_short_beo": ods.DataTypeEnum.DT_SHORT,
|
|
62
|
+
"dt_long_beo": ods.DataTypeEnum.DT_LONG,
|
|
63
|
+
"dt_longlong_beo": ods.DataTypeEnum.DT_LONGLONG,
|
|
64
|
+
"ieeefloat4_beo": ods.DataTypeEnum.DT_FLOAT,
|
|
65
|
+
"ieeefloat8_beo": ods.DataTypeEnum.DT_DOUBLE,
|
|
66
|
+
# Unsigned types: promote to signed type that can hold the full value range
|
|
67
|
+
"dt_ushort": ods.DataTypeEnum.DT_LONG,
|
|
68
|
+
"dt_ushort_beo": ods.DataTypeEnum.DT_LONG,
|
|
69
|
+
"dt_ulong": ods.DataTypeEnum.DT_LONGLONG,
|
|
70
|
+
"dt_ulong_beo": ods.DataTypeEnum.DT_LONGLONG,
|
|
71
|
+
"dt_string": ods.DataTypeEnum.DT_STRING,
|
|
72
|
+
"dt_string_utf8": ods.DataTypeEnum.DT_STRING,
|
|
73
|
+
"dt_bytestr_leo": ods.DataTypeEnum.DT_BYTESTR,
|
|
74
|
+
"dt_bytestr_beo": ods.DataTypeEnum.DT_BYTESTR,
|
|
75
|
+
# Bit types are resolved dynamically from bitcount — DT_UNKNOWN is the fallback
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
# Typespecs that require bit-level extraction
|
|
79
|
+
_BIT_TYPESPECS: frozenset[str] = frozenset(
|
|
80
|
+
{
|
|
81
|
+
"dt_bit_int",
|
|
82
|
+
"dt_bit_int_beo",
|
|
83
|
+
"dt_bit_uint",
|
|
84
|
+
"dt_bit_uint_beo",
|
|
85
|
+
"dt_bit_ieeefloat",
|
|
86
|
+
"dt_bit_ieeefloat_beo",
|
|
87
|
+
}
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
# Typespecs that require specialized string/bytestr parsing
|
|
91
|
+
_STRING_TYPESPECS: frozenset[str] = frozenset({"dt_string", "dt_string_utf8"})
|
|
92
|
+
_BYTESTR_TYPESPECS: frozenset[str] = frozenset({"dt_bytestr_leo", "dt_bytestr_beo"})
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _bit_ods_dt(typespec: str, bitcount: int) -> int:
|
|
96
|
+
"""Return the ODS DataTypeEnum for a bit-extraction typespec given its bitcount."""
|
|
97
|
+
if "ieeefloat" in typespec:
|
|
98
|
+
return ods.DataTypeEnum.DT_FLOAT if bitcount <= 32 else ods.DataTypeEnum.DT_DOUBLE
|
|
99
|
+
elif "uint" in typespec:
|
|
100
|
+
if bitcount <= 8:
|
|
101
|
+
return ods.DataTypeEnum.DT_BYTE
|
|
102
|
+
elif bitcount <= 16:
|
|
103
|
+
return ods.DataTypeEnum.DT_SHORT
|
|
104
|
+
elif bitcount <= 32:
|
|
105
|
+
return ods.DataTypeEnum.DT_LONG
|
|
106
|
+
else:
|
|
107
|
+
return ods.DataTypeEnum.DT_LONGLONG
|
|
108
|
+
else: # signed int
|
|
109
|
+
if bitcount <= 16:
|
|
110
|
+
return ods.DataTypeEnum.DT_SHORT
|
|
111
|
+
elif bitcount <= 32:
|
|
112
|
+
return ods.DataTypeEnum.DT_LONG
|
|
113
|
+
else:
|
|
114
|
+
return ods.DataTypeEnum.DT_LONGLONG
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _read_bit_values(
|
|
118
|
+
data: bytes,
|
|
119
|
+
ref: ExternalComponentRef,
|
|
120
|
+
signed: bool,
|
|
121
|
+
is_float: bool,
|
|
122
|
+
big_endian: bool,
|
|
123
|
+
) -> list[Any]:
|
|
124
|
+
"""Extract bit-field values from a binary buffer.
|
|
125
|
+
|
|
126
|
+
Supports LE/BE, signed/unsigned integers, and IEEE floats of arbitrary
|
|
127
|
+
bit-width packed at a given bit offset within multi-byte blocks.
|
|
128
|
+
"""
|
|
129
|
+
count = ref.length
|
|
130
|
+
bitcount = ref.bitcount
|
|
131
|
+
bitoffset = ref.bitoffset
|
|
132
|
+
blocksize = ref.blocksize if ref.blocksize > 0 else (bitoffset + bitcount + 7) // 8
|
|
133
|
+
valperblock = ref.valperblock if ref.valperblock > 0 else 1
|
|
134
|
+
inioffset = ref.inioffset
|
|
135
|
+
byte_offset_in_block = ref.valoffsets[0] if ref.valoffsets else 0
|
|
136
|
+
mask = (1 << bitcount) - 1
|
|
137
|
+
|
|
138
|
+
results: list[Any] = []
|
|
139
|
+
remaining = count
|
|
140
|
+
block_num = 0
|
|
141
|
+
|
|
142
|
+
while remaining > 0:
|
|
143
|
+
n = min(valperblock, remaining)
|
|
144
|
+
block_start = inioffset + block_num * blocksize
|
|
145
|
+
block = data[block_start : block_start + blocksize]
|
|
146
|
+
|
|
147
|
+
for j in range(n):
|
|
148
|
+
# Absolute bit address within block
|
|
149
|
+
bit_addr = byte_offset_in_block * 8 + bitoffset + j * bitcount
|
|
150
|
+
byte_addr = bit_addr // 8
|
|
151
|
+
local_bit = bit_addr % 8
|
|
152
|
+
word_bytes_needed = (local_bit + bitcount + 7) // 8
|
|
153
|
+
word = block[byte_addr : byte_addr + word_bytes_needed]
|
|
154
|
+
|
|
155
|
+
if big_endian:
|
|
156
|
+
# BEO stores bytes in big-endian order; reverse to LE then
|
|
157
|
+
# apply the same LSB-first bit extraction as the LE case.
|
|
158
|
+
word = word[::-1]
|
|
159
|
+
word_int = int.from_bytes(word, "little")
|
|
160
|
+
value_bits = (word_int >> local_bit) & mask
|
|
161
|
+
|
|
162
|
+
if is_float:
|
|
163
|
+
if bitcount == 32:
|
|
164
|
+
(value,) = struct.unpack("f", struct.pack("I", value_bits))
|
|
165
|
+
elif bitcount == 64:
|
|
166
|
+
(value,) = struct.unpack("d", struct.pack("Q", value_bits))
|
|
167
|
+
else:
|
|
168
|
+
value = float(value_bits)
|
|
169
|
+
elif signed:
|
|
170
|
+
value = value_bits - (1 << bitcount) if value_bits & (1 << (bitcount - 1)) else value_bits
|
|
171
|
+
else:
|
|
172
|
+
value = value_bits
|
|
173
|
+
|
|
174
|
+
results.append(value)
|
|
175
|
+
|
|
176
|
+
remaining -= n
|
|
177
|
+
block_num += 1
|
|
178
|
+
|
|
179
|
+
return results
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _read_null_terminated_strings(data: bytes, offset: int, total_bytes: int) -> list[str]:
|
|
183
|
+
"""Parse null-terminated strings packed sequentially in a byte buffer."""
|
|
184
|
+
segment = data[offset : offset + total_bytes]
|
|
185
|
+
results: list[str] = []
|
|
186
|
+
start = 0
|
|
187
|
+
while start < len(segment):
|
|
188
|
+
end = segment.find(b"\x00", start)
|
|
189
|
+
if end == -1:
|
|
190
|
+
s = segment[start:].decode("utf-8", errors="replace").rstrip("\x00")
|
|
191
|
+
if s:
|
|
192
|
+
results.append(s)
|
|
193
|
+
break
|
|
194
|
+
results.append(segment[start:end].decode("utf-8", errors="replace"))
|
|
195
|
+
start = end + 1
|
|
196
|
+
return results
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _read_bytestr_sequence(data: bytes, offset: int, total_bytes: int, big_endian: bool) -> list[bytes]:
|
|
200
|
+
"""Parse length-prefixed (4-byte) byte strings packed in a byte buffer."""
|
|
201
|
+
fmt = ">I" if big_endian else "<I"
|
|
202
|
+
segment = data[offset : offset + total_bytes]
|
|
203
|
+
results: list[bytes] = []
|
|
204
|
+
pos = 0
|
|
205
|
+
while pos + 4 <= len(segment):
|
|
206
|
+
(length,) = struct.unpack_from(fmt, segment, pos)
|
|
207
|
+
pos += 4
|
|
208
|
+
results.append(segment[pos : pos + length])
|
|
209
|
+
pos += length
|
|
210
|
+
return results
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def read_external_component_typed(ref: ExternalComponentRef, file_map: dict[str, Path]) -> TypedValues:
|
|
214
|
+
"""Read binary values and return them with the corresponding ODS DataTypeEnum.
|
|
215
|
+
|
|
216
|
+
Handles string and bytestr types that require specialized parsing,
|
|
217
|
+
in addition to all numeric/boolean types.
|
|
218
|
+
"""
|
|
219
|
+
typespec = ref.datatype.lower()
|
|
220
|
+
ods_dt = _TYPESPEC_TO_ODS_DT.get(typespec, ods.DataTypeEnum.DT_UNKNOWN)
|
|
221
|
+
|
|
222
|
+
file_path = file_map.get(ref.identifier)
|
|
223
|
+
if file_path is None:
|
|
224
|
+
msg = f"External component identifier '{ref.identifier}' not found in file map"
|
|
225
|
+
raise FileNotFoundError(msg)
|
|
226
|
+
|
|
227
|
+
_log.debug(
|
|
228
|
+
"Reading external component %s (typespec=%s, length=%d)",
|
|
229
|
+
ref.identifier,
|
|
230
|
+
typespec,
|
|
231
|
+
ref.length,
|
|
232
|
+
)
|
|
233
|
+
data = file_path.read_bytes()
|
|
234
|
+
|
|
235
|
+
if typespec in _BIT_TYPESPECS:
|
|
236
|
+
is_float = "ieeefloat" in typespec
|
|
237
|
+
signed = not ("uint" in typespec or is_float)
|
|
238
|
+
big_endian = typespec.endswith("_beo")
|
|
239
|
+
ods_dt = _bit_ods_dt(typespec, ref.bitcount)
|
|
240
|
+
values = _read_bit_values(data, ref, signed, is_float, big_endian)
|
|
241
|
+
return TypedValues(ods_dt, values)
|
|
242
|
+
|
|
243
|
+
if typespec in _STRING_TYPESPECS:
|
|
244
|
+
strings = _read_null_terminated_strings(data, ref.inioffset, ref.length)
|
|
245
|
+
return TypedValues(ods_dt, strings)
|
|
246
|
+
|
|
247
|
+
if typespec in _BYTESTR_TYPESPECS:
|
|
248
|
+
big_endian = typespec == "dt_bytestr_beo"
|
|
249
|
+
bytestrings = _read_bytestr_sequence(data, ref.inioffset, ref.length, big_endian)
|
|
250
|
+
return TypedValues(ods_dt, bytestrings)
|
|
251
|
+
|
|
252
|
+
# Numeric / boolean: use numpy reader
|
|
253
|
+
arr = read_external_component(ref, file_map)
|
|
254
|
+
return TypedValues(ods_dt, arr.tolist())
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def read_external_component(ref: ExternalComponentRef, file_map: dict[str, Path]) -> NDArray[Any]:
|
|
258
|
+
"""Read binary values from an external file using the component reference.
|
|
259
|
+
|
|
260
|
+
Args:
|
|
261
|
+
ref: The external component reference with offset/size metadata.
|
|
262
|
+
file_map: Mapping from component identifier to resolved file path.
|
|
263
|
+
|
|
264
|
+
Returns:
|
|
265
|
+
Numpy array of decoded values.
|
|
266
|
+
"""
|
|
267
|
+
file_path = file_map.get(ref.identifier)
|
|
268
|
+
if file_path is None:
|
|
269
|
+
msg = f"External component identifier '{ref.identifier}' not found in file map"
|
|
270
|
+
raise FileNotFoundError(msg)
|
|
271
|
+
|
|
272
|
+
typespec = ref.datatype.lower()
|
|
273
|
+
if typespec not in _TYPESPEC_MAP:
|
|
274
|
+
# Fallback: read raw bytes
|
|
275
|
+
dtype_str, elem_size = "|u1", 1
|
|
276
|
+
else:
|
|
277
|
+
dtype_str, elem_size = _TYPESPEC_MAP[typespec]
|
|
278
|
+
dtype = np.dtype(dtype_str)
|
|
279
|
+
|
|
280
|
+
count = ref.length
|
|
281
|
+
if count <= 0:
|
|
282
|
+
return np.array([], dtype=dtype)
|
|
283
|
+
|
|
284
|
+
data = file_path.read_bytes()
|
|
285
|
+
|
|
286
|
+
blocksize = ref.blocksize if ref.blocksize > 0 else elem_size
|
|
287
|
+
valperblock = ref.valperblock if ref.valperblock > 0 else 1
|
|
288
|
+
inioffset = ref.inioffset
|
|
289
|
+
valoffset = ref.valoffsets[0] if ref.valoffsets else 0
|
|
290
|
+
|
|
291
|
+
if blocksize == elem_size and valperblock == 1 and valoffset == 0:
|
|
292
|
+
# Simple contiguous case: values packed sequentially from inioffset
|
|
293
|
+
start = inioffset
|
|
294
|
+
end = start + count * elem_size
|
|
295
|
+
return np.frombuffer(data[start:end], dtype=dtype, count=count)
|
|
296
|
+
|
|
297
|
+
# General case: values interleaved in blocks
|
|
298
|
+
values: list[Any] = []
|
|
299
|
+
offset = inioffset
|
|
300
|
+
remaining = count
|
|
301
|
+
while remaining > 0:
|
|
302
|
+
n = min(valperblock, remaining)
|
|
303
|
+
for i in range(n):
|
|
304
|
+
val_start = offset + valoffset + i * elem_size
|
|
305
|
+
val_end = val_start + elem_size
|
|
306
|
+
val = np.frombuffer(data[val_start:val_end], dtype=dtype, count=1)[0]
|
|
307
|
+
values.append(val)
|
|
308
|
+
remaining -= n
|
|
309
|
+
offset += blocksize
|
|
310
|
+
|
|
311
|
+
return np.array(values, dtype=dtype)
|