decode-easyae-dta 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- decode_easyae_dta-0.1.0.dist-info/METADATA +193 -0
- decode_easyae_dta-0.1.0.dist-info/RECORD +7 -0
- decode_easyae_dta-0.1.0.dist-info/WHEEL +4 -0
- decode_easyae_dta-0.1.0.dist-info/entry_points.txt +2 -0
- decode_easyae_dta-0.1.0.dist-info/licenses/LICENSE +21 -0
- decode_easyae_dta.py +445 -0
- mistras_dta.py +11 -0
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: decode-easyae-dta
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Decode AEwin64 / EasyAE DTA hit and time-driven data and export CSV
|
|
5
|
+
Project-URL: Homepage, https://github.com/spier16/decode-easyae-dta
|
|
6
|
+
Project-URL: Source, https://github.com/spier16/decode-easyae-dta
|
|
7
|
+
Project-URL: Issues, https://github.com/spier16/decode-easyae-dta/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/spier16/decode-easyae-dta/blob/main/CHANGELOG.md
|
|
9
|
+
Author: Stephen Pierson
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: AEwin64,DTA,EasyAE,MISTRAS,acoustic-emission
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Environment :: Console
|
|
15
|
+
Classifier: Intended Audience :: Science/Research
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
19
|
+
Classifier: Topic :: Scientific/Engineering
|
|
20
|
+
Requires-Python: >=3.10
|
|
21
|
+
Provides-Extra: dev
|
|
22
|
+
Requires-Dist: build>=1.2; extra == 'dev'
|
|
23
|
+
Requires-Dist: hatchling>=1.26; extra == 'dev'
|
|
24
|
+
Requires-Dist: twine>=6; extra == 'dev'
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
|
|
27
|
+
# decode-easyae-dta
|
|
28
|
+
|
|
29
|
+
Read hits and time-driven data directly from MISTRAS EasyAE / AEwin64 `.DTA` files using
|
|
30
|
+
Python 3.10 or newer and the standard library.
|
|
31
|
+
|
|
32
|
+
The EMPIRE acceptance corpus passes: **180 files, 1,056,916 decoded hits**.
|
|
33
|
+
All **20 columns** agree with all **32 manual AEwin64 exports**, covering
|
|
34
|
+
170,345 hits and 3,406,900 values, at the exports' printed precision.
|
|
35
|
+
|
|
36
|
+
Time-data extraction also passes: **180 files, 1,705,249 decoded rows**.
|
|
37
|
+
All **eight columns** match all **32 manual time exports**, covering 277,656
|
|
38
|
+
rows and 2,221,248 values.
|
|
39
|
+
|
|
40
|
+
## Install
|
|
41
|
+
|
|
42
|
+
```shell
|
|
43
|
+
python -m pip install decode-easyae-dta
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
From a checkout, including before the first PyPI release, use `python -m pip install .`.
|
|
47
|
+
The package has no runtime dependencies. It provides the `decode-easyae-dta`
|
|
48
|
+
command and the `decode_easyae_dta` import. Earlier `mistras_dta` imports and
|
|
49
|
+
`python mistras_dta.py ...` commands remain compatible.
|
|
50
|
+
|
|
51
|
+
## Read hits
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
from decode_easyae_dta import read_hits, iter_hits
|
|
55
|
+
|
|
56
|
+
hits = read_hits(r"C:\path\to\experiment.DTA")
|
|
57
|
+
print(len(hits), hits[0])
|
|
58
|
+
|
|
59
|
+
# Stream larger recordings; exhaust the iterator to validate the whole file.
|
|
60
|
+
for hit in iter_hits(r"C:\path\to\experiment.DTA"):
|
|
61
|
+
amplitude = hit["amplitude_db"]
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
Both `read_hits()` and `read_time_data()` return **lists of dictionaries**, not
|
|
65
|
+
NumPy arrays or pandas DataFrames. Their `iter_` variants yield dictionaries one
|
|
66
|
+
at a time. For optional pandas use, call `pd.DataFrame(read_hits(path))`.
|
|
67
|
+
|
|
68
|
+
Each hit is a dictionary containing `id` (message type 1), `time_s` (relative
|
|
69
|
+
to the recording clock), `channel`, the configured features in physical units,
|
|
70
|
+
and `parametric_1` etc. Parametrics are volts under the supported unity-scaling
|
|
71
|
+
profile. Feature keys identify units: `_us` for microseconds, `_khz` for kHz,
|
|
72
|
+
`_db` for dB, `_v` for volts, `_pvs` for pVs and `_aj` for aJ. Counts and
|
|
73
|
+
`energy` retain instrument counts. Hits retain file order.
|
|
74
|
+
|
|
75
|
+
## Extract CSV
|
|
76
|
+
|
|
77
|
+
```powershell
|
|
78
|
+
decode-easyae-dta 'C:\path\to\experiment.DTA' --output results/experiment_hits.csv
|
|
79
|
+
|
|
80
|
+
# Extract the time data instead:
|
|
81
|
+
decode-easyae-dta 'C:\path\to\experiment.DTA' --kind time --output results/experiment_time.csv
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
The command validates the input before creating the CSV, preserves numeric
|
|
85
|
+
precision, and requires a new output filename. CSV headings and column order match
|
|
86
|
+
the AEwin64 exports, including `PARA1`, `SIG STRNGTH`, and `R-FRQ`.
|
|
87
|
+
|
|
88
|
+
Each CSV starts with four metadata rows and a blank line:
|
|
89
|
+
|
|
90
|
+
```csv
|
|
91
|
+
Source File,C:\path\to\experiment.DTA
|
|
92
|
+
Product,EasyAE (r) Location Version
|
|
93
|
+
Version,V1.00
|
|
94
|
+
Test Start,Mon Oct 26 11:33:41 2026
|
|
95
|
+
|
|
96
|
+
ID,SSSSSSSS.mmmuuun,PARA1,CH,...
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
The product/version come from binary message ID 41; the test-start text comes
|
|
100
|
+
from ID 99. The source path is the current input file's absolute path. The stored
|
|
101
|
+
test time has no timezone, and missing metadata is shown as `Not recorded`.
|
|
102
|
+
Use `--no-metadata` for a CSV beginning directly with the data column headings.
|
|
103
|
+
For pandas, read the default CSV using `pd.read_csv(path, skiprows=5)`.
|
|
104
|
+
|
|
105
|
+
Metadata is also available separately in Python:
|
|
106
|
+
|
|
107
|
+
```python
|
|
108
|
+
from decode_easyae_dta import read_metadata
|
|
109
|
+
|
|
110
|
+
metadata = read_metadata(r"C:\path\to\experiment.DTA")
|
|
111
|
+
print(metadata.source_file, metadata.test_start)
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
## Read time data
|
|
115
|
+
|
|
116
|
+
```python
|
|
117
|
+
from decode_easyae_dta import read_time_data, iter_time_data, write_time_csv, read_metadata
|
|
118
|
+
|
|
119
|
+
path = r"C:\path\to\experiment.DTA"
|
|
120
|
+
rows = read_time_data(path)
|
|
121
|
+
print(len(rows), rows[0])
|
|
122
|
+
|
|
123
|
+
with open("experiment_time.csv", "x", encoding="utf-8", newline="") as stream:
|
|
124
|
+
write_time_csv(rows, stream, metadata=read_metadata(path))
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
The time CSV uses `ID,SSSSSSSS.mmmuuun,PARA1,CH,RMS,THR,ABS-ENERGY,ASL`
|
|
128
|
+
for this corpus. AEwin64's text exports place the channel values in a separate
|
|
129
|
+
bracketed block below each timestamp. The CSV flattens that block into one row
|
|
130
|
+
per timestamp and channel, repeating timestamp and parametric values across
|
|
131
|
+
channels. The corpus uses one channel. Numeric values retain full decoded
|
|
132
|
+
precision; timestamps use the recording clock, as in the hit CSV.
|
|
133
|
+
|
|
134
|
+
Python rows use the same descriptive keys and physical units as hits:
|
|
135
|
+
`time_s`, `parametric_1`, `channel`, `rms_v`, `threshold_db`,
|
|
136
|
+
`absolute_energy_aj`, `asl_db`, and `id` (2 for time-driven records).
|
|
137
|
+
`iter_time_data(path)` streams these rows. Manufacturer-documented ID 3 sample
|
|
138
|
+
records use the same layout and retain ID 3; none occur in this corpus.
|
|
139
|
+
|
|
140
|
+
## Validate
|
|
141
|
+
|
|
142
|
+
```powershell
|
|
143
|
+
python -m unittest -v
|
|
144
|
+
$env:EMPIRE_AE_DATA = 'C:\path\to\EMPIRE_acoustic_tests'
|
|
145
|
+
python validate_mistras_dta.py "$env:EMPIRE_AE_DATA" --output results/empire_hit_validation.json
|
|
146
|
+
python validate_mistras_dta.py "$env:EMPIRE_AE_DATA" --kind time --output results/empire_time_validation.json
|
|
147
|
+
|
|
148
|
+
# Or compare a single pair:
|
|
149
|
+
python validate_mistras_dta.py 'C:\path\to\experiment.DTA' --export 'C:\path\to\experiment_hits.TXT'
|
|
150
|
+
python validate_mistras_dta.py 'C:\path\to\experiment.DTA' --kind time --export 'C:\path\to\experiment_time.TXT'
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
The corpus command visits numbered `EMPIRE_acoustic_<number>` folders, excluding
|
|
154
|
+
`misc`. It reads every DTA, compares every column in every corresponding
|
|
155
|
+
`<DTA stem>_hits.txt` export (or `_time.txt` with `--kind time`), and writes a
|
|
156
|
+
report outside the raw-data directory.
|
|
157
|
+
The acquisition corpus and generated reports are kept locally and are not
|
|
158
|
+
included in the public repository or package.
|
|
159
|
+
Counts and integer features must agree exactly; other values must agree within
|
|
160
|
+
half the last printed decimal place. Failures produce a nonzero exit status.
|
|
161
|
+
The comparator checks schema equality and treats `SIG STRNGTH` as one column.
|
|
162
|
+
|
|
163
|
+
The earlier framing-only survey remains available:
|
|
164
|
+
|
|
165
|
+
```powershell
|
|
166
|
+
python survey_mistras_dta.py "$env:EMPIRE_AE_DATA" --output results/empire_dta_survey.json
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
## Supported scope
|
|
170
|
+
|
|
171
|
+
This reader supports the **EasyAE V1.00 profile and calibration/scaling metadata
|
|
172
|
+
verified in the EMPIRE corpus**. It reads the configured feature order and checks
|
|
173
|
+
the product, hardware version, preamp/gain, channel calibration and parametric
|
|
174
|
+
scaling records. Unsupported characteristics or calibration settings raise
|
|
175
|
+
`DtaFormatError`, including byte offsets. General custom multiplier/offset
|
|
176
|
+
decoding remains unimplemented; the verified unity-scaling record is recognized
|
|
177
|
+
as a whole. Agreement at export precision does not establish extra digits of
|
|
178
|
+
parametric calibration accuracy.
|
|
179
|
+
|
|
180
|
+
Start markers are optional; the extra control byte is safely framed. Unknown
|
|
181
|
+
message IDs are skipped using declared boundaries. Waveforms,
|
|
182
|
+
continued-file setup and other instrument profiles are outside this milestone.
|
|
183
|
+
Time-data characteristics are restricted to the verified THR, RMS, ABS-ENERGY
|
|
184
|
+
and ASL encodings. Custom parametric scaling and cycle counters are unsupported.
|
|
185
|
+
Two-channel decoding also matches a separate real dual-sensor acquisition's
|
|
186
|
+
hit and time exports.
|
|
187
|
+
Multiple analog parametrics, schema updates, and ID 3 have synthetic coverage
|
|
188
|
+
only. The real time exports exercise PARA1.
|
|
189
|
+
|
|
190
|
+
Raw binaries, TXT exports and the vendor manual stay external to this repository.
|
|
191
|
+
The project uses the MIT license in [LICENSE](https://github.com/spier16/decode-easyae-dta/blob/main/LICENSE).
|
|
192
|
+
For development, see [CONTRIBUTING.md](https://github.com/spier16/decode-easyae-dta/blob/main/CONTRIBUTING.md). The separate
|
|
193
|
+
[decode-wfs](https://github.com/spier16/decode-wfs) project decodes waveform streams.
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
decode_easyae_dta.py,sha256=xVQjSFL8JDfRkMupaW1hyV8R_PMHeL2ApKuQZYbVB0w,19967
|
|
2
|
+
mistras_dta.py,sha256=f9KvViDCIuxHQ8BiRBFwNAs3SIGowlG6_08wn0EaGEk,389
|
|
3
|
+
decode_easyae_dta-0.1.0.dist-info/METADATA,sha256=CALQ-6lx9aCb_tgOYeTSX3qCFQEXow1wZz9_qz7UNR8,8450
|
|
4
|
+
decode_easyae_dta-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
5
|
+
decode_easyae_dta-0.1.0.dist-info/entry_points.txt,sha256=cQRermCs6bb69zZEToShU-gl7zlexIECChqeKz6FdnQ,61
|
|
6
|
+
decode_easyae_dta-0.1.0.dist-info/licenses/LICENSE,sha256=oWqAmW_bhO7fGZ6Psv0_s6-CkMPE0IOqHaJE-R8FI1c,1066
|
|
7
|
+
decode_easyae_dta-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 UARK-NED3
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
decode_easyae_dta.py
ADDED
|
@@ -0,0 +1,445 @@
|
|
|
1
|
+
"""Native hit and time-data reader for the validated EasyAE V1.00 DTA profile.
|
|
2
|
+
|
|
3
|
+
See README.md for usage, physical units and supported-profile limits.
|
|
4
|
+
No third-party packages are required.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
import argparse
|
|
9
|
+
import csv
|
|
10
|
+
import math
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
import struct
|
|
13
|
+
from typing import BinaryIO, Iterator, Optional
|
|
14
|
+
|
|
15
|
+
__version__ = "0.1.0"
|
|
16
|
+
|
|
17
|
+
__all__ = [
|
|
18
|
+
"AEWIN_COLUMNS", "DtaFormatError", "DtaMetadata", "Message",
|
|
19
|
+
"event_characteristics", "iter_hits", "iter_messages", "iter_time_data",
|
|
20
|
+
"read_hits", "read_metadata", "read_time_data", "setup_subrecords",
|
|
21
|
+
"write_hits_csv", "write_time_csv",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class DtaFormatError(ValueError):
|
|
26
|
+
"""A DTA message or setup record is incomplete or malformed."""
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class Message:
|
|
31
|
+
offset: int
|
|
32
|
+
payload: bytes # Includes the ID and any extended header, excludes length.
|
|
33
|
+
|
|
34
|
+
@property
|
|
35
|
+
def id(self) -> int:
|
|
36
|
+
return self.payload[0]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass(frozen=True)
|
|
40
|
+
class DtaMetadata:
|
|
41
|
+
source_file: str
|
|
42
|
+
product: Optional[str] = None
|
|
43
|
+
version: Optional[str] = None
|
|
44
|
+
test_start: Optional[str] = None # Stored clock text; no timezone is supplied.
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def iter_messages(stream: BinaryIO) -> Iterator[Message]:
|
|
48
|
+
"""Read complete messages so unrecognized fields cannot shift the next ID.
|
|
49
|
+
|
|
50
|
+
Length is a little-endian unsigned 16-bit count of the payload bytes.
|
|
51
|
+
Unknown IDs and trailing bytes remain available in ``payload``.
|
|
52
|
+
"""
|
|
53
|
+
offset = 0
|
|
54
|
+
while True:
|
|
55
|
+
length_bytes = stream.read(2)
|
|
56
|
+
if not length_bytes:
|
|
57
|
+
return
|
|
58
|
+
if len(length_bytes) != 2:
|
|
59
|
+
raise DtaFormatError(f"Truncated length at byte {offset}")
|
|
60
|
+
length = int.from_bytes(length_bytes, "little")
|
|
61
|
+
if length == 0:
|
|
62
|
+
raise DtaFormatError(f"Empty message at byte {offset}")
|
|
63
|
+
payload = stream.read(length)
|
|
64
|
+
if len(payload) != length:
|
|
65
|
+
raise DtaFormatError(
|
|
66
|
+
f"Truncated message at byte {offset}: expected {length} payload "
|
|
67
|
+
f"bytes, got {len(payload)}"
|
|
68
|
+
)
|
|
69
|
+
yield Message(offset, payload)
|
|
70
|
+
offset += 2 + length
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def event_characteristics(message: Message) -> list[tuple[int, ...]]:
|
|
74
|
+
"""Extract ordered ID lists from ID 42 / sub-ID 5 setup records.
|
|
75
|
+
|
|
76
|
+
Other setup records are skipped by their declared lengths. The extended
|
|
77
|
+
ID and two-byte version occupy the first four bytes of message 42.
|
|
78
|
+
"""
|
|
79
|
+
result = []
|
|
80
|
+
for subrecord in setup_subrecords(message):
|
|
81
|
+
if subrecord[0] == 5:
|
|
82
|
+
if len(subrecord) < 2 or len(subrecord) < 2 + subrecord[1]:
|
|
83
|
+
raise DtaFormatError(f"Truncated characteristic list at byte {message.offset}")
|
|
84
|
+
result.append(tuple(subrecord[2:2 + subrecord[1]]))
|
|
85
|
+
return result
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def setup_subrecords(message: Message) -> Iterator[bytes]:
|
|
89
|
+
"""Yield bounded hardware setup subrecords, including their sub-ID."""
|
|
90
|
+
if message.id != 42:
|
|
91
|
+
return
|
|
92
|
+
payload = message.payload
|
|
93
|
+
if len(payload) < 4 or payload[1] != 0:
|
|
94
|
+
raise DtaFormatError(f"Invalid ID 42 header at byte {message.offset}")
|
|
95
|
+
offset = 4
|
|
96
|
+
while offset < len(payload):
|
|
97
|
+
if offset + 2 > len(payload):
|
|
98
|
+
raise DtaFormatError(f"Truncated setup length at byte {message.offset}")
|
|
99
|
+
length = int.from_bytes(payload[offset:offset + 2], "little")
|
|
100
|
+
end = offset + 2 + length
|
|
101
|
+
if length < 1 or end > len(payload):
|
|
102
|
+
raise DtaFormatError(f"Invalid setup boundary at byte {message.offset}")
|
|
103
|
+
subrecord = payload[offset + 2:end]
|
|
104
|
+
yield subrecord
|
|
105
|
+
offset = end
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _float32(value):
|
|
109
|
+
return struct.unpack("<f", struct.pack("<f", value))[0]
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
# (reader key, storage, physical-unit multiplier). Order comes from ID 5.
|
|
113
|
+
_FEATURES = {
|
|
114
|
+
1: ("rise_us", "H", 1), 2: ("counts_to_peak", "H", 1),
|
|
115
|
+
3: ("counts", "H", 1), 4: ("energy", "H", 1),
|
|
116
|
+
5: ("duration_us", "I", 1), 10: ("threshold_db", "B", 1),
|
|
117
|
+
13: ("average_frequency_khz", "H", 1), 17: ("rms_v", "H", 1 / 5000),
|
|
118
|
+
18: ("reverberation_frequency_khz", "H", 1),
|
|
119
|
+
19: ("initiation_frequency_khz", "H", 1),
|
|
120
|
+
20: ("signal_strength_pvs", "I", 3.05),
|
|
121
|
+
21: ("absolute_energy_aj", "f", 0.000931),
|
|
122
|
+
23: ("centroid_frequency_khz", "H", 1), 24: ("peak_frequency_khz", "H", 1),
|
|
123
|
+
27: ("amplitude_db", "H", 0.1), 28: ("asl_db", "H", 0.1),
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
# ID 107's product-specific layout is undocumented in the installed appendix.
|
|
127
|
+
# Recognize the verified unity-scaling blob, and reject other settings.
|
|
128
|
+
_UNITY_SCALING = b"\x6b\x10\x27\x08\x00" + bytes.fromhex("00 00 00 00 00 e8 03 00 00 00 00") * 8
|
|
129
|
+
_PRODUCT = b")\x00\xbc\x02EasyAE (r) Location Version \n\rVersion V1.00\n\r\n\r\x1a\x00"
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
class _HitDecoder:
|
|
133
|
+
def __init__(self):
|
|
134
|
+
self.product = False
|
|
135
|
+
self.scaling = False
|
|
136
|
+
self.schema = None
|
|
137
|
+
self.features = []
|
|
138
|
+
self.max_parametrics = 0
|
|
139
|
+
self.preamp = {}
|
|
140
|
+
self.gain = {}
|
|
141
|
+
self.calibration = set()
|
|
142
|
+
|
|
143
|
+
def set_features(self, ids, offset):
|
|
144
|
+
unknown = set(ids) - _FEATURES.keys()
|
|
145
|
+
if unknown:
|
|
146
|
+
raise DtaFormatError(f"Unsupported characteristic IDs {sorted(unknown)} at byte {offset}")
|
|
147
|
+
if len(ids) != len(set(ids)):
|
|
148
|
+
raise DtaFormatError(f"Duplicate characteristics at byte {offset}")
|
|
149
|
+
self.features = [(ident, *_FEATURES[ident]) for ident in ids]
|
|
150
|
+
self.schema = struct.Struct("<" + "".join(item[2] for item in self.features))
|
|
151
|
+
|
|
152
|
+
def configure(self, payload, offset):
|
|
153
|
+
ident = payload[0]
|
|
154
|
+
if ident == 5:
|
|
155
|
+
if len(payload) < 3 or len(payload) != 3 + payload[1]:
|
|
156
|
+
raise DtaFormatError(f"Invalid event definition at byte {offset}")
|
|
157
|
+
ids = payload[2:-1]
|
|
158
|
+
self.set_features(ids, offset)
|
|
159
|
+
self.max_parametrics = payload[-1]
|
|
160
|
+
if self.max_parametrics > 8:
|
|
161
|
+
raise DtaFormatError(f"Unsupported parametric count at byte {offset}")
|
|
162
|
+
elif ident == 20:
|
|
163
|
+
if len(payload) != 3 or payload[2] != 26:
|
|
164
|
+
raise DtaFormatError(f"Unsupported preamp calibration at byte {offset}")
|
|
165
|
+
self.preamp[payload[1]] = payload[2]
|
|
166
|
+
elif ident == 23:
|
|
167
|
+
if len(payload) != 4 or payload[2:] != b"\x00\x14":
|
|
168
|
+
raise DtaFormatError(f"Unsupported gain/reference gain at byte {offset}")
|
|
169
|
+
self.gain[payload[1]] = (payload[2], payload[3])
|
|
170
|
+
elif ident == 176:
|
|
171
|
+
if (len(payload) != 17 or payload[:4] != b"\xb0\x2a\x65\x00"
|
|
172
|
+
or payload[5:] != struct.pack("<dI", 1.0, 1000000)):
|
|
173
|
+
raise DtaFormatError(f"Unsupported channel calibration at byte {offset}")
|
|
174
|
+
self.calibration.add(payload[4])
|
|
175
|
+
|
|
176
|
+
def require_setup(self, offset, channel=None):
|
|
177
|
+
if not self.product or not self.scaling or self.schema is None:
|
|
178
|
+
raise DtaFormatError(f"Missing supported EasyAE product, scaling or data setup at byte {offset}")
|
|
179
|
+
if channel is not None and (channel not in self.preamp or channel not in self.gain
|
|
180
|
+
or channel not in self.calibration):
|
|
181
|
+
raise DtaFormatError(f"Missing calibration/gain for channel {channel} at byte {offset}")
|
|
182
|
+
|
|
183
|
+
def decode(self, message):
|
|
184
|
+
payload = message.payload
|
|
185
|
+
if len(payload) < 8:
|
|
186
|
+
raise DtaFormatError(f"Short hit at byte {message.offset}")
|
|
187
|
+
channel = payload[7]
|
|
188
|
+
self.require_setup(message.offset, channel)
|
|
189
|
+
end = 8 + self.schema.size
|
|
190
|
+
remaining = len(payload) - end
|
|
191
|
+
if remaining < 0 or remaining % 3 or remaining // 3 > self.max_parametrics:
|
|
192
|
+
raise DtaFormatError(f"Invalid hit payload length {len(payload)} at byte {message.offset}")
|
|
193
|
+
hit = {"id": 1, "time_s": int.from_bytes(payload[1:7], "little") / 4000000,
|
|
194
|
+
"channel": channel}
|
|
195
|
+
hit.update(self.decode_features(payload, 8, message.offset))
|
|
196
|
+
for start in range(end, len(payload), 3):
|
|
197
|
+
pid = payload[start]
|
|
198
|
+
key = f"parametric_{pid}"
|
|
199
|
+
if not 1 <= pid <= 8 or key in hit:
|
|
200
|
+
raise DtaFormatError(f"Unsupported/duplicate parametric ID {pid} at byte {message.offset}")
|
|
201
|
+
# EasyAE specifies a 0-10 V, 16-bit input. ID 107 has unity scaling.
|
|
202
|
+
hit[key] = int.from_bytes(payload[start + 1:start + 3], "little") * (10 / 65536)
|
|
203
|
+
return hit
|
|
204
|
+
|
|
205
|
+
def decode_features(self, payload, start, offset):
|
|
206
|
+
values = {}
|
|
207
|
+
for (ident, key, _, multiplier), raw in zip(self.features, self.schema.unpack_from(payload, start)):
|
|
208
|
+
if not math.isfinite(raw) or raw < 0 or (ident in (5, 20) and raw > 0x7fffffff):
|
|
209
|
+
raise DtaFormatError(f"Invalid or unsupported {key} value at byte {offset}")
|
|
210
|
+
if ident in (20, 21):
|
|
211
|
+
value = _float32(_float32(raw) * _float32(multiplier))
|
|
212
|
+
elif ident in (27, 28):
|
|
213
|
+
value = raw / 10
|
|
214
|
+
else:
|
|
215
|
+
value = raw * multiplier
|
|
216
|
+
if not math.isfinite(value):
|
|
217
|
+
raise DtaFormatError(f"Non-finite {key} at byte {offset}")
|
|
218
|
+
values[key] = value
|
|
219
|
+
return values
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
class _TimeDecoder(_HitDecoder):
|
|
223
|
+
def __init__(self):
|
|
224
|
+
super().__init__()
|
|
225
|
+
self.parametric_ids = ()
|
|
226
|
+
|
|
227
|
+
def configure(self, payload, offset):
|
|
228
|
+
if payload[0] == 5:
|
|
229
|
+
return # Hit definitions do not change the time-data schema.
|
|
230
|
+
if payload[0] != 6:
|
|
231
|
+
return super().configure(payload, offset)
|
|
232
|
+
if len(payload) < 3 or len(payload) < 3 + payload[1]:
|
|
233
|
+
raise DtaFormatError(f"Truncated time-data definition at byte {offset}")
|
|
234
|
+
end = 2 + payload[1]
|
|
235
|
+
ids = payload[2:end]
|
|
236
|
+
if set(ids) - {10, 17, 21, 28}:
|
|
237
|
+
raise DtaFormatError(f"Unsupported time-data characteristics {list(ids)} at byte {offset}")
|
|
238
|
+
if len(payload) != end + 1 + payload[end]:
|
|
239
|
+
raise DtaFormatError(f"Invalid time parametric definition at byte {offset}")
|
|
240
|
+
pids = tuple(payload[end + 1:])
|
|
241
|
+
if len(pids) != len(set(pids)) or any(not 1 <= pid <= 8 for pid in pids):
|
|
242
|
+
raise DtaFormatError(f"Unsupported/duplicate time parametric IDs at byte {offset}")
|
|
243
|
+
self.set_features(ids, offset)
|
|
244
|
+
self.parametric_ids = pids
|
|
245
|
+
|
|
246
|
+
def decode(self, message):
|
|
247
|
+
payload, offset = message.payload, message.offset
|
|
248
|
+
self.require_setup(offset)
|
|
249
|
+
end = 7 + 3 * len(self.parametric_ids)
|
|
250
|
+
if len(payload) < end:
|
|
251
|
+
raise DtaFormatError(f"Short time-data payload at byte {offset}")
|
|
252
|
+
common = {"id": message.id, "time_s": int.from_bytes(payload[1:7], "little") / 4000000}
|
|
253
|
+
for start, pid in zip(range(7, end, 3), self.parametric_ids):
|
|
254
|
+
if payload[start] != pid:
|
|
255
|
+
raise DtaFormatError(f"Time parametric ID differs from setup at byte {offset}")
|
|
256
|
+
common[f"parametric_{pid}"] = int.from_bytes(payload[start + 1:start + 3], "little") * (10 / 65536)
|
|
257
|
+
if not self.features:
|
|
258
|
+
if len(payload) != end:
|
|
259
|
+
raise DtaFormatError(f"Unexpected time-data channel bytes at byte {offset}")
|
|
260
|
+
return [common]
|
|
261
|
+
width = 1 + self.schema.size
|
|
262
|
+
if len(payload) == end or (len(payload) - end) % width:
|
|
263
|
+
raise DtaFormatError(f"Invalid time-data channel length at byte {offset}")
|
|
264
|
+
rows, channels = [], set()
|
|
265
|
+
for start in range(end, len(payload), width):
|
|
266
|
+
channel = payload[start]
|
|
267
|
+
if channel in channels:
|
|
268
|
+
raise DtaFormatError(f"Duplicate time-data channel {channel} at byte {offset}")
|
|
269
|
+
self.require_setup(offset, channel)
|
|
270
|
+
channels.add(channel)
|
|
271
|
+
rows.append({**common, "channel": channel,
|
|
272
|
+
**self.decode_features(payload, start + 1, offset)})
|
|
273
|
+
return rows
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def iter_hits(path):
|
|
277
|
+
"""Yield hit dictionaries in file order, in physical units.
|
|
278
|
+
|
|
279
|
+
Unknown messages are skipped by their declared boundaries. Unsupported
|
|
280
|
+
product/calibration/feature layouts raise DtaFormatError. A start marker
|
|
281
|
+
is optional. Exhaust the iterator to validate the complete file.
|
|
282
|
+
"""
|
|
283
|
+
yield from _iter_data(path, _HitDecoder(), (1,))
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def _iter_data(path, decoder, message_ids):
|
|
287
|
+
with Path(path).open("rb") as stream:
|
|
288
|
+
for message in iter_messages(stream):
|
|
289
|
+
if message.id == 41:
|
|
290
|
+
if message.payload != _PRODUCT:
|
|
291
|
+
raise DtaFormatError(f"Unsupported product definition at byte {message.offset}")
|
|
292
|
+
decoder.product = True
|
|
293
|
+
elif message.id == 107:
|
|
294
|
+
if message.payload != _UNITY_SCALING:
|
|
295
|
+
raise DtaFormatError(f"Unsupported parametric scaling at byte {message.offset}")
|
|
296
|
+
decoder.scaling = True
|
|
297
|
+
elif message.id == 42:
|
|
298
|
+
if message.payload[2:4] != b"\x67\x00":
|
|
299
|
+
raise DtaFormatError(f"Unsupported hardware setup version at byte {message.offset}")
|
|
300
|
+
for payload in setup_subrecords(message):
|
|
301
|
+
decoder.configure(payload, message.offset)
|
|
302
|
+
elif message.id in (5, 6, 20, 23, 176):
|
|
303
|
+
decoder.configure(message.payload, message.offset)
|
|
304
|
+
elif message.id == 8:
|
|
305
|
+
raise DtaFormatError(f"Continued-file setup is unsupported at byte {message.offset}")
|
|
306
|
+
elif message.id in message_ids:
|
|
307
|
+
decoded = decoder.decode(message)
|
|
308
|
+
if isinstance(decoder, _TimeDecoder):
|
|
309
|
+
yield from decoded
|
|
310
|
+
else:
|
|
311
|
+
yield decoded
|
|
312
|
+
decoder.require_setup(stream.tell())
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def read_hits(path):
|
|
316
|
+
"""Return all decoded hit dictionaries; see iter_hits for streaming reads."""
|
|
317
|
+
return list(iter_hits(path))
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def iter_time_data(path):
|
|
321
|
+
"""Yield time-driven (ID 2) and forced-sample (ID 3) rows in file order.
|
|
322
|
+
|
|
323
|
+
Each channel becomes a row, repeating its timestamp and parametric values.
|
|
324
|
+
Parametric-only records have no channel field. Exhaust to validate the file.
|
|
325
|
+
"""
|
|
326
|
+
yield from _iter_data(path, _TimeDecoder(), (2, 3))
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def read_time_data(path):
|
|
330
|
+
"""Return all time-data rows; see iter_time_data for streaming reads."""
|
|
331
|
+
return list(iter_time_data(path))
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def _metadata_text(message, start):
|
|
335
|
+
try:
|
|
336
|
+
text = message.payload[start:].rstrip(b"\x00\x1a\r\n ").decode("ascii")
|
|
337
|
+
except UnicodeDecodeError as error:
|
|
338
|
+
raise DtaFormatError(f"Non-ASCII metadata in ID {message.id} at byte {message.offset}") from error
|
|
339
|
+
if not text.strip() or "\x00" in text:
|
|
340
|
+
raise DtaFormatError(f"Invalid metadata in ID {message.id} at byte {message.offset}")
|
|
341
|
+
return text
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def read_metadata(path):
|
|
345
|
+
"""Read product (ID 41) and test-start text (ID 99) from the DTA.
|
|
346
|
+
|
|
347
|
+
The source path is the supplied file's resolved location. Missing fields
|
|
348
|
+
remain None; filesystem dates are never used as recording timestamps.
|
|
349
|
+
"""
|
|
350
|
+
path = Path(path)
|
|
351
|
+
product = version = test_start = None
|
|
352
|
+
with path.open("rb") as stream:
|
|
353
|
+
for message in iter_messages(stream):
|
|
354
|
+
if message.id == 41:
|
|
355
|
+
if len(message.payload) < 4 or message.payload[1] != 0:
|
|
356
|
+
raise DtaFormatError(f"Invalid ID 41 header at byte {message.offset}")
|
|
357
|
+
lines = [line.strip() for line in _metadata_text(message, 4).splitlines() if line.strip()]
|
|
358
|
+
new_product = lines[0]
|
|
359
|
+
new_version = next((line.removeprefix("Version ") for line in lines[1:]
|
|
360
|
+
if line.startswith("Version ")), None)
|
|
361
|
+
if product is not None and (product, version) != (new_product, new_version):
|
|
362
|
+
raise DtaFormatError(f"Conflicting product metadata at byte {message.offset}")
|
|
363
|
+
product, version = new_product, new_version
|
|
364
|
+
elif message.id == 99:
|
|
365
|
+
text = _metadata_text(message, 1)
|
|
366
|
+
if len(text.splitlines()) != 1:
|
|
367
|
+
raise DtaFormatError(f"Invalid test-start text at byte {message.offset}")
|
|
368
|
+
if test_start is not None and test_start != text:
|
|
369
|
+
raise DtaFormatError(f"Conflicting test-start records at byte {message.offset}")
|
|
370
|
+
test_start = text
|
|
371
|
+
return DtaMetadata(str(path.resolve()), product, version, test_start)
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
# AEwin64's ASCII export order; only columns present in the hits are written.
|
|
375
|
+
AEWIN_COLUMNS = {
|
|
376
|
+
"ID": "id", "SSSSSSSS.mmmuuun": "time_s",
|
|
377
|
+
**{f"PARA{pid}": f"parametric_{pid}" for pid in range(1, 9)},
|
|
378
|
+
"CH": "channel", "RISE": "rise_us", "COUN": "counts", "ENER": "energy",
|
|
379
|
+
"DURATION": "duration_us", "A-FRQ": "average_frequency_khz", "RMS": "rms_v",
|
|
380
|
+
"PCNTS": "counts_to_peak", "THR": "threshold_db",
|
|
381
|
+
"R-FRQ": "reverberation_frequency_khz", "I-FRQ": "initiation_frequency_khz",
|
|
382
|
+
"SIG STRNGTH": "signal_strength_pvs", "ABS-ENERGY": "absolute_energy_aj",
|
|
383
|
+
"FRQ-C": "centroid_frequency_khz", "P-FRQ": "peak_frequency_khz",
|
|
384
|
+
"AMP": "amplitude_db", "ASL": "asl_db",
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
def write_hits_csv(hits, stream, *, metadata=None):
|
|
389
|
+
"""Write AEwin64 hit columns, optionally preceded by recording metadata."""
|
|
390
|
+
_write_data_csv(hits, stream, metadata=metadata, empty_fields={"id", "time_s", "channel"})
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def write_time_csv(rows, stream, *, metadata=None):
|
|
394
|
+
"""Write a regular time-data table using AEwin64 column names."""
|
|
395
|
+
_write_data_csv(rows, stream, metadata=metadata, empty_fields={"id", "time_s"})
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
def _write_data_csv(hits, stream, *, metadata, empty_fields):
|
|
399
|
+
hits = list(hits)
|
|
400
|
+
fields = {key for hit in hits for key in hit} or empty_fields
|
|
401
|
+
unknown = fields - set(AEWIN_COLUMNS.values())
|
|
402
|
+
if unknown:
|
|
403
|
+
raise ValueError(f"Unsupported CSV fields: {sorted(unknown)}")
|
|
404
|
+
columns = [(heading, key) for heading, key in AEWIN_COLUMNS.items() if key in fields]
|
|
405
|
+
writer = csv.writer(stream)
|
|
406
|
+
if metadata is not None:
|
|
407
|
+
writer.writerows([
|
|
408
|
+
["Source File", metadata.source_file],
|
|
409
|
+
["Product", metadata.product or "Not recorded"],
|
|
410
|
+
["Version", metadata.version or "Not recorded"],
|
|
411
|
+
["Test Start", metadata.test_start or "Not recorded"],
|
|
412
|
+
[],
|
|
413
|
+
])
|
|
414
|
+
writer.writerow([heading for heading, _ in columns])
|
|
415
|
+
writer.writerows([hit.get(key, "") for _, key in columns] for hit in hits)
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
def main():
|
|
419
|
+
parser = argparse.ArgumentParser(description="Extract EasyAE DTA hits or time data to CSV")
|
|
420
|
+
parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
|
421
|
+
parser.add_argument("path", type=Path, help="Input DTA file")
|
|
422
|
+
parser.add_argument("--output", type=Path, required=True, help="New CSV file (must not exist)")
|
|
423
|
+
parser.add_argument("--kind", choices=("hits", "time"), default="hits", help="Data to extract (default: hits)")
|
|
424
|
+
parser.add_argument("--no-metadata", action="store_true", help="Write only the data table, without the metadata preamble")
|
|
425
|
+
args = parser.parse_args()
|
|
426
|
+
if args.output.suffix.lower() != ".csv":
|
|
427
|
+
parser.error("output must have a .csv extension")
|
|
428
|
+
if args.output.resolve() == args.path.resolve():
|
|
429
|
+
parser.error("output must differ from the input")
|
|
430
|
+
try:
|
|
431
|
+
# Validate the entire input before opening an output file.
|
|
432
|
+
hits = read_hits(args.path) if args.kind == "hits" else read_time_data(args.path)
|
|
433
|
+
metadata = None if args.no_metadata else read_metadata(args.path)
|
|
434
|
+
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
435
|
+
with args.output.open("x", encoding="utf-8", newline="") as stream:
|
|
436
|
+
writer = write_hits_csv if args.kind == "hits" else write_time_csv
|
|
437
|
+
writer(hits, stream, metadata=metadata)
|
|
438
|
+
except (OSError, ValueError) as error:
|
|
439
|
+
parser.exit(1, f"{error}\n")
|
|
440
|
+
label = "hits" if args.kind == "hits" else "time rows"
|
|
441
|
+
print(f"Extracted {len(hits)} {label} to {args.output}")
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
if __name__ == "__main__":
|
|
445
|
+
main()
|
mistras_dta.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""Compatibility module; new code should import decode_easyae_dta."""
|
|
2
|
+
|
|
3
|
+
from decode_easyae_dta import (
|
|
4
|
+
AEWIN_COLUMNS, DtaFormatError, DtaMetadata, Message, __version__,
|
|
5
|
+
event_characteristics, iter_hits, iter_messages, iter_time_data,
|
|
6
|
+
read_hits, read_metadata, read_time_data, setup_subrecords,
|
|
7
|
+
write_hits_csv, write_time_csv, main,
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
if __name__ == "__main__":
|
|
11
|
+
main()
|