harp-python 0.1.0__py3-none-any.whl → 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- harp/__init__.py +3 -0
- harp/io.py +91 -20
- harp/model.py +72 -54
- harp/reader.py +133 -31
- harp/schema.py +37 -15
- {harp_python-0.1.0.dist-info → harp_python-0.2.0.dist-info}/METADATA +33 -16
- harp_python-0.2.0.dist-info/RECORD +11 -0
- harp_python-0.1.0.dist-info/RECORD +0 -10
- {harp_python-0.1.0.dist-info → harp_python-0.2.0.dist-info}/LICENSE +0 -0
- {harp_python-0.1.0.dist-info → harp_python-0.2.0.dist-info}/WHEEL +0 -0
- {harp_python-0.1.0.dist-info → harp_python-0.2.0.dist-info}/top_level.txt +0 -0
harp/__init__.py
ADDED
harp/io.py
CHANGED
|
@@ -1,11 +1,27 @@
|
|
|
1
|
+
from datetime import datetime
|
|
2
|
+
from enum import IntEnum
|
|
1
3
|
from os import PathLike
|
|
2
4
|
from typing import Any, BinaryIO, Optional, Union
|
|
3
5
|
from pandas._typing import Axes
|
|
4
6
|
import numpy as np
|
|
5
7
|
import pandas as pd
|
|
6
8
|
|
|
9
|
+
REFERENCE_EPOCH = datetime(1904, 1, 1)
|
|
10
|
+
"""The reference epoch for UTC harp time."""
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class MessageType(IntEnum):
|
|
14
|
+
"""Specifies the type of a Harp message."""
|
|
15
|
+
|
|
16
|
+
NA = 0
|
|
17
|
+
READ = 1
|
|
18
|
+
WRITE = 2
|
|
19
|
+
EVENT = 3
|
|
20
|
+
|
|
21
|
+
|
|
7
22
|
_SECONDS_PER_TICK = 32e-6
|
|
8
|
-
|
|
23
|
+
_messagetypes = [type.name for type in MessageType]
|
|
24
|
+
_payloadtypes = {
|
|
9
25
|
1: np.dtype(np.uint8),
|
|
10
26
|
2: np.dtype(np.uint16),
|
|
11
27
|
4: np.dtype(np.uint32),
|
|
@@ -20,39 +36,94 @@ payloadtypes = {
|
|
|
20
36
|
|
|
21
37
|
def read(
|
|
22
38
|
file: Union[str, bytes, PathLike[Any], BinaryIO],
|
|
39
|
+
address: Optional[int] = None,
|
|
40
|
+
dtype: Optional[np.dtype] = None,
|
|
41
|
+
length: Optional[int] = None,
|
|
23
42
|
columns: Optional[Axes] = None,
|
|
43
|
+
epoch: Optional[datetime] = None,
|
|
44
|
+
keep_type: bool = False,
|
|
24
45
|
):
|
|
25
|
-
"""
|
|
26
|
-
Read single-register Harp data from the specified file.
|
|
46
|
+
"""Read single-register Harp data from the specified file.
|
|
27
47
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
48
|
+
Parameters
|
|
49
|
+
----------
|
|
50
|
+
file
|
|
51
|
+
Open file object or filename containing binary data from
|
|
52
|
+
a single device register.
|
|
53
|
+
address
|
|
54
|
+
Expected register address. If specified, the address of
|
|
55
|
+
the first message in the file is used for validation.
|
|
56
|
+
dtype
|
|
57
|
+
Expected data type of the register payload. If specified, the
|
|
58
|
+
payload type of the first message in the file is used for validation.
|
|
59
|
+
length
|
|
60
|
+
Expected number of elements in register payload. If specified, the
|
|
61
|
+
payload length of the first message in the file is used for validation.
|
|
62
|
+
columns
|
|
63
|
+
The optional column labels to use for the data values.
|
|
64
|
+
epoch
|
|
65
|
+
Reference datetime at which time zero begins. If specified,
|
|
66
|
+
the result data frame will have a datetime index.
|
|
67
|
+
keep_type
|
|
68
|
+
Specifies whether to include a column with the message type.
|
|
69
|
+
|
|
70
|
+
Returns
|
|
71
|
+
-------
|
|
72
|
+
A pandas data frame containing message data, sorted by time.
|
|
33
73
|
"""
|
|
34
74
|
data = np.fromfile(file, dtype=np.uint8)
|
|
35
75
|
if len(data) == 0:
|
|
36
76
|
return pd.DataFrame(
|
|
37
|
-
columns=columns, index=pd.Index([], dtype=np.float64, name="
|
|
77
|
+
columns=columns, index=pd.Index([], dtype=np.float64, name="Time")
|
|
38
78
|
)
|
|
39
79
|
|
|
80
|
+
if address is not None and address != data[2]:
|
|
81
|
+
raise ValueError(f"expected address {address} but got {data[2]}")
|
|
82
|
+
|
|
83
|
+
index = None
|
|
40
84
|
stride = data[1] + 2
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
85
|
+
nrows = len(data) // stride
|
|
86
|
+
payloadtype = data[4]
|
|
87
|
+
payloadoffset = 5
|
|
88
|
+
if payloadtype & 0x10 != 0:
|
|
89
|
+
seconds = np.ndarray(
|
|
90
|
+
nrows, dtype=np.uint32, buffer=data, offset=payloadoffset, strides=stride
|
|
91
|
+
)
|
|
92
|
+
payloadoffset += 4
|
|
93
|
+
micros = np.ndarray(
|
|
94
|
+
nrows, dtype=np.uint16, buffer=data, offset=payloadoffset, strides=stride
|
|
95
|
+
)
|
|
96
|
+
payloadoffset += 2
|
|
97
|
+
time = micros * _SECONDS_PER_TICK + seconds
|
|
98
|
+
payloadtype = payloadtype & ~0x10
|
|
99
|
+
if epoch is not None:
|
|
100
|
+
time = epoch + pd.to_timedelta(time, "s") # type: ignore
|
|
101
|
+
index = pd.Series(time)
|
|
102
|
+
index.name = "Time"
|
|
103
|
+
|
|
104
|
+
payloadsize = stride - payloadoffset - 1
|
|
105
|
+
payloadtype = _payloadtypes[payloadtype]
|
|
106
|
+
if dtype is not None and dtype != payloadtype:
|
|
107
|
+
raise ValueError(f"expected payload type {dtype} but got {payloadtype}")
|
|
108
|
+
|
|
44
109
|
elementsize = payloadtype.itemsize
|
|
45
|
-
payloadshape = (
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
110
|
+
payloadshape = (nrows, payloadsize // elementsize)
|
|
111
|
+
if length is not None and length != payloadshape[1]:
|
|
112
|
+
raise ValueError(f"expected payload length {length} but got {payloadshape[1]}")
|
|
113
|
+
|
|
49
114
|
payload = np.ndarray(
|
|
50
115
|
payloadshape,
|
|
51
116
|
dtype=payloadtype,
|
|
52
117
|
buffer=data,
|
|
53
|
-
offset=
|
|
118
|
+
offset=payloadoffset,
|
|
54
119
|
strides=(stride, elementsize),
|
|
55
120
|
)
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
121
|
+
|
|
122
|
+
result = pd.DataFrame(payload, index=index, columns=columns)
|
|
123
|
+
if keep_type:
|
|
124
|
+
msgtype = np.ndarray(
|
|
125
|
+
nrows, dtype=np.uint8, buffer=data, offset=0, strides=stride
|
|
126
|
+
)
|
|
127
|
+
msgtype = pd.Categorical.from_codes(msgtype, categories=_messagetypes) # type: ignore
|
|
128
|
+
result[MessageType.__name__] = msgtype
|
|
129
|
+
return result
|
harp/model.py
CHANGED
|
@@ -5,36 +5,44 @@
|
|
|
5
5
|
from __future__ import annotations
|
|
6
6
|
|
|
7
7
|
from enum import Enum
|
|
8
|
-
from typing import
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
8
|
+
from typing import Dict, List, Optional, Union
|
|
9
|
+
from typing_extensions import Annotated
|
|
10
|
+
|
|
11
|
+
from pydantic import (
|
|
12
|
+
BaseModel,
|
|
13
|
+
BeforeValidator,
|
|
14
|
+
ConfigDict,
|
|
15
|
+
Field,
|
|
16
|
+
RootModel,
|
|
17
|
+
field_serializer,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class PayloadType(str, Enum):
|
|
22
|
+
U8 = "uint8"
|
|
23
|
+
S8 = "int8"
|
|
24
|
+
U16 = "uint16"
|
|
25
|
+
S16 = "int16"
|
|
26
|
+
U32 = "uint32"
|
|
27
|
+
S32 = "int32"
|
|
28
|
+
U64 = "uint64"
|
|
29
|
+
S64 = "int64"
|
|
30
|
+
Float = "float32"
|
|
23
31
|
|
|
24
32
|
|
|
25
33
|
class Access(Enum):
|
|
26
|
-
Read =
|
|
27
|
-
Write =
|
|
28
|
-
Event =
|
|
34
|
+
Read = "Read"
|
|
35
|
+
Write = "Write"
|
|
36
|
+
Event = "Event"
|
|
29
37
|
|
|
30
38
|
|
|
31
39
|
class MaskValueItem(BaseModel):
|
|
32
40
|
model_config = ConfigDict(
|
|
33
|
-
extra=
|
|
41
|
+
extra="forbid",
|
|
34
42
|
)
|
|
35
|
-
value: int = Field(..., description=
|
|
43
|
+
value: int = Field(..., description="Specifies the numerical mask value.")
|
|
36
44
|
description: Optional[str] = Field(
|
|
37
|
-
None, description=
|
|
45
|
+
None, description="Specifies a summary description of the mask value function."
|
|
38
46
|
)
|
|
39
47
|
|
|
40
48
|
def __int__(self):
|
|
@@ -47,14 +55,14 @@ class MaskValue(RootModel[Union[int, MaskValueItem]]):
|
|
|
47
55
|
|
|
48
56
|
class BitMask(BaseModel):
|
|
49
57
|
description: Optional[str] = Field(
|
|
50
|
-
None, description=
|
|
58
|
+
None, description="Specifies a summary description of the bit mask function."
|
|
51
59
|
)
|
|
52
60
|
bits: Dict[str, MaskValue]
|
|
53
61
|
|
|
54
62
|
|
|
55
63
|
class GroupMask(BaseModel):
|
|
56
64
|
description: Optional[str] = Field(
|
|
57
|
-
None, description=
|
|
65
|
+
None, description="Specifies a summary description of the group mask function."
|
|
58
66
|
)
|
|
59
67
|
values: Dict[str, MaskValue]
|
|
60
68
|
|
|
@@ -62,55 +70,55 @@ class GroupMask(BaseModel):
|
|
|
62
70
|
class MaskType(RootModel[str]):
|
|
63
71
|
root: str = Field(
|
|
64
72
|
...,
|
|
65
|
-
description=
|
|
73
|
+
description="Specifies the name of the bit mask or group mask used to represent the payload value.",
|
|
66
74
|
)
|
|
67
75
|
|
|
68
76
|
|
|
69
77
|
class InterfaceType(RootModel[str]):
|
|
70
78
|
root: str = Field(
|
|
71
79
|
...,
|
|
72
|
-
description=
|
|
80
|
+
description="Specifies the name of the type used to represent the payload value in the high-level interface.",
|
|
73
81
|
)
|
|
74
82
|
|
|
75
83
|
|
|
76
84
|
class Converter(Enum):
|
|
77
|
-
None_ =
|
|
78
|
-
Payload =
|
|
79
|
-
RawPayload =
|
|
85
|
+
None_ = "None"
|
|
86
|
+
Payload = "Payload"
|
|
87
|
+
RawPayload = "RawPayload"
|
|
80
88
|
|
|
81
89
|
|
|
82
90
|
class MinValue(RootModel[float]):
|
|
83
91
|
root: float = Field(
|
|
84
92
|
...,
|
|
85
|
-
description=
|
|
93
|
+
description="Specifies the minimum allowable value for the payload or payload member.",
|
|
86
94
|
)
|
|
87
95
|
|
|
88
96
|
|
|
89
97
|
class MaxValue(RootModel[float]):
|
|
90
98
|
root: float = Field(
|
|
91
99
|
...,
|
|
92
|
-
description=
|
|
100
|
+
description="Specifies the maximum allowable value for the payload or payload member.",
|
|
93
101
|
)
|
|
94
102
|
|
|
95
103
|
|
|
96
104
|
class DefaultValue(RootModel[float]):
|
|
97
105
|
root: float = Field(
|
|
98
106
|
...,
|
|
99
|
-
description=
|
|
107
|
+
description="Specifies the default value for the payload or payload member.",
|
|
100
108
|
)
|
|
101
109
|
|
|
102
110
|
|
|
103
111
|
class PayloadMember(BaseModel):
|
|
104
112
|
mask: Optional[int] = Field(
|
|
105
113
|
None,
|
|
106
|
-
description=
|
|
114
|
+
description="Specifies the mask used to read and write this payload member.",
|
|
107
115
|
)
|
|
108
116
|
offset: Optional[int] = Field(
|
|
109
117
|
None,
|
|
110
|
-
description=
|
|
118
|
+
description="Specifies the payload array offset where this payload member is stored.",
|
|
111
119
|
)
|
|
112
120
|
description: Optional[str] = Field(
|
|
113
|
-
None, description=
|
|
121
|
+
None, description="Specifies a summary description of the payload member."
|
|
114
122
|
)
|
|
115
123
|
minValue: Optional[MinValue] = None
|
|
116
124
|
maxValue: Optional[MaxValue] = None
|
|
@@ -121,23 +129,29 @@ class PayloadMember(BaseModel):
|
|
|
121
129
|
|
|
122
130
|
|
|
123
131
|
class Visibility(Enum):
|
|
124
|
-
public =
|
|
125
|
-
private =
|
|
132
|
+
public = "public"
|
|
133
|
+
private = "private"
|
|
126
134
|
|
|
127
135
|
|
|
128
136
|
class Register(BaseModel):
|
|
129
|
-
address:
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
)
|
|
137
|
+
address: Annotated[
|
|
138
|
+
int,
|
|
139
|
+
Field(
|
|
140
|
+
le=255, description="Specifies the unique 8-bit address of the register."
|
|
141
|
+
),
|
|
142
|
+
]
|
|
143
|
+
type: Annotated[PayloadType, BeforeValidator(lambda v: PayloadType[v])]
|
|
144
|
+
length: Annotated[
|
|
145
|
+
Optional[int],
|
|
146
|
+
Field(
|
|
147
|
+
ge=1, default=1, description="Specifies the length of the register payload."
|
|
148
|
+
),
|
|
149
|
+
]
|
|
136
150
|
access: Union[Access, List[Access]] = Field(
|
|
137
|
-
..., description=
|
|
151
|
+
..., description="Specifies the expected use of the register."
|
|
138
152
|
)
|
|
139
153
|
description: Optional[str] = Field(
|
|
140
|
-
None, description=
|
|
154
|
+
None, description="Specifies a summary description of the register function."
|
|
141
155
|
)
|
|
142
156
|
minValue: Optional[MinValue] = None
|
|
143
157
|
maxValue: Optional[MaxValue] = None
|
|
@@ -145,40 +159,44 @@ class Register(BaseModel):
|
|
|
145
159
|
maskType: Optional[MaskType] = None
|
|
146
160
|
visibility: Optional[Visibility] = Field(
|
|
147
161
|
None,
|
|
148
|
-
description=
|
|
162
|
+
description="Specifies whether the register function is exposed in the high-level interface.",
|
|
149
163
|
)
|
|
150
164
|
volatile: Optional[bool] = Field(
|
|
151
165
|
None,
|
|
152
|
-
description=
|
|
166
|
+
description="Specifies whether register values can be saved in non-volatile memory.",
|
|
153
167
|
)
|
|
154
168
|
payloadSpec: Optional[Dict[str, PayloadMember]] = None
|
|
155
169
|
interfaceType: Optional[InterfaceType] = None
|
|
156
170
|
converter: Optional[Converter] = None
|
|
157
171
|
|
|
172
|
+
@field_serializer("type")
|
|
173
|
+
def _serialize_type(self, type: PayloadType):
|
|
174
|
+
return type.name
|
|
175
|
+
|
|
158
176
|
|
|
159
177
|
class Registers(BaseModel):
|
|
160
178
|
registers: Dict[str, Register] = Field(
|
|
161
179
|
...,
|
|
162
|
-
description=
|
|
180
|
+
description="Specifies the collection of registers implementing the device function.",
|
|
163
181
|
)
|
|
164
182
|
bitMasks: Optional[Dict[str, BitMask]] = Field(
|
|
165
183
|
None,
|
|
166
|
-
description=
|
|
184
|
+
description="Specifies the collection of masks available to be used with the different registers.",
|
|
167
185
|
)
|
|
168
186
|
groupMasks: Optional[Dict[str, GroupMask]] = Field(
|
|
169
187
|
None,
|
|
170
|
-
description=
|
|
188
|
+
description="Specifies the collection of group masks available to be used with the different registers.",
|
|
171
189
|
)
|
|
172
190
|
|
|
173
191
|
|
|
174
192
|
class Model(Registers):
|
|
175
|
-
device: str = Field(..., description=
|
|
193
|
+
device: str = Field(..., description="Specifies the name of the device.")
|
|
176
194
|
whoAmI: int = Field(
|
|
177
|
-
..., description=
|
|
195
|
+
..., description="Specifies the unique identifier for this device type."
|
|
178
196
|
)
|
|
179
197
|
firmwareVersion: str = Field(
|
|
180
|
-
..., description=
|
|
198
|
+
..., description="Specifies the semantic version of the device firmware."
|
|
181
199
|
)
|
|
182
200
|
hardwareTargets: str = Field(
|
|
183
|
-
..., description=
|
|
201
|
+
..., description="Specifies the semantic version of the device hardware."
|
|
184
202
|
)
|
harp/reader.py
CHANGED
|
@@ -1,19 +1,45 @@
|
|
|
1
|
-
import
|
|
1
|
+
import os
|
|
2
2
|
from math import log2
|
|
3
|
+
from os import PathLike
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from datetime import datetime
|
|
3
6
|
from functools import partial
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from numpy import dtype
|
|
4
9
|
from pandas import DataFrame, Series
|
|
5
|
-
from typing import
|
|
6
|
-
from
|
|
7
|
-
from harp.
|
|
10
|
+
from typing import Any, BinaryIO, Callable, Iterable, Optional, Protocol, Union
|
|
11
|
+
from pandas._typing import Axes
|
|
12
|
+
from harp.model import BitMask, GroupMask, Model, PayloadMember, Register
|
|
13
|
+
from harp.io import MessageType, read
|
|
14
|
+
from harp.schema import read_schema
|
|
8
15
|
|
|
9
|
-
|
|
16
|
+
|
|
17
|
+
@dataclass
|
|
18
|
+
class _ReaderParams:
|
|
19
|
+
path: Path
|
|
20
|
+
epoch: Optional[datetime] = None
|
|
21
|
+
keep_type: bool = False
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class _ReadRegister(Protocol):
|
|
25
|
+
def __call__(
|
|
26
|
+
self,
|
|
27
|
+
file: Optional[Union[str, bytes, PathLike[Any], BinaryIO]] = None,
|
|
28
|
+
epoch: Optional[datetime] = None,
|
|
29
|
+
keep_type: bool = False,
|
|
30
|
+
) -> DataFrame:
|
|
31
|
+
...
|
|
10
32
|
|
|
11
33
|
|
|
12
34
|
class RegisterReader:
|
|
13
35
|
register: Register
|
|
14
|
-
read:
|
|
36
|
+
read: _ReadRegister
|
|
15
37
|
|
|
16
|
-
def __init__(
|
|
38
|
+
def __init__(
|
|
39
|
+
self,
|
|
40
|
+
register: Register,
|
|
41
|
+
read: _ReadRegister,
|
|
42
|
+
) -> None:
|
|
17
43
|
self.register = register
|
|
18
44
|
self.read = read
|
|
19
45
|
|
|
@@ -33,12 +59,25 @@ class DeviceReader:
|
|
|
33
59
|
return self.registers[__name]
|
|
34
60
|
|
|
35
61
|
|
|
36
|
-
def
|
|
37
|
-
|
|
62
|
+
def _compose_parser(
|
|
63
|
+
f: Callable[[DataFrame], DataFrame],
|
|
64
|
+
g: Callable[..., DataFrame],
|
|
65
|
+
params: _ReaderParams,
|
|
66
|
+
) -> Callable[..., DataFrame]:
|
|
67
|
+
def parser(
|
|
68
|
+
file: Optional[Union[str, bytes, PathLike[Any], BinaryIO]] = None,
|
|
69
|
+
columns: Optional[Axes] = None,
|
|
70
|
+
epoch: Optional[datetime] = params.epoch,
|
|
71
|
+
keep_type: bool = params.keep_type,
|
|
72
|
+
):
|
|
73
|
+
df = g(file, columns, epoch, keep_type)
|
|
74
|
+
result = f(df)
|
|
75
|
+
type_col = df.get(MessageType.__name__)
|
|
76
|
+
if type_col is not None:
|
|
77
|
+
result[MessageType.__name__] = type_col
|
|
78
|
+
return result
|
|
38
79
|
|
|
39
|
-
|
|
40
|
-
def _id_camel_to_snake(id: str):
|
|
41
|
-
return _camel_to_snake_regex.sub("_", id).lower()
|
|
80
|
+
return parser
|
|
42
81
|
|
|
43
82
|
|
|
44
83
|
def _create_bit_parser(mask: int):
|
|
@@ -49,10 +88,7 @@ def _create_bit_parser(mask: int):
|
|
|
49
88
|
|
|
50
89
|
|
|
51
90
|
def _create_bitmask_parser(bitMask: BitMask):
|
|
52
|
-
lookup = [
|
|
53
|
-
(_id_camel_to_snake(k), _create_bit_parser(int(v.root)))
|
|
54
|
-
for k, v in bitMask.bits.items()
|
|
55
|
-
]
|
|
91
|
+
lookup = [(k, _create_bit_parser(int(v.root))) for k, v in bitMask.bits.items()]
|
|
56
92
|
|
|
57
93
|
def parser(df: DataFrame):
|
|
58
94
|
return DataFrame({n: f(df[0]) for n, f in lookup}, index=df.index)
|
|
@@ -83,14 +119,14 @@ def _create_payloadmember_parser(device: Model, member: PayloadMember):
|
|
|
83
119
|
if offset is None:
|
|
84
120
|
offset = 0
|
|
85
121
|
|
|
86
|
-
shift =
|
|
122
|
+
shift = 0
|
|
87
123
|
if member.mask is not None:
|
|
88
124
|
shift = _mask_shift(member.mask)
|
|
89
125
|
|
|
90
126
|
lookup = None
|
|
91
127
|
if member.maskType is not None:
|
|
92
128
|
key = member.maskType.root
|
|
93
|
-
groupMask = device.groupMasks.get(key)
|
|
129
|
+
groupMask = None if device.groupMasks is None else device.groupMasks.get(key)
|
|
94
130
|
if groupMask is not None:
|
|
95
131
|
lookup = _create_groupmask_lookup(groupMask)
|
|
96
132
|
|
|
@@ -103,7 +139,7 @@ def _create_payloadmember_parser(device: Model, member: PayloadMember):
|
|
|
103
139
|
if member.mask is not None:
|
|
104
140
|
series = series & member.mask
|
|
105
141
|
if shift > 0:
|
|
106
|
-
series = Series(series.values >> shift, series.index)
|
|
142
|
+
series = Series(series.values >> shift, series.index) # type: ignore
|
|
107
143
|
if is_boolean:
|
|
108
144
|
series = series != 0
|
|
109
145
|
elif lookup is not None:
|
|
@@ -113,43 +149,109 @@ def _create_payloadmember_parser(device: Model, member: PayloadMember):
|
|
|
113
149
|
return parser
|
|
114
150
|
|
|
115
151
|
|
|
116
|
-
def _create_register_reader(
|
|
152
|
+
def _create_register_reader(register: Register, params: _ReaderParams):
|
|
153
|
+
def reader(
|
|
154
|
+
file: Optional[Union[str, bytes, PathLike[Any], BinaryIO]] = None,
|
|
155
|
+
columns: Optional[Axes] = None,
|
|
156
|
+
epoch: Optional[datetime] = params.epoch,
|
|
157
|
+
keep_type: bool = params.keep_type,
|
|
158
|
+
):
|
|
159
|
+
if file is None:
|
|
160
|
+
file = f"{params.path}_{register.address}.bin"
|
|
161
|
+
|
|
162
|
+
data = read(
|
|
163
|
+
file,
|
|
164
|
+
address=register.address,
|
|
165
|
+
dtype=dtype(register.type),
|
|
166
|
+
length=register.length,
|
|
167
|
+
columns=columns,
|
|
168
|
+
epoch=epoch,
|
|
169
|
+
keep_type=keep_type,
|
|
170
|
+
)
|
|
171
|
+
return data
|
|
172
|
+
|
|
173
|
+
return reader
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _create_register_parser(device: Model, name: str, params: _ReaderParams):
|
|
117
177
|
register = device.registers[name]
|
|
118
|
-
reader =
|
|
178
|
+
reader = _create_register_reader(register, params)
|
|
119
179
|
|
|
120
180
|
if register.maskType is not None:
|
|
121
181
|
key = register.maskType.root
|
|
122
|
-
bitMask = device.bitMasks.get(key)
|
|
182
|
+
bitMask = None if device.bitMasks is None else device.bitMasks.get(key)
|
|
123
183
|
if bitMask is not None:
|
|
124
184
|
parser = _create_bitmask_parser(bitMask)
|
|
125
|
-
reader =
|
|
185
|
+
reader = _compose_parser(parser, reader, params)
|
|
126
186
|
return RegisterReader(register, reader)
|
|
127
187
|
|
|
128
|
-
groupMask = device.groupMasks.get(key)
|
|
188
|
+
groupMask = None if device.groupMasks is None else device.groupMasks.get(key)
|
|
129
189
|
if groupMask is not None:
|
|
130
190
|
parser = _create_groupmask_parser(name, groupMask)
|
|
131
|
-
reader =
|
|
191
|
+
reader = _compose_parser(parser, reader, params)
|
|
132
192
|
return RegisterReader(register, reader)
|
|
133
193
|
|
|
134
194
|
if register.payloadSpec is not None:
|
|
135
195
|
payload_parsers = [
|
|
136
|
-
(
|
|
196
|
+
(key, _create_payloadmember_parser(device, member))
|
|
137
197
|
for key, member in register.payloadSpec.items()
|
|
138
198
|
]
|
|
139
199
|
|
|
140
200
|
def parser(df: DataFrame):
|
|
141
201
|
return DataFrame({n: f(df) for n, f in payload_parsers}, index=df.index)
|
|
142
202
|
|
|
143
|
-
reader =
|
|
203
|
+
reader = _compose_parser(parser, reader, params)
|
|
144
204
|
return RegisterReader(register, reader)
|
|
145
205
|
|
|
146
|
-
|
|
147
|
-
reader = partial(reader, columns=columns)
|
|
206
|
+
reader = partial(reader, columns=[name])
|
|
148
207
|
return RegisterReader(register, reader)
|
|
149
208
|
|
|
150
209
|
|
|
151
|
-
def create_reader(
|
|
210
|
+
def create_reader(
|
|
211
|
+
device: Union[str, PathLike, Model],
|
|
212
|
+
include_common_registers: bool = True,
|
|
213
|
+
epoch: Optional[datetime] = None,
|
|
214
|
+
keep_type: bool = False,
|
|
215
|
+
):
|
|
216
|
+
"""Creates a device reader object from the specified dataset or schema.
|
|
217
|
+
|
|
218
|
+
Parameters
|
|
219
|
+
----------
|
|
220
|
+
device
|
|
221
|
+
A path to the device schema, dataset folder, or parsed device schema object
|
|
222
|
+
describing the device.
|
|
223
|
+
include_common_registers
|
|
224
|
+
Specifies whether to include the set of Harp common registers in the
|
|
225
|
+
parsed device schema object. If a parsed device schema object is provided,
|
|
226
|
+
this parameter is ignored.
|
|
227
|
+
epoch
|
|
228
|
+
The default reference datetime at which time zero begins. If specified,
|
|
229
|
+
the data frames returned by each register reader will have a datetime index.
|
|
230
|
+
keep_type
|
|
231
|
+
Specifies whether to include a column with the message type by default.
|
|
232
|
+
|
|
233
|
+
Returns
|
|
234
|
+
-------
|
|
235
|
+
A device reader object which can be used to read binary data for each
|
|
236
|
+
register or to access metadata about each register. Individual registers
|
|
237
|
+
can be accessed using dot notation using the name of the register as the
|
|
238
|
+
key.
|
|
239
|
+
"""
|
|
240
|
+
if isinstance(device, Model):
|
|
241
|
+
base_path = Path(device.device)
|
|
242
|
+
else:
|
|
243
|
+
path = Path(device).absolute().resolve()
|
|
244
|
+
is_dir = os.path.isdir(path)
|
|
245
|
+
if is_dir:
|
|
246
|
+
device = path / "device.yml"
|
|
247
|
+
|
|
248
|
+
device = read_schema(device, include_common_registers)
|
|
249
|
+
base_path = path / device.device if is_dir else path.parent / device.device
|
|
250
|
+
|
|
152
251
|
reg_readers = {
|
|
153
|
-
name:
|
|
252
|
+
name: _create_register_parser(
|
|
253
|
+
device, name, _ReaderParams(base_path, epoch, keep_type)
|
|
254
|
+
)
|
|
255
|
+
for name in device.registers.keys()
|
|
154
256
|
}
|
|
155
257
|
return DeviceReader(device, reg_readers)
|
harp/schema.py
CHANGED
|
@@ -1,31 +1,53 @@
|
|
|
1
1
|
from os import PathLike
|
|
2
|
-
from pathlib import Path
|
|
3
2
|
from typing import TextIO, Union
|
|
4
3
|
from harp.model import Model, Registers
|
|
5
4
|
from pydantic_yaml import parse_yaml_raw_as
|
|
5
|
+
from importlib import resources
|
|
6
6
|
|
|
7
|
-
_common_yaml_path = Path(__file__).absolute().parent.joinpath("common.yml")
|
|
8
7
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
return _read_common_registers(fileIO)
|
|
14
|
-
except TypeError:
|
|
15
|
-
return parse_yaml_raw_as(Registers, file.read())
|
|
8
|
+
def _read_common_registers() -> Registers:
|
|
9
|
+
file = resources.files(__package__) / "common.yml"
|
|
10
|
+
with file.open("rt") as fileIO:
|
|
11
|
+
return parse_yaml_raw_as(Registers, fileIO.read())
|
|
16
12
|
|
|
17
13
|
|
|
18
14
|
def read_schema(
|
|
19
15
|
file: Union[str, PathLike, TextIO], include_common_registers: bool = True
|
|
20
16
|
) -> Model:
|
|
21
|
-
|
|
17
|
+
"""Read and parse a device schema from the specified file.
|
|
18
|
+
|
|
19
|
+
Parameters
|
|
20
|
+
----------
|
|
21
|
+
file
|
|
22
|
+
Open file object or filename containing a YAML text stream describing
|
|
23
|
+
a device schema.
|
|
24
|
+
include_common_registers
|
|
25
|
+
Specifies whether to include the set of Harp common registers in the
|
|
26
|
+
returned device schema object.
|
|
27
|
+
|
|
28
|
+
Returns
|
|
29
|
+
-------
|
|
30
|
+
A Pydantic model object representing the Harp device schema.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
if isinstance(file, (str, PathLike)):
|
|
22
34
|
with open(file) as fileIO:
|
|
23
|
-
return read_schema(fileIO
|
|
24
|
-
|
|
35
|
+
return read_schema(fileIO)
|
|
36
|
+
else:
|
|
25
37
|
schema = parse_yaml_raw_as(Model, file.read())
|
|
26
38
|
if not "WhoAmI" in schema.registers and include_common_registers:
|
|
27
|
-
common = _read_common_registers(
|
|
39
|
+
common = _read_common_registers()
|
|
28
40
|
schema.registers = dict(common.registers, **schema.registers)
|
|
29
|
-
|
|
30
|
-
|
|
41
|
+
if common.bitMasks:
|
|
42
|
+
schema.bitMasks = (
|
|
43
|
+
common.bitMasks
|
|
44
|
+
if schema.bitMasks is None
|
|
45
|
+
else dict(common.bitMasks, **schema.bitMasks)
|
|
46
|
+
)
|
|
47
|
+
if common.groupMasks:
|
|
48
|
+
schema.groupMasks = (
|
|
49
|
+
common.groupMasks
|
|
50
|
+
if schema.groupMasks is None
|
|
51
|
+
else dict(common.groupMasks, **schema.groupMasks)
|
|
52
|
+
)
|
|
31
53
|
return schema
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: harp-python
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: A low-level interface for loading binary Harp protocol data
|
|
5
5
|
Author-email: Goncalo Lopes <g.lopes@neurogears.org>
|
|
6
6
|
License: MIT License
|
|
@@ -20,6 +20,7 @@ Requires-Dist: pydantic-yaml
|
|
|
20
20
|
Requires-Dist: pandas
|
|
21
21
|
Provides-Extra: dev
|
|
22
22
|
Requires-Dist: datamodel-code-generator ; extra == 'dev'
|
|
23
|
+
Requires-Dist: pytest ; extra == 'dev'
|
|
23
24
|
Requires-Dist: black ; extra == 'dev'
|
|
24
25
|
Provides-Extra: jupyter
|
|
25
26
|
Requires-Dist: ipykernel ; extra == 'jupyter'
|
|
@@ -31,35 +32,33 @@ A low-level interface to data collected with the [Harp](https://harp-tech.org/)
|
|
|
31
32
|
|
|
32
33
|
## Data model
|
|
33
34
|
|
|
34
|
-
|
|
35
|
+
The interface makes use of a Pydantic data model generated from Harp device schema definitions. The schema data classes are used to automatically generate binary readers for each device.
|
|
36
|
+
|
|
37
|
+
All binary data files from a single device need to be stored in the same folder alongside the device meta-schema, named `device.yml`. Each register file should have the following naming convention `<deviceName>_<registerAddress>.bin`.
|
|
38
|
+
|
|
39
|
+
For example, for a dataset collected with a `Behavior` device, you might have:
|
|
35
40
|
|
|
36
41
|
```
|
|
37
|
-
|
|
42
|
+
📦device.harp
|
|
43
|
+
┣ 📜Behavior_0.bin
|
|
44
|
+
┣ 📜Behavior_1.bin
|
|
45
|
+
...
|
|
46
|
+
┗ 📜device.yml
|
|
38
47
|
```
|
|
39
48
|
|
|
40
|
-
> [!IMPORTANT]
|
|
41
|
-
> Currently code generation adds an unwanted field at the very end of the data model definition `registers: Optional[Any] = None`. This declaration needs to be removed for serialization to work properly.
|
|
42
|
-
|
|
43
49
|
## How to use
|
|
44
50
|
|
|
45
|
-
### Read Harp device schema from YML file
|
|
46
|
-
|
|
47
|
-
```python
|
|
48
|
-
from harp.schema import read_schema
|
|
49
|
-
schema = read_schema('device.yml')
|
|
50
|
-
```
|
|
51
|
-
|
|
52
51
|
### Create device reader object from schema
|
|
53
52
|
|
|
54
53
|
```python
|
|
55
|
-
|
|
56
|
-
reader = create_reader(
|
|
54
|
+
import harp
|
|
55
|
+
reader = harp.create_reader("device.harp")
|
|
57
56
|
```
|
|
58
57
|
|
|
59
58
|
### Read data from named register
|
|
60
59
|
|
|
61
60
|
```python
|
|
62
|
-
reader.OperationControl.read(
|
|
61
|
+
reader.OperationControl.read()
|
|
63
62
|
```
|
|
64
63
|
|
|
65
64
|
### Access register metadata
|
|
@@ -67,3 +66,21 @@ reader.OperationControl.read("data/Behavior_10.bin")
|
|
|
67
66
|
```python
|
|
68
67
|
reader.OperationControl.register.address
|
|
69
68
|
```
|
|
69
|
+
|
|
70
|
+
### Create device reader object with UTC datetime format
|
|
71
|
+
|
|
72
|
+
```python
|
|
73
|
+
reader = harp.create_reader("device.harp", epoch=harp.REFERENCE_EPOCH)
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
### Read data with message type information
|
|
77
|
+
|
|
78
|
+
```python
|
|
79
|
+
reader.OperationControl.read(keep_type=True)
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
### Read data from a specific file
|
|
83
|
+
|
|
84
|
+
```python
|
|
85
|
+
reader.OperationControl.read("data/Behavior_10.bin")
|
|
86
|
+
```
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
harp/__init__.py,sha256=7_D-5szHZTqSdy1lBiFIkg7_WkEjdn8vzMacxpDkkCE,132
|
|
2
|
+
harp/common.yml,sha256=qcejMFu00JjI5rkubuEDdrUu_uFkN17f_Ljb6yBd-RM,4452
|
|
3
|
+
harp/io.py,sha256=7Mdoiz7QCoNYV7R-xTR-zFYDhVY-9-gpuDckbmBPfPs,4258
|
|
4
|
+
harp/model.py,sha256=8apjJu4no1yZ_s0h8ZvCF__z5qAc4qg3koHEBukg84Q,5931
|
|
5
|
+
harp/reader.py,sha256=_8J5niscJ5AenZtrcYWvmmoWbxQAfO78E3Lk30azsjc,8060
|
|
6
|
+
harp/schema.py,sha256=3Y8tuwXqO5LkJroVKU1TV5TznpCAqmR0G2Cod7NQoHQ,1867
|
|
7
|
+
harp_python-0.2.0.dist-info/LICENSE,sha256=KxnuV3gNCp4WmLUC8DYWJvCKPdxMpwmWIzAYBYdqPrM,1104
|
|
8
|
+
harp_python-0.2.0.dist-info/METADATA,sha256=atBivmIIVWmjjN_C11YnjZV6W7JoW9MmAnjvZuXe5MM,2547
|
|
9
|
+
harp_python-0.2.0.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
|
|
10
|
+
harp_python-0.2.0.dist-info/top_level.txt,sha256=ztK1iHjgwmfXqxHCpu-e2uHIwdmodywYEUaWSODMvyM,5
|
|
11
|
+
harp_python-0.2.0.dist-info/RECORD,,
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
harp/common.yml,sha256=qcejMFu00JjI5rkubuEDdrUu_uFkN17f_Ljb6yBd-RM,4452
|
|
2
|
-
harp/io.py,sha256=qqdkoZe4Tqg6SEF8ZQmgN0iOGDbdBkgsXF6HaoUhfmQ,1907
|
|
3
|
-
harp/model.py,sha256=20joms_-BLswhwPE4-AFpnGY8mh_qG6vvM8BdzOPhaU,5545
|
|
4
|
-
harp/reader.py,sha256=8jAKcphayOWINvJF-uXcqC3QH7s4g_VjlKfNd_SrG2E,4515
|
|
5
|
-
harp/schema.py,sha256=Xz5dA4dr13kr1vayB6debA5T2rNbFPfG7ssS3RPVFok,1202
|
|
6
|
-
harp_python-0.1.0.dist-info/LICENSE,sha256=KxnuV3gNCp4WmLUC8DYWJvCKPdxMpwmWIzAYBYdqPrM,1104
|
|
7
|
-
harp_python-0.1.0.dist-info/METADATA,sha256=K_U2ZcsXcxaInc_0JO7gXwQiKnaGpVuca-ct5HwIyj0,2193
|
|
8
|
-
harp_python-0.1.0.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
|
|
9
|
-
harp_python-0.1.0.dist-info/top_level.txt,sha256=ztK1iHjgwmfXqxHCpu-e2uHIwdmodywYEUaWSODMvyM,5
|
|
10
|
-
harp_python-0.1.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|