harp-python 0.1.0__tar.gz → 0.2.0__tar.gz

{harp-python-0.1.0 → harp-python-0.2.0}/.gitignore

@@ -9,4 +9,4 @@ __pycache__
 dist
 
 # Data files
-data
+/data

{harp-python-0.1.0 → harp-python-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: harp-python
-Version: 0.1.0
+Version: 0.2.0
 Summary: A low-level interface for loading binary Harp protocol data
 Author-email: Goncalo Lopes <g.lopes@neurogears.org>
 License: MIT License
@@ -20,6 +20,7 @@ Requires-Dist: pydantic-yaml
 Requires-Dist: pandas
 Provides-Extra: dev
 Requires-Dist: datamodel-code-generator; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
 Requires-Dist: black; extra == "dev"
 Provides-Extra: jupyter
 Requires-Dist: ipykernel; extra == "jupyter"
@@ -31,35 +32,33 @@ A low-level interface to data collected with the [Harp](https://harp-tech.org/)
 
 ## Data model 
 
-To regenerate Pydantic data models from device schema definitions, activate a virtual environment with `dev` dependencies, and run:
+The interface makes use of a Pydantic data model generated from Harp device schema definitions. The schema data classes are used to automatically generate binary readers for each device.
+
+All binary data files from a single device need to be stored in the same folder alongside the device meta-schema, named `device.yml`. Each register file should have the following naming convention `<deviceName>_<registerAddress>.bin`.
+
+For example, for a dataset collected with a `Behavior` device, you might have:
 
 ```
-datamodel-codegen --input ./reflex-generator/schema/device.json --output harp/model.py --output-model-type pydantic_v2.BaseModel
+📦device.harp
+ ┣ 📜Behavior_0.bin
+ ┣ 📜Behavior_1.bin
+...
+ ┗ 📜device.yml
 ```
 
-> [!IMPORTANT]
-> Currently code generation adds an unwanted field at the very end of the data model definition `registers: Optional[Any] = None`. This declaration needs to be removed for serialization to work properly.
-
 ## How to use
 
-### Read Harp device schema from YML file
-
-```python
-from harp.schema import read_schema
-schema = read_schema('device.yml')
-```
-
 ### Create device reader object from schema
 
 ```python
-from harp.reader import create_reader
-reader = create_reader(schema)
+import harp
+reader = harp.create_reader("device.harp")
 ```
 
 ### Read data from named register
 
 ```python
-reader.OperationControl.read("data/Behavior_10.bin")
+reader.OperationControl.read()
 ```
 
 ### Access register metadata
@@ -67,3 +66,21 @@ reader.OperationControl.read("data/Behavior_10.bin")
 ```python
 reader.OperationControl.register.address
 ```
+
+### Create device reader object with UTC datetime format
+
+```python
+reader = harp.create_reader("device.harp", epoch=harp.REFERENCE_EPOCH)
+```
+
+### Read data with message type information
+
+```python
+reader.OperationControl.read(keep_type=True)
+```
+
+### Read data from a specific file
+
+```python
+reader.OperationControl.read("data/Behavior_10.bin")
+```

harp-python-0.2.0/README.md

@@ -0,0 +1,58 @@
+# harp-python
+
+A low-level interface to data collected with the [Harp](https://harp-tech.org/) binary protocol.
+
+## Data model 
+
+The interface makes use of a Pydantic data model generated from Harp device schema definitions. The schema data classes are used to automatically generate binary readers for each device.
+
+All binary data files from a single device need to be stored in the same folder alongside the device meta-schema, named `device.yml`. Each register file should have the following naming convention `<deviceName>_<registerAddress>.bin`.
+
+For example, for a dataset collected with a `Behavior` device, you might have:
+
+```
+📦device.harp
+ ┣ 📜Behavior_0.bin
+ ┣ 📜Behavior_1.bin
+...
+ ┗ 📜device.yml
+```
+
+## How to use
+
+### Create device reader object from schema
+
+```python
+import harp
+reader = harp.create_reader("device.harp")
+```
+
+### Read data from named register
+
+```python
+reader.OperationControl.read()
+```
+
+### Access register metadata
+
+```python
+reader.OperationControl.register.address
+```
+
+### Create device reader object with UTC datetime format
+
+```python
+reader = harp.create_reader("device.harp", epoch=harp.REFERENCE_EPOCH)
+```
+
+### Read data with message type information
+
+```python
+reader.OperationControl.read(keep_type=True)
+```
+
+### Read data from a specific file
+
+```python
+reader.OperationControl.read("data/Behavior_10.bin")
+```

harp-python-0.2.0/harp/__init__.py

@@ -0,0 +1,3 @@
+from harp.io import REFERENCE_EPOCH, MessageType, read
+from harp.reader import create_reader
+from harp.schema import read_schema

harp-python-0.2.0/harp/io.py

@@ -0,0 +1,129 @@
+from datetime import datetime
+from enum import IntEnum
+from os import PathLike
+from typing import Any, BinaryIO, Optional, Union
+from pandas._typing import Axes
+import numpy as np
+import pandas as pd
+
+REFERENCE_EPOCH = datetime(1904, 1, 1)
+"""The reference epoch for UTC harp time."""
+
+
+class MessageType(IntEnum):
+    """Specifies the type of a Harp message."""
+
+    NA = 0
+    READ = 1
+    WRITE = 2
+    EVENT = 3
+
+
+_SECONDS_PER_TICK = 32e-6
+_messagetypes = [type.name for type in MessageType]
+_payloadtypes = {
+    1: np.dtype(np.uint8),
+    2: np.dtype(np.uint16),
+    4: np.dtype(np.uint32),
+    8: np.dtype(np.uint64),
+    129: np.dtype(np.int8),
+    130: np.dtype(np.int16),
+    132: np.dtype(np.int32),
+    136: np.dtype(np.int64),
+    68: np.dtype(np.float32),
+}
+
+
+def read(
+    file: Union[str, bytes, PathLike[Any], BinaryIO],
+    address: Optional[int] = None,
+    dtype: Optional[np.dtype] = None,
+    length: Optional[int] = None,
+    columns: Optional[Axes] = None,
+    epoch: Optional[datetime] = None,
+    keep_type: bool = False,
+):
+    """Read single-register Harp data from the specified file.
+
+    Parameters
+    ----------
+    file
+        Open file object or filename containing binary data from
+        a single device register.
+    address
+        Expected register address. If specified, the address of
+        the first message in the file is used for validation.
+    dtype
+        Expected data type of the register payload. If specified, the
+        payload type of the first message in the file is used for validation.
+    length
+        Expected number of elements in register payload. If specified, the
+        payload length of the first message in the file is used for validation.
+    columns
+        The optional column labels to use for the data values.
+    epoch
+        Reference datetime at which time zero begins. If specified,
+        the result data frame will have a datetime index.
+    keep_type
+        Specifies whether to include a column with the message type.
+
+    Returns
+    -------
+        A pandas data frame containing message data, sorted by time.
+    """
+    data = np.fromfile(file, dtype=np.uint8)
+    if len(data) == 0:
+        return pd.DataFrame(
+            columns=columns, index=pd.Index([], dtype=np.float64, name="Time")
+        )
+
+    if address is not None and address != data[2]:
+        raise ValueError(f"expected address {address} but got {data[2]}")
+
+    index = None
+    stride = data[1] + 2
+    nrows = len(data) // stride
+    payloadtype = data[4]
+    payloadoffset = 5
+    if payloadtype & 0x10 != 0:
+        seconds = np.ndarray(
+            nrows, dtype=np.uint32, buffer=data, offset=payloadoffset, strides=stride
+        )
+        payloadoffset += 4
+        micros = np.ndarray(
+            nrows, dtype=np.uint16, buffer=data, offset=payloadoffset, strides=stride
+        )
+        payloadoffset += 2
+        time = micros * _SECONDS_PER_TICK + seconds
+        payloadtype = payloadtype & ~0x10
+        if epoch is not None:
+            time = epoch + pd.to_timedelta(time, "s")  # type: ignore
+        index = pd.Series(time)
+        index.name = "Time"
+
+    payloadsize = stride - payloadoffset - 1
+    payloadtype = _payloadtypes[payloadtype]
+    if dtype is not None and dtype != payloadtype:
+        raise ValueError(f"expected payload type {dtype} but got {payloadtype}")
+
+    elementsize = payloadtype.itemsize
+    payloadshape = (nrows, payloadsize // elementsize)
+    if length is not None and length != payloadshape[1]:
+        raise ValueError(f"expected payload length {length} but got {payloadshape[1]}")
+
+    payload = np.ndarray(
+        payloadshape,
+        dtype=payloadtype,
+        buffer=data,
+        offset=payloadoffset,
+        strides=(stride, elementsize),
+    )
+
+    result = pd.DataFrame(payload, index=index, columns=columns)
+    if keep_type:
+        msgtype = np.ndarray(
+            nrows, dtype=np.uint8, buffer=data, offset=0, strides=stride
+        )
+        msgtype = pd.Categorical.from_codes(msgtype, categories=_messagetypes)  # type: ignore
+        result[MessageType.__name__] = msgtype
+    return result

harp-python-0.2.0/harp/model.py

@@ -0,0 +1,202 @@
+# generated by datamodel-codegen:
+#   filename:  device.json
+#   timestamp: 2023-10-30T11:46:57+00:00
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Dict, List, Optional, Union
+from typing_extensions import Annotated
+
+from pydantic import (
+    BaseModel,
+    BeforeValidator,
+    ConfigDict,
+    Field,
+    RootModel,
+    field_serializer,
+)
+
+
+class PayloadType(str, Enum):
+    U8 = "uint8"
+    S8 = "int8"
+    U16 = "uint16"
+    S16 = "int16"
+    U32 = "uint32"
+    S32 = "int32"
+    U64 = "uint64"
+    S64 = "int64"
+    Float = "float32"
+
+
+class Access(Enum):
+    Read = "Read"
+    Write = "Write"
+    Event = "Event"
+
+
+class MaskValueItem(BaseModel):
+    model_config = ConfigDict(
+        extra="forbid",
+    )
+    value: int = Field(..., description="Specifies the numerical mask value.")
+    description: Optional[str] = Field(
+        None, description="Specifies a summary description of the mask value function."
+    )
+
+    def __int__(self):
+        return self.value
+
+
+class MaskValue(RootModel[Union[int, MaskValueItem]]):
+    root: Union[int, MaskValueItem]
+
+
+class BitMask(BaseModel):
+    description: Optional[str] = Field(
+        None, description="Specifies a summary description of the bit mask function."
+    )
+    bits: Dict[str, MaskValue]
+
+
+class GroupMask(BaseModel):
+    description: Optional[str] = Field(
+        None, description="Specifies a summary description of the group mask function."
+    )
+    values: Dict[str, MaskValue]
+
+
+class MaskType(RootModel[str]):
+    root: str = Field(
+        ...,
+        description="Specifies the name of the bit mask or group mask used to represent the payload value.",
+    )
+
+
+class InterfaceType(RootModel[str]):
+    root: str = Field(
+        ...,
+        description="Specifies the name of the type used to represent the payload value in the high-level interface.",
+    )
+
+
+class Converter(Enum):
+    None_ = "None"
+    Payload = "Payload"
+    RawPayload = "RawPayload"
+
+
+class MinValue(RootModel[float]):
+    root: float = Field(
+        ...,
+        description="Specifies the minimum allowable value for the payload or payload member.",
+    )
+
+
+class MaxValue(RootModel[float]):
+    root: float = Field(
+        ...,
+        description="Specifies the maximum allowable value for the payload or payload member.",
+    )
+
+
+class DefaultValue(RootModel[float]):
+    root: float = Field(
+        ...,
+        description="Specifies the default value for the payload or payload member.",
+    )
+
+
+class PayloadMember(BaseModel):
+    mask: Optional[int] = Field(
+        None,
+        description="Specifies the mask used to read and write this payload member.",
+    )
+    offset: Optional[int] = Field(
+        None,
+        description="Specifies the payload array offset where this payload member is stored.",
+    )
+    description: Optional[str] = Field(
+        None, description="Specifies a summary description of the payload member."
+    )
+    minValue: Optional[MinValue] = None
+    maxValue: Optional[MaxValue] = None
+    defaultValue: Optional[DefaultValue] = None
+    maskType: Optional[MaskType] = None
+    interfaceType: Optional[InterfaceType] = None
+    converter: Optional[Converter] = None
+
+
+class Visibility(Enum):
+    public = "public"
+    private = "private"
+
+
+class Register(BaseModel):
+    address: Annotated[
+        int,
+        Field(
+            le=255, description="Specifies the unique 8-bit address of the register."
+        ),
+    ]
+    type: Annotated[PayloadType, BeforeValidator(lambda v: PayloadType[v])]
+    length: Annotated[
+        Optional[int],
+        Field(
+            ge=1, default=1, description="Specifies the length of the register payload."
+        ),
+    ]
+    access: Union[Access, List[Access]] = Field(
+        ..., description="Specifies the expected use of the register."
+    )
+    description: Optional[str] = Field(
+        None, description="Specifies a summary description of the register function."
+    )
+    minValue: Optional[MinValue] = None
+    maxValue: Optional[MaxValue] = None
+    defaultValue: Optional[DefaultValue] = None
+    maskType: Optional[MaskType] = None
+    visibility: Optional[Visibility] = Field(
+        None,
+        description="Specifies whether the register function is exposed in the high-level interface.",
+    )
+    volatile: Optional[bool] = Field(
+        None,
+        description="Specifies whether register values can be saved in non-volatile memory.",
+    )
+    payloadSpec: Optional[Dict[str, PayloadMember]] = None
+    interfaceType: Optional[InterfaceType] = None
+    converter: Optional[Converter] = None
+
+    @field_serializer("type")
+    def _serialize_type(self, type: PayloadType):
+        return type.name
+
+
+class Registers(BaseModel):
+    registers: Dict[str, Register] = Field(
+        ...,
+        description="Specifies the collection of registers implementing the device function.",
+    )
+    bitMasks: Optional[Dict[str, BitMask]] = Field(
+        None,
+        description="Specifies the collection of masks available to be used with the different registers.",
+    )
+    groupMasks: Optional[Dict[str, GroupMask]] = Field(
+        None,
+        description="Specifies the collection of group masks available to be used with the different registers.",
+    )
+
+
+class Model(Registers):
+    device: str = Field(..., description="Specifies the name of the device.")
+    whoAmI: int = Field(
+        ..., description="Specifies the unique identifier for this device type."
+    )
+    firmwareVersion: str = Field(
+        ..., description="Specifies the semantic version of the device firmware."
+    )
+    hardwareTargets: str = Field(
+        ..., description="Specifies the semantic version of the device hardware."
+    )