harp-python 0.2.0__tar.gz → 0.4.0__tar.gz

harp_python-0.4.0/.github/workflows/build.yml

@@ -0,0 +1,37 @@
+# Builds the python environment; linter and formatting via ruff; type annotations via pyright;
+# tests via pytest; reports test coverage via pytest-cov.
+
+name: build
+on:
+  push:
+    branches: ['*']
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  build_run_tests:
+    name: Python ${{ matrix.python-version }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    if: github.event.pull_request.draft == false
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: [3.9, 3.12]
+      fail-fast: false
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+      - name: Install dependencies
+        run: pip install -e .[dev]
+
+      - name: ruff
+        run: ruff check .
+      - name: pyright
+        run: pyright . 
+      - name: pytest
+        run: pytest --cov harp

harp_python-0.4.0/.gitignore

@@ -0,0 +1,23 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+
+# Distribution / packaging
+dist/
+_version.py
+*.egg-info/
+*.egg
+
+# IDE
+.vscode/*
+
+# misc
+log*.txt
+scratch/
+scratch*.py
+
+# Test
+.coverage
+
+# Environment
+.venv/
+uv.lock

{harp-python-0.2.0 → harp_python-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: harp-python
-Version: 0.2.0
+Version: 0.4.0
 Summary: A low-level interface for loading binary Harp protocol data
 Author-email: Goncalo Lopes <g.lopes@neurogears.org>
 License: MIT License
@@ -20,15 +20,32 @@ Requires-Dist: pydantic-yaml
 Requires-Dist: pandas
 Provides-Extra: dev
 Requires-Dist: datamodel-code-generator; extra == "dev"
+Requires-Dist: pandas-stubs; extra == "dev"
 Requires-Dist: pytest; extra == "dev"
-Requires-Dist: black; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: pyright; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: codespell; extra == "dev"
 Provides-Extra: jupyter
 Requires-Dist: ipykernel; extra == "jupyter"
 Requires-Dist: matplotlib; extra == "jupyter"
+Dynamic: license-file
 
 # harp-python
 
-A low-level interface to data collected with the [Harp](https://harp-tech.org/) binary protocol.
+A low-level interface to data collected with the [Harp binary protocol](https://harp-tech.org/protocol/BinaryProtocol-8bit.html).
+
+## How to install
+
+The source code is currently hosted on [GitHub](https://github.com/harp-tech/harp-python).
+
+Binary installers for the latest released version are available at the [Python Package Index (PyPI)](https://pypi.org/project/harp-python) and can be installed using `pip`.
+
+```sh
+pip install harp-python
+```
+
+The list of changes between each release can be found [here](https://github.com/harp-tech/harp-python/releases).
 
 ## Data model 
 

{harp-python-0.2.0 → harp_python-0.4.0}/README.md

@@ -1,6 +1,18 @@
 # harp-python
 
-A low-level interface to data collected with the [Harp](https://harp-tech.org/) binary protocol.
+A low-level interface to data collected with the [Harp binary protocol](https://harp-tech.org/protocol/BinaryProtocol-8bit.html).
+
+## How to install
+
+The source code is currently hosted on [GitHub](https://github.com/harp-tech/harp-python).
+
+Binary installers for the latest released version are available at the [Python Package Index (PyPI)](https://pypi.org/project/harp-python) and can be installed using `pip`.
+
+```sh
+pip install harp-python
+```
+
+The list of changes between each release can be found [here](https://github.com/harp-tech/harp-python/releases).
 
 ## Data model 
 

harp_python-0.4.0/harp/__init__.py

@@ -0,0 +1,5 @@
+from harp.io import REFERENCE_EPOCH, MessageType, read, to_buffer, to_file
+from harp.reader import create_reader
+from harp.schema import read_schema
+
+__all__ = ["REFERENCE_EPOCH", "MessageType", "read", "to_buffer", "to_file", "create_reader", "read_schema"]

harp_python-0.4.0/harp/io.py

@@ -0,0 +1,274 @@
+from datetime import datetime
+from enum import IntEnum
+from os import PathLike
+from typing import Any, BinaryIO, Optional, Union
+
+import numpy as np
+import numpy.typing as npt
+import pandas as pd
+from pandas._typing import Axes
+
+from harp.typing import _BufferLike, _FileLike
+
+REFERENCE_EPOCH = datetime(1904, 1, 1)
+"""The reference epoch for UTC harp time."""
+
+
+class MessageType(IntEnum):
+    """Specifies the type of a Harp message."""
+
+    NA = 0
+    READ = 1
+    WRITE = 2
+    EVENT = 3
+
+
+_SECONDS_PER_TICK = 32e-6
+_PAYLOAD_TIMESTAMP_MASK = 0x10
+_messagetypes = [type.name for type in MessageType]
+_dtypefrompayloadtype = {
+    1: np.dtype(np.uint8),
+    2: np.dtype(np.uint16),
+    4: np.dtype(np.uint32),
+    8: np.dtype(np.uint64),
+    129: np.dtype(np.int8),
+    130: np.dtype(np.int16),
+    132: np.dtype(np.int32),
+    136: np.dtype(np.int64),
+    68: np.dtype(np.float32),
+}
+_payloadtypefromdtype = {v: k for k, v in _dtypefrompayloadtype.items()}
+
+
+def read(
+    file_or_buf: Union[_FileLike, _BufferLike],
+    address: Optional[int] = None,
+    dtype: Optional[np.dtype] = None,
+    length: Optional[int] = None,
+    columns: Optional[Axes] = None,
+    epoch: Optional[datetime] = None,
+    keep_type: bool = False,
+):
+    """Read single-register Harp data from the specified file or buffer.
+
+    Parameters
+    ----------
+    file_or_buf
+        File path, open file object, or buffer containing binary data from
+        a single device register.
+    address
+        Expected register address. If specified, the address of
+        the first message is used for validation.
+    dtype
+        Expected data type of the register payload. If specified, the
+        payload type of the first message is used for validation.
+    length
+        Expected number of elements in register payload. If specified, the
+        payload length of the first message is used for validation.
+    columns
+        The optional column labels to use for the data values.
+    epoch
+        Reference datetime at which time zero begins. If specified,
+        the result data frame will have a datetime index.
+    keep_type
+        Specifies whether to include a column with the message type.
+
+    Returns
+    -------
+        A pandas data frame containing message data, sorted by time.
+    """
+    if isinstance(file_or_buf, (str, PathLike, BinaryIO)) or hasattr(file_or_buf, "readinto"):
+        # TODO: in the below we ignore the type as otherwise
+        # we have no way to runtime check _IOProtocol
+        data = np.fromfile(file_or_buf, dtype=np.uint8)  # type: ignore
+    else:
+        data = np.frombuffer(file_or_buf, dtype=np.uint8)
+
+    if len(data) == 0:
+        return pd.DataFrame(
+            columns=columns,
+            index=pd.DatetimeIndex([], name="Time")
+            if epoch
+            else pd.Index([], dtype=np.float64, name="Time"),
+        )
+
+    if address is not None and address != data[2]:
+        raise ValueError(f"expected address {address} but got {data[2]}")
+
+    index = None
+    stride = int(data[1] + 2)
+    nrows = len(data) // stride
+    payloadtype = data[4]
+    payloadoffset = 5
+    if payloadtype & _PAYLOAD_TIMESTAMP_MASK != 0:
+        seconds = np.ndarray(nrows, dtype=np.uint32, buffer=data, offset=payloadoffset, strides=stride)
+        payloadoffset += 4
+        micros = np.ndarray(nrows, dtype=np.uint16, buffer=data, offset=payloadoffset, strides=stride)
+        payloadoffset += 2
+        time = micros * _SECONDS_PER_TICK + seconds
+        payloadtype = payloadtype & ~np.uint8(_PAYLOAD_TIMESTAMP_MASK)
+        if epoch is not None:
+            time = epoch + pd.to_timedelta(time, "s")  # type: ignore
+        index = pd.Series(time)
+        index.name = "Time"
+
+    payloadsize = stride - payloadoffset - 1
+    payloadtype = _dtypefrompayloadtype[payloadtype]
+    if dtype is not None and dtype != payloadtype:
+        raise ValueError(f"expected payload type {dtype} but got {payloadtype}")
+
+    elementsize = payloadtype.itemsize
+    payloadshape = (nrows, payloadsize // elementsize)
+    if length is not None and length != payloadshape[1]:
+        raise ValueError(f"expected payload length {length} but got {payloadshape[1]}")
+
+    payload = np.ndarray(
+        payloadshape,
+        dtype=payloadtype,
+        buffer=data,
+        offset=payloadoffset,
+        strides=(stride, elementsize),
+    )
+
+    result = pd.DataFrame(payload, index=index, columns=columns)
+    if keep_type:
+        msgtype = np.ndarray(nrows, dtype=np.uint8, buffer=data, offset=0, strides=stride)
+        msgtype = pd.Categorical.from_codes(msgtype, categories=_messagetypes)  # type: ignore
+        result[MessageType.__name__] = msgtype
+    return result
+
+
+def to_file(
+    data: pd.DataFrame,
+    file: _FileLike,
+    address: int,
+    dtype: Optional[np.dtype] = None,
+    length: Optional[int] = None,
+    port: Optional[int] = None,
+    epoch: Optional[datetime] = None,
+    message_type: Optional[MessageType] = None,
+):
+    """Write single-register Harp data to the specified file.
+
+    Parameters
+    ----------
+    data
+        Pandas data frame containing message payload.
+    file
+        File path, or open file object in which to store binary data from
+        a single device register.
+    address
+        Register address used to identify all formatted Harp messages.
+    dtype
+        Data type of the register payload. If specified, all data will
+        be converted before formatting the binary payload.
+    length
+        Expected number of elements in register payload. If specified, the
+        number of columns in the input data frame is validated.
+    port
+        Optional port value used for all formatted Harp messages.
+    epoch
+        Reference datetime at which time zero begins. If specified,
+        the input data frame must have a datetime index.
+    message_type
+        Optional message type used for all formatted Harp messages.
+        If not specified, data must contain a MessageType column.
+    """
+    buffer = to_buffer(data, address, dtype, port, length, epoch, message_type)
+    buffer.tofile(file)
+
+
+def to_buffer(
+    data: pd.DataFrame,
+    address: int,
+    dtype: Optional[np.dtype] = None,
+    length: Optional[int] = None,
+    port: Optional[int] = None,
+    epoch: Optional[datetime] = None,
+    message_type: Optional[MessageType] = None,
+) -> npt.NDArray[np.uint8]:
+    """Convert single-register Harp data to a flat binary buffer.
+
+    Parameters
+    ----------
+    data
+        Pandas data frame containing message payload.
+    address
+        Register address used to identify all formatted Harp messages.
+    dtype
+        Data type of the register payload. If specified, all data will
+        be converted before formatting the binary payload.
+    length
+        Expected number of elements in register payload. If specified, the
+        number of columns in the input data frame is validated.
+    port
+        Optional port value used for all formatted Harp messages.
+    epoch
+        Reference datetime at which time zero begins. If specified,
+        the input data frame must have a datetime index.
+    message_type
+        Optional message type used for all formatted Harp messages.
+        If not specified, data must contain a MessageType column.
+
+    Returns
+    -------
+        An array object containing message data formatted according
+        to the Harp binary protocol.
+    """
+    nrows = len(data)
+    if nrows == 0:
+        return np.empty(0, dtype=np.uint8)
+
+    if MessageType.__name__ in data.columns:
+        msgtype = data[MessageType.__name__].cat.codes
+        payload = data[data.columns.drop(MessageType.__name__)].values
+    elif message_type is not None:
+        msgtype = message_type
+        payload = data.values
+    else:
+        raise ValueError(f"message type must be specified either in the data or as argument")
+
+    time = data.index
+    is_timestamped = True
+    if epoch is not None:
+        if not isinstance(time, pd.DatetimeIndex):
+            raise ValueError(f"expected datetime index to encode with epoch but got {time.inferred_type}")
+        time = (time - epoch).total_seconds()
+    elif isinstance(time, pd.RangeIndex):
+        is_timestamped = False
+
+    if dtype is not None:
+        payload = payload.astype(dtype, copy=False)
+
+    ncols = payload.shape[1]
+    if length is not None and ncols != length:
+        raise ValueError(f"expected payload length {length} but got {ncols}")
+
+    if port is None:
+        port = 255
+
+    payloadtype = _payloadtypefromdtype[payload.dtype]
+    payloadlength = ncols * payload.dtype.itemsize
+    stride = payloadlength + 6
+    if is_timestamped:
+        payloadtype |= _PAYLOAD_TIMESTAMP_MASK
+        stride += 6
+
+    buffer = np.empty((nrows, stride), dtype=np.uint8)
+    buffer[:, 0] = msgtype
+    buffer[:, 1:5] = [stride - 2, address, port, payloadtype]
+
+    payloadoffset = 5
+    if is_timestamped:
+        seconds = time.astype(np.uint32)
+        micros = np.around(((time - seconds) / _SECONDS_PER_TICK).values).astype(np.uint16)
+        buffer[:, 5:9] = np.ndarray((nrows, 4), dtype=np.uint8, buffer=seconds.values)
+        buffer[:, 9:11] = np.ndarray((nrows, 2), dtype=np.uint8, buffer=micros)
+        payloadoffset += 6
+
+    payloadstop = payloadoffset + payloadlength
+    buffer[:, payloadoffset:payloadstop] = np.ndarray(
+        (nrows, payloadlength), dtype=np.uint8, buffer=np.ascontiguousarray(payload)
+    )
+    buffer[:, -1] = np.sum(buffer[:, 0:-1], axis=1, dtype=np.uint8)
+    return buffer.reshape(-1)

{harp-python-0.2.0 → harp_python-0.4.0}/harp/model.py

@@ -5,17 +5,9 @@
 from __future__ import annotations
 
 from enum import Enum
-from typing import Dict, List, Optional, Union
-from typing_extensions import Annotated
+from typing import Annotated, Dict, List, Optional, Union
 
-from pydantic import (
-    BaseModel,
-    BeforeValidator,
-    ConfigDict,
-    Field,
-    RootModel,
-    field_serializer,
-)
+from pydantic import BaseModel, BeforeValidator, ConfigDict, Field, RootModel, field_serializer
 
 
 class PayloadType(str, Enum):
@@ -136,16 +128,12 @@ class Visibility(Enum):
 class Register(BaseModel):
     address: Annotated[
         int,
-        Field(
-            le=255, description="Specifies the unique 8-bit address of the register."
-        ),
+        Field(le=255, description="Specifies the unique 8-bit address of the register."),
     ]
     type: Annotated[PayloadType, BeforeValidator(lambda v: PayloadType[v])]
     length: Annotated[
         Optional[int],
-        Field(
-            ge=1, default=1, description="Specifies the length of the register payload."
-        ),
+        Field(ge=1, default=1, description="Specifies the length of the register payload."),
     ]
     access: Union[Access, List[Access]] = Field(
         ..., description="Specifies the expected use of the register."
@@ -191,12 +179,6 @@ class Registers(BaseModel):
 
 class Model(Registers):
     device: str = Field(..., description="Specifies the name of the device.")
-    whoAmI: int = Field(
-        ..., description="Specifies the unique identifier for this device type."
-    )
-    firmwareVersion: str = Field(
-        ..., description="Specifies the semantic version of the device firmware."
-    )
-    hardwareTargets: str = Field(
-        ..., description="Specifies the semantic version of the device hardware."
-    )
+    whoAmI: int = Field(..., description="Specifies the unique identifier for this device type.")
+    firmwareVersion: str = Field(..., description="Specifies the semantic version of the device firmware.")
+    hardwareTargets: str = Field(..., description="Specifies the semantic version of the device hardware.")

{harp-python-0.2.0 → harp_python-0.4.0}/harp/reader.py

@@ -1,22 +1,26 @@
 import os
+from collections import UserDict
+from dataclasses import dataclass
+from datetime import datetime
+from functools import partial
 from math import log2
 from os import PathLike
 from pathlib import Path
-from datetime import datetime
-from functools import partial
-from dataclasses import dataclass
+from typing import Callable, Iterable, Mapping, Optional, Protocol, Union
+
 from numpy import dtype
 from pandas import DataFrame, Series
-from typing import Any, BinaryIO, Callable, Iterable, Optional, Protocol, Union
 from pandas._typing import Axes
-from harp.model import BitMask, GroupMask, Model, PayloadMember, Register
+
 from harp.io import MessageType, read
+from harp.model import BitMask, GroupMask, Model, PayloadMember, Register
 from harp.schema import read_schema
+from harp.typing import _BufferLike, _FileLike
 
 
 @dataclass
 class _ReaderParams:
-    path: Path
+    base_path: Path
     epoch: Optional[datetime] = None
     keep_type: bool = False
 
@@ -24,11 +28,10 @@ class _ReaderParams:
 class _ReadRegister(Protocol):
     def __call__(
         self,
-        file: Optional[Union[str, bytes, PathLike[Any], BinaryIO]] = None,
+        file_or_buf: Optional[Union[_FileLike, _BufferLike]] = None,
         epoch: Optional[datetime] = None,
         keep_type: bool = False,
-    ) -> DataFrame:
-        ...
+    ) -> DataFrame: ...
 
 
 class RegisterReader:
@@ -44,13 +47,27 @@ class RegisterReader:
         self.read = read
 
 
+class RegisterMap(UserDict[str, RegisterReader]):
+    _address_map: Mapping[int, RegisterReader]
+
+    def __init__(self, registers: Mapping[str, RegisterReader]) -> None:
+        super().__init__(registers)
+        self._address_map = {value.register.address: value for value in registers.values()}
+
+    def __getitem__(self, key: Union[str, int]) -> RegisterReader:
+        if isinstance(key, int):
+            return self._address_map[key]
+        else:
+            return super().__getitem__(key)
+
+
 class DeviceReader:
     device: Model
-    registers: dict[str, RegisterReader]
+    registers: RegisterMap
 
-    def __init__(self, device: Model, registers: dict[str, RegisterReader]) -> None:
+    def __init__(self, device: Model, registers: Mapping[str, RegisterReader]) -> None:
         self.device = device
-        self.registers = registers
+        self.registers = RegisterMap(registers)
 
     def __dir__(self) -> Iterable[str]:
         return self.registers.keys()
@@ -65,12 +82,12 @@ def _compose_parser(
     params: _ReaderParams,
 ) -> Callable[..., DataFrame]:
     def parser(
-        file: Optional[Union[str, bytes, PathLike[Any], BinaryIO]] = None,
+        data,
         columns: Optional[Axes] = None,
         epoch: Optional[datetime] = params.epoch,
         keep_type: bool = params.keep_type,
     ):
-        df = g(file, columns, epoch, keep_type)
+        df = g(data, columns, epoch, keep_type)
         result = f(df)
         type_col = df.get(MessageType.__name__)
         if type_col is not None:
@@ -151,16 +168,16 @@ def _create_payloadmember_parser(device: Model, member: PayloadMember):
 
 def _create_register_reader(register: Register, params: _ReaderParams):
     def reader(
-        file: Optional[Union[str, bytes, PathLike[Any], BinaryIO]] = None,
+        file_or_buf: Optional[Union[_FileLike, _BufferLike]] = None,
         columns: Optional[Axes] = None,
         epoch: Optional[datetime] = params.epoch,
         keep_type: bool = params.keep_type,
     ):
-        if file is None:
-            file = f"{params.path}_{register.address}.bin"
+        if file_or_buf is None:
+            file_or_buf = f"{params.base_path}_{register.address}.bin"
 
         data = read(
-            file,
+            file_or_buf,
             address=register.address,
             dtype=dtype(register.type),
             length=register.length,
@@ -173,7 +190,7 @@ def _create_register_reader(register: Register, params: _ReaderParams):
     return reader
 
 
-def _create_register_parser(device: Model, name: str, params: _ReaderParams):
+def _create_register_handler(device: Model, name: str, params: _ReaderParams):
     register = device.registers[name]
     reader = _create_register_reader(register, params)
 
@@ -181,29 +198,34 @@ def _create_register_parser(device: Model, name: str, params: _ReaderParams):
         key = register.maskType.root
         bitMask = None if device.bitMasks is None else device.bitMasks.get(key)
         if bitMask is not None:
-            parser = _create_bitmask_parser(bitMask)
-            reader = _compose_parser(parser, reader, params)
+            bitmask_parser = _create_bitmask_parser(bitMask)
+            reader = _compose_parser(bitmask_parser, reader, params)
             return RegisterReader(register, reader)
 
         groupMask = None if device.groupMasks is None else device.groupMasks.get(key)
         if groupMask is not None:
-            parser = _create_groupmask_parser(name, groupMask)
-            reader = _compose_parser(parser, reader, params)
+            groupmask_parser = _create_groupmask_parser(name, groupMask)
+            reader = _compose_parser(groupmask_parser, reader, params)
             return RegisterReader(register, reader)
 
     if register.payloadSpec is not None:
-        payload_parsers = [
+        member_parsers = [
             (key, _create_payloadmember_parser(device, member))
             for key, member in register.payloadSpec.items()
         ]
 
-        def parser(df: DataFrame):
-            return DataFrame({n: f(df) for n, f in payload_parsers}, index=df.index)
+        def payload_parser(df: DataFrame):
+            return DataFrame({n: f(df) for n, f in member_parsers}, index=df.index)
 
-        reader = _compose_parser(parser, reader, params)
+        reader = _compose_parser(payload_parser, reader, params)
         return RegisterReader(register, reader)
 
-    reader = partial(reader, columns=[name])
+    columns = (
+        [name]
+        if register.length is None or register.length == 1
+        else [f"{name}_{i}" for i in range(register.length)]
+    )
+    reader = partial(reader, columns=columns)
     return RegisterReader(register, reader)
 
 
@@ -214,7 +236,7 @@ def create_reader(
     keep_type: bool = False,
 ):
     """Creates a device reader object from the specified dataset or schema.
-    
+
     Parameters
     ----------
     device
@@ -249,9 +271,7 @@ def create_reader(
         base_path = path / device.device if is_dir else path.parent / device.device
 
     reg_readers = {
-        name: _create_register_parser(
-            device, name, _ReaderParams(base_path, epoch, keep_type)
-        )
+        name: _create_register_handler(device, name, _ReaderParams(base_path, epoch, keep_type))
         for name in device.registers.keys()
     }
     return DeviceReader(device, reg_readers)

{harp-python-0.2.0 → harp_python-0.4.0}/harp/schema.py

@@ -1,21 +1,24 @@
+from importlib import resources
 from os import PathLike
 from typing import TextIO, Union
-from harp.model import Model, Registers
+
 from pydantic_yaml import parse_yaml_raw_as
-from importlib import resources
+
+from harp.model import Model, Registers
 
 
 def _read_common_registers() -> Registers:
+    if __package__ is None:
+        raise ValueError("__package__ is None: unable to read common registers")
+
     file = resources.files(__package__) / "common.yml"
-    with file.open("rt") as fileIO:
+    with file.open("r") as fileIO:
         return parse_yaml_raw_as(Registers, fileIO.read())
 
 
-def read_schema(
-    file: Union[str, PathLike, TextIO], include_common_registers: bool = True
-) -> Model:
+def read_schema(file: Union[str, PathLike, TextIO], include_common_registers: bool = True) -> Model:
     """Read and parse a device schema from the specified file.
-    
+
     Parameters
     ----------
     file
@@ -35,14 +38,12 @@ def read_schema(
             return read_schema(fileIO)
     else:
         schema = parse_yaml_raw_as(Model, file.read())
-        if not "WhoAmI" in schema.registers and include_common_registers:
+        if "WhoAmI" not in schema.registers and include_common_registers:
             common = _read_common_registers()
             schema.registers = dict(common.registers, **schema.registers)
             if common.bitMasks:
                 schema.bitMasks = (
-                    common.bitMasks
-                    if schema.bitMasks is None
-                    else dict(common.bitMasks, **schema.bitMasks)
+                    common.bitMasks if schema.bitMasks is None else dict(common.bitMasks, **schema.bitMasks)
                 )
             if common.groupMasks:
                 schema.groupMasks = (

harp_python-0.4.0/harp/typing.py

@@ -0,0 +1,13 @@
+import mmap
+import sys
+from os import PathLike
+from typing import Any, BinaryIO, Union
+
+from numpy.typing import NDArray
+
+if sys.version_info >= (3, 12):
+    from collections.abc import Buffer as _BufferLike
+else:
+    _BufferLike = Union[bytes, bytearray, memoryview, mmap.mmap, NDArray[Any]]
+
+_FileLike = Union[str, PathLike[str], BinaryIO]

{harp-python-0.2.0 → harp_python-0.4.0}/harp_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: harp-python
-Version: 0.2.0
+Version: 0.4.0
 Summary: A low-level interface for loading binary Harp protocol data
 Author-email: Goncalo Lopes <g.lopes@neurogears.org>
 License: MIT License
@@ -20,15 +20,32 @@ Requires-Dist: pydantic-yaml
 Requires-Dist: pandas
 Provides-Extra: dev
 Requires-Dist: datamodel-code-generator; extra == "dev"
+Requires-Dist: pandas-stubs; extra == "dev"
 Requires-Dist: pytest; extra == "dev"
-Requires-Dist: black; extra == "dev"
+Requires-Dist: pytest-cov; extra == "dev"
+Requires-Dist: pyright; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: codespell; extra == "dev"
 Provides-Extra: jupyter
 Requires-Dist: ipykernel; extra == "jupyter"
 Requires-Dist: matplotlib; extra == "jupyter"
+Dynamic: license-file
 
 # harp-python
 
-A low-level interface to data collected with the [Harp](https://harp-tech.org/) binary protocol.
+A low-level interface to data collected with the [Harp binary protocol](https://harp-tech.org/protocol/BinaryProtocol-8bit.html).
+
+## How to install
+
+The source code is currently hosted on [GitHub](https://github.com/harp-tech/harp-python).
+
+Binary installers for the latest released version are available at the [Python Package Index (PyPI)](https://pypi.org/project/harp-python) and can be installed using `pip`.
+
+```sh
+pip install harp-python
+```
+
+The list of changes between each release can be found [here](https://github.com/harp-tech/harp-python/releases).
 
 ## Data model 
 

{harp-python-0.2.0 → harp_python-0.4.0}/harp_python.egg-info/SOURCES.txt

@@ -4,12 +4,14 @@ LICENSE
 MANIFEST.in
 README.md
 pyproject.toml
+.github/workflows/build.yml
 harp/__init__.py
 harp/common.yml
 harp/io.py
 harp/model.py
 harp/reader.py
 harp/schema.py
+harp/typing.py
 harp_python.egg-info/PKG-INFO
 harp_python.egg-info/SOURCES.txt
 harp_python.egg-info/dependency_links.txt
@@ -22,4 +24,6 @@ tests/test_reader.py
 tests/test_schema.py
 tests/data/device.yml
 tests/data/device_0.bin
+tests/data/device_44.bin
+tests/data/empty_0.bin
 tests/data/write_0.bin

{harp-python-0.2.0 → harp_python-0.4.0}/harp_python.egg-info/requires.txt

@@ -3,8 +3,12 @@ pandas
 
 [dev]
 datamodel-code-generator
+pandas-stubs
 pytest
-black
+pytest-cov
+pyright
+ruff
+codespell
 
 [jupyter]
 ipykernel

{harp-python-0.2.0 → harp_python-0.4.0}/pyproject.toml

@@ -31,9 +31,14 @@ classifiers = [
 [project.optional-dependencies]
 dev = [
     "datamodel-code-generator",
+    "pandas-stubs",
     "pytest",
-    "black"
+    "pytest-cov",
+    "pyright",
+    "ruff",
+    "codespell"
 ]
+
 jupyter = [
     "ipykernel",
     "matplotlib"
@@ -55,16 +60,24 @@ include = ["harp*"]
 
 [tool.setuptools_scm]
 
-[tool.black]
-line-length = 88
-target-version = ['py39']
-include = '\.pyi?$'
-extend-exclude = '''
-# A regex preceded with ^/ will apply only to files and directories
-# in the root of the project.
-(
-  ^/LICENSE
-  ^/README.md
-  | reflex-generator
-)
-'''
+[tool.ruff]
+line-length = 108
+target-version = "py39"
+exclude = [
+    "reflex-generator"
+]
+
+[tool.ruff.lint]
+select = ["I"]
+
+[tool.pyright]
+venvPath = "."
+venv = ".venv"
+exclude = [
+    ".venv/*",
+    "reflex-generator"
+]
+
+[tool.codespell]
+skip = '.git,*.pdf,*.svg'
+ignore-words-list = 'nd'

{harp-python-0.2.0 → harp_python-0.4.0}/tests/data/device.yml

@@ -18,6 +18,12 @@ registers:
     type: U8
     maskType: DigitalInputs
     description: Reports the state of the digital input lines.
+  AnalogData:
+    address: 44
+    type: S16
+    length: 3
+    access: Event
+    description: Reports the current values of the analog input lines.
 bitMasks:
   DigitalInputs:
     description: Specifies the state of the digital input lines.

{harp-python-0.2.0 → harp_python-0.4.0}/tests/params.py

@@ -1,8 +1,11 @@
-import numpy as np
+from dataclasses import dataclass
+from datetime import datetime
 from os import PathLike
 from pathlib import Path
-from dataclasses import dataclass
 from typing import Iterable, Optional, Type, Union
+
+import numpy as np
+
 from harp.model import Model
 
 datapath = Path(__file__).parent
@@ -17,6 +20,8 @@ class DataFileParam:
     expected_dtype: Optional[np.dtype] = None
     expected_length: Optional[int] = None
     expected_error: Optional[Type[BaseException]] = None
+    repeat_data: Optional[int] = None
+    epoch: Optional[datetime] = None
     keep_type: bool = False
 
     def __post_init__(self):

harp_python-0.4.0/tests/test_io.py

@@ -0,0 +1,88 @@
+from contextlib import nullcontext
+
+import numpy as np
+import pandas as pd
+import pytest
+from pytest import mark
+
+from harp.io import REFERENCE_EPOCH, MessageType, read, to_buffer
+from tests.params import DataFileParam
+
+testdata = [
+    DataFileParam(path="data/device_0.bin", expected_rows=1),
+    DataFileParam(
+        path="data/device_0.bin",
+        expected_rows=1,
+        expected_address=1,  # actual address is 0
+        expected_error=ValueError,
+    ),
+    DataFileParam(
+        path="data/device_0.bin",
+        expected_rows=1,
+        expected_dtype=np.dtype("uint8"),  # actual dtype is uint16
+        expected_error=ValueError,
+    ),
+    DataFileParam(
+        path="data/device_0.bin",
+        expected_rows=1,
+        expected_length=2,  # actual length is 1
+        expected_error=ValueError,
+    ),
+    DataFileParam(path="data/write_0.bin", expected_address=0, expected_rows=4),
+    DataFileParam(path="data/write_0.bin", expected_address=0, expected_rows=4, keep_type=True),
+    DataFileParam(path="data/device_0.bin", expected_rows=300, repeat_data=300),
+    DataFileParam(path="data/device_0.bin", expected_rows=1, epoch=REFERENCE_EPOCH),
+    DataFileParam(path="data/empty_0.bin", expected_rows=0, epoch=REFERENCE_EPOCH),
+    DataFileParam(path="data/empty_0.bin", expected_rows=0),
+]
+
+
+@mark.parametrize("dataFile", testdata)
+def test_read(dataFile: DataFileParam):
+    context = pytest.raises if dataFile.expected_error else nullcontext
+    with context(dataFile.expected_error):  # type: ignore
+        file_or_buf = dataFile.path
+        if dataFile.repeat_data:
+            with open(file_or_buf, "rb") as f:
+                file_or_buf = f.read() * dataFile.repeat_data
+
+        data = read(
+            file_or_buf,
+            address=dataFile.expected_address,
+            dtype=dataFile.expected_dtype,
+            length=dataFile.expected_length,
+            epoch=dataFile.epoch,
+            keep_type=dataFile.keep_type,
+        )
+
+        assert len(data) == dataFile.expected_rows
+        assert isinstance(data.index, pd.DatetimeIndex if dataFile.epoch else pd.Index)
+        if dataFile.keep_type:
+            assert MessageType.__name__ in data.columns and data[MessageType.__name__].dtype == "category"
+
+        if dataFile.expected_cols:
+            for col in dataFile.expected_cols:
+                assert col in data.columns
+
+
+writedata = [
+    DataFileParam(path="data/device_0.bin", expected_rows=1, expected_address=0, keep_type=True),
+]
+
+
+@mark.parametrize("dataFile", writedata)
+def test_write(dataFile: DataFileParam):
+    if dataFile.expected_address is None:
+        raise AssertionError("expected address must be defined for all write tests")
+
+    buffer = np.fromfile(dataFile.path, np.uint8)
+    data = read(
+        buffer,
+        address=dataFile.expected_address,
+        dtype=dataFile.expected_dtype,
+        length=dataFile.expected_length,
+        keep_type=dataFile.keep_type,
+    )
+    assert len(data) == dataFile.expected_rows
+    write_buffer = to_buffer(data, address=dataFile.expected_address, length=dataFile.expected_length)
+    assert np.array_equal(buffer, write_buffer)

{harp-python-0.2.0 → harp_python-0.4.0}/tests/test_reader.py

@@ -1,5 +1,6 @@
-import numpy as np
+import pandas as pd
 from pytest import mark
+
 from harp.io import REFERENCE_EPOCH, MessageType
 from harp.reader import create_reader
 from tests.params import DeviceSchemaParam
@@ -8,12 +9,12 @@ testdata = [
     DeviceSchemaParam(
         path="data",
         expected_whoAmI=0,
-        expected_registers=["DigitalInputMode"],
+        expected_registers=["AnalogData"],
     ),
     DeviceSchemaParam(
         path="data/device.yml",
         expected_whoAmI=0,
-        expected_registers=["DigitalInputMode"],
+        expected_registers=["AnalogData"],
     ),
 ]
 
@@ -25,8 +26,13 @@ def test_create_reader(schemaFile: DeviceSchemaParam):
 
     whoAmI = reader.WhoAmI.read()
     assert reader.device.whoAmI == whoAmI.iloc[0, 0]
-    assert whoAmI.index.dtype.type == np.datetime64
+    assert isinstance(whoAmI.index, pd.DatetimeIndex)
 
     whoAmI = reader.WhoAmI.read(epoch=None, keep_type=True)
-    assert whoAmI.index.dtype.type == np.float64
+    assert isinstance(whoAmI.index, pd.Index)
     assert whoAmI.iloc[0, -1] == MessageType.READ.name
+
+    if schemaFile.expected_registers:
+        for register_name in schemaFile.expected_registers:
+            data = reader.registers[register_name].read()
+            assert isinstance(data.index, pd.DatetimeIndex)

{harp-python-0.2.0 → harp_python-0.4.0}/tests/test_schema.py

@@ -1,4 +1,5 @@
 from pytest import mark
+
 from harp.schema import read_schema
 from tests.params import DeviceSchemaParam
 

harp-python-0.2.0/.gitignore

@@ -1,12 +0,0 @@
-# Environment files
-.vscode
-.venv
-
-# Python temp files
-__pycache__
-*.pyc
-*.egg-info
-dist
-
-# Data files
-/data

harp-python-0.2.0/harp/__init__.py

@@ -1,3 +0,0 @@
-from harp.io import REFERENCE_EPOCH, MessageType, read
-from harp.reader import create_reader
-from harp.schema import read_schema

harp-python-0.2.0/harp/io.py

@@ -1,129 +0,0 @@
-from datetime import datetime
-from enum import IntEnum
-from os import PathLike
-from typing import Any, BinaryIO, Optional, Union
-from pandas._typing import Axes
-import numpy as np
-import pandas as pd
-
-REFERENCE_EPOCH = datetime(1904, 1, 1)
-"""The reference epoch for UTC harp time."""
-
-
-class MessageType(IntEnum):
-    """Specifies the type of a Harp message."""
-
-    NA = 0
-    READ = 1
-    WRITE = 2
-    EVENT = 3
-
-
-_SECONDS_PER_TICK = 32e-6
-_messagetypes = [type.name for type in MessageType]
-_payloadtypes = {
-    1: np.dtype(np.uint8),
-    2: np.dtype(np.uint16),
-    4: np.dtype(np.uint32),
-    8: np.dtype(np.uint64),
-    129: np.dtype(np.int8),
-    130: np.dtype(np.int16),
-    132: np.dtype(np.int32),
-    136: np.dtype(np.int64),
-    68: np.dtype(np.float32),
-}
-
-
-def read(
-    file: Union[str, bytes, PathLike[Any], BinaryIO],
-    address: Optional[int] = None,
-    dtype: Optional[np.dtype] = None,
-    length: Optional[int] = None,
-    columns: Optional[Axes] = None,
-    epoch: Optional[datetime] = None,
-    keep_type: bool = False,
-):
-    """Read single-register Harp data from the specified file.
-
-    Parameters
-    ----------
-    file
-        Open file object or filename containing binary data from
-        a single device register.
-    address
-        Expected register address. If specified, the address of
-        the first message in the file is used for validation.
-    dtype
-        Expected data type of the register payload. If specified, the
-        payload type of the first message in the file is used for validation.
-    length
-        Expected number of elements in register payload. If specified, the
-        payload length of the first message in the file is used for validation.
-    columns
-        The optional column labels to use for the data values.
-    epoch
-        Reference datetime at which time zero begins. If specified,
-        the result data frame will have a datetime index.
-    keep_type
-        Specifies whether to include a column with the message type.
-
-    Returns
-    -------
-        A pandas data frame containing message data, sorted by time.
-    """
-    data = np.fromfile(file, dtype=np.uint8)
-    if len(data) == 0:
-        return pd.DataFrame(
-            columns=columns, index=pd.Index([], dtype=np.float64, name="Time")
-        )
-
-    if address is not None and address != data[2]:
-        raise ValueError(f"expected address {address} but got {data[2]}")
-
-    index = None
-    stride = data[1] + 2
-    nrows = len(data) // stride
-    payloadtype = data[4]
-    payloadoffset = 5
-    if payloadtype & 0x10 != 0:
-        seconds = np.ndarray(
-            nrows, dtype=np.uint32, buffer=data, offset=payloadoffset, strides=stride
-        )
-        payloadoffset += 4
-        micros = np.ndarray(
-            nrows, dtype=np.uint16, buffer=data, offset=payloadoffset, strides=stride
-        )
-        payloadoffset += 2
-        time = micros * _SECONDS_PER_TICK + seconds
-        payloadtype = payloadtype & ~0x10
-        if epoch is not None:
-            time = epoch + pd.to_timedelta(time, "s")  # type: ignore
-        index = pd.Series(time)
-        index.name = "Time"
-
-    payloadsize = stride - payloadoffset - 1
-    payloadtype = _payloadtypes[payloadtype]
-    if dtype is not None and dtype != payloadtype:
-        raise ValueError(f"expected payload type {dtype} but got {payloadtype}")
-
-    elementsize = payloadtype.itemsize
-    payloadshape = (nrows, payloadsize // elementsize)
-    if length is not None and length != payloadshape[1]:
-        raise ValueError(f"expected payload length {length} but got {payloadshape[1]}")
-
-    payload = np.ndarray(
-        payloadshape,
-        dtype=payloadtype,
-        buffer=data,
-        offset=payloadoffset,
-        strides=(stride, elementsize),
-    )
-
-    result = pd.DataFrame(payload, index=index, columns=columns)
-    if keep_type:
-        msgtype = np.ndarray(
-            nrows, dtype=np.uint8, buffer=data, offset=0, strides=stride
-        )
-        msgtype = pd.Categorical.from_codes(msgtype, categories=_messagetypes)  # type: ignore
-        result[MessageType.__name__] = msgtype
-    return result

harp-python-0.2.0/tests/test_io.py

@@ -1,55 +0,0 @@
-import pytest
-import numpy as np
-from contextlib import nullcontext
-from pytest import mark
-from harp.io import read, MessageType
-from tests.params import DataFileParam
-
-testdata = [
-    DataFileParam(path="data/device_0.bin", expected_rows=1),
-    DataFileParam(
-        path="data/device_0.bin",
-        expected_rows=1,
-        expected_address=1,  # actual address is 0
-        expected_error=ValueError,
-    ),
-    DataFileParam(
-        path="data/device_0.bin",
-        expected_rows=1,
-        expected_dtype=np.dtype("uint8"),  # actual dtype is uint16
-        expected_error=ValueError,
-    ),
-    DataFileParam(
-        path="data/device_0.bin",
-        expected_rows=1,
-        expected_length=2,  # actual length is 1
-        expected_error=ValueError,
-    ),
-    DataFileParam(path="data/write_0.bin", expected_address=0, expected_rows=4),
-    DataFileParam(
-        path="data/write_0.bin", expected_address=0, expected_rows=4, keep_type=True
-    ),
-]
-
-
-@mark.parametrize("dataFile", testdata)
-def test_read(dataFile: DataFileParam):
-    context = pytest.raises if dataFile.expected_error else nullcontext
-    with context(dataFile.expected_error):  # type: ignore
-        data = read(
-            dataFile.path,
-            address=dataFile.expected_address,
-            dtype=dataFile.expected_dtype,
-            length=dataFile.expected_length,
-            keep_type=dataFile.keep_type,
-        )
-        assert len(data) == dataFile.expected_rows
-        if dataFile.keep_type:
-            assert (
-                MessageType.__name__ in data.columns
-                and data[MessageType.__name__].dtype == "category"
-            )
-
-        if dataFile.expected_cols:
-            for col in dataFile.expected_cols:
-                assert col in data.columns