harp-python 0.3.0__py3-none-any.whl → 0.4.1__py3-none-any.whl

harp/__init__.py

@@ -1,5 +1,5 @@
-from harp.io import REFERENCE_EPOCH, MessageType, read
+from harp.io import REFERENCE_EPOCH, MessageType, read, to_buffer, to_file
 from harp.reader import create_reader
 from harp.schema import read_schema
 
-__all__ = ["REFERENCE_EPOCH", "MessageType", "read", "create_reader", "read_schema"]
+__all__ = ["REFERENCE_EPOCH", "MessageType", "read", "to_buffer", "to_file", "create_reader", "read_schema"]

harp/io.py

@@ -4,8 +4,11 @@ from os import PathLike
 from typing import Any, BinaryIO, Optional, Union
 
 import numpy as np
+import numpy.typing as npt
 import pandas as pd
-from pandas._typing import Axes
+from pandas._typing import Axes  # pyright: ignore[reportPrivateImportUsage]
+
+from harp.typing import _BufferLike, _FileLike
 
 REFERENCE_EPOCH = datetime(1904, 1, 1)
 """The reference epoch for UTC harp time."""
@@ -21,8 +24,9 @@ class MessageType(IntEnum):
 
 
 _SECONDS_PER_TICK = 32e-6
+_PAYLOAD_TIMESTAMP_MASK = 0x10
 _messagetypes = [type.name for type in MessageType]
-_payloadtypes = {
+_dtypefrompayloadtype = {
     1: np.dtype(np.uint8),
     2: np.dtype(np.uint16),
     4: np.dtype(np.uint32),
@@ -33,10 +37,11 @@ _payloadtypes = {
     136: np.dtype(np.int64),
     68: np.dtype(np.float32),
 }
+_payloadtypefromdtype = {v: k for k, v in _dtypefrompayloadtype.items()}
 
 
 def read(
-    file: Union[str, bytes, PathLike[Any], BinaryIO],
+    file_or_buf: Union[_FileLike, _BufferLike],
     address: Optional[int] = None,
     dtype: Optional[np.dtype] = None,
     length: Optional[int] = None,
@@ -44,22 +49,22 @@ def read(
     epoch: Optional[datetime] = None,
     keep_type: bool = False,
 ):
-    """Read single-register Harp data from the specified file.
+    """Read single-register Harp data from the specified file or buffer.
 
     Parameters
     ----------
-    file
-        Open file object or filename containing binary data from
+    file_or_buf
+        File path, open file object, or buffer containing binary data from
         a single device register.
     address
         Expected register address. If specified, the address of
-        the first message in the file is used for validation.
+        the first message is used for validation.
     dtype
         Expected data type of the register payload. If specified, the
-        payload type of the first message in the file is used for validation.
+        payload type of the first message is used for validation.
     length
         Expected number of elements in register payload. If specified, the
-        payload length of the first message in the file is used for validation.
+        payload length of the first message is used for validation.
     columns
         The optional column labels to use for the data values.
     epoch
@@ -72,9 +77,20 @@ def read(
     -------
         A pandas data frame containing message data, sorted by time.
     """
-    data = np.fromfile(file, dtype=np.uint8)
+    if isinstance(file_or_buf, (str, PathLike, BinaryIO)) or hasattr(file_or_buf, "readinto"):
+        # TODO: in the below we ignore the type as otherwise
+        # we have no way to runtime check _IOProtocol
+        data = np.fromfile(file_or_buf, dtype=np.uint8)  # type: ignore
+    else:
+        data = np.frombuffer(file_or_buf, dtype=np.uint8)
+
     if len(data) == 0:
-        return pd.DataFrame(columns=columns, index=pd.Index([], dtype=np.float64, name="Time"))
+        return pd.DataFrame(
+            columns=columns,
+            index=pd.DatetimeIndex([], name="Time")
+            if epoch
+            else pd.Index([], dtype=np.float64, name="Time"),
+        )
 
     if address is not None and address != data[2]:
         raise ValueError(f"expected address {address} but got {data[2]}")
@@ -84,20 +100,20 @@ def read(
     nrows = len(data) // stride
     payloadtype = data[4]
     payloadoffset = 5
-    if payloadtype & 0x10 != 0:
+    if payloadtype & _PAYLOAD_TIMESTAMP_MASK != 0:
         seconds = np.ndarray(nrows, dtype=np.uint32, buffer=data, offset=payloadoffset, strides=stride)
         payloadoffset += 4
         micros = np.ndarray(nrows, dtype=np.uint16, buffer=data, offset=payloadoffset, strides=stride)
         payloadoffset += 2
         time = micros * _SECONDS_PER_TICK + seconds
-        payloadtype = payloadtype & ~np.uint8(0x10)
+        payloadtype = payloadtype & ~np.uint8(_PAYLOAD_TIMESTAMP_MASK)
         if epoch is not None:
             time = epoch + pd.to_timedelta(time, "s")  # type: ignore
         index = pd.Series(time)
         index.name = "Time"
 
     payloadsize = stride - payloadoffset - 1
-    payloadtype = _payloadtypes[payloadtype]
+    payloadtype = _dtypefrompayloadtype[payloadtype]
     if dtype is not None and dtype != payloadtype:
         raise ValueError(f"expected payload type {dtype} but got {payloadtype}")
 
@@ -120,3 +136,139 @@ def read(
         msgtype = pd.Categorical.from_codes(msgtype, categories=_messagetypes)  # type: ignore
         result[MessageType.__name__] = msgtype
     return result
+
+
+def to_file(
+    data: pd.DataFrame,
+    file: _FileLike,
+    address: int,
+    dtype: Optional[np.dtype] = None,
+    length: Optional[int] = None,
+    port: Optional[int] = None,
+    epoch: Optional[datetime] = None,
+    message_type: Optional[MessageType] = None,
+):
+    """Write single-register Harp data to the specified file.
+
+    Parameters
+    ----------
+    data
+        Pandas data frame containing message payload.
+    file
+        File path, or open file object in which to store binary data from
+        a single device register.
+    address
+        Register address used to identify all formatted Harp messages.
+    dtype
+        Data type of the register payload. If specified, all data will
+        be converted before formatting the binary payload.
+    length
+        Expected number of elements in register payload. If specified, the
+        number of columns in the input data frame is validated.
+    port
+        Optional port value used for all formatted Harp messages.
+    epoch
+        Reference datetime at which time zero begins. If specified,
+        the input data frame must have a datetime index.
+    message_type
+        Optional message type used for all formatted Harp messages.
+        If not specified, data must contain a MessageType column.
+    """
+    buffer = to_buffer(data, address, dtype, port, length, epoch, message_type)
+    buffer.tofile(file)
+
+
+def to_buffer(
+    data: pd.DataFrame,
+    address: int,
+    dtype: Optional[np.dtype] = None,
+    length: Optional[int] = None,
+    port: Optional[int] = None,
+    epoch: Optional[datetime] = None,
+    message_type: Optional[MessageType] = None,
+) -> npt.NDArray[np.uint8]:
+    """Convert single-register Harp data to a flat binary buffer.
+
+    Parameters
+    ----------
+    data
+        Pandas data frame containing message payload.
+    address
+        Register address used to identify all formatted Harp messages.
+    dtype
+        Data type of the register payload. If specified, all data will
+        be converted before formatting the binary payload.
+    length
+        Expected number of elements in register payload. If specified, the
+        number of columns in the input data frame is validated.
+    port
+        Optional port value used for all formatted Harp messages.
+    epoch
+        Reference datetime at which time zero begins. If specified,
+        the input data frame must have a datetime index.
+    message_type
+        Optional message type used for all formatted Harp messages.
+        If not specified, data must contain a MessageType column.
+
+    Returns
+    -------
+        An array object containing message data formatted according
+        to the Harp binary protocol.
+    """
+    nrows = len(data)
+    if nrows == 0:
+        return np.empty(0, dtype=np.uint8)
+
+    if MessageType.__name__ in data.columns:
+        msgtype = data[MessageType.__name__].cat.codes
+        payload = data[data.columns.drop(MessageType.__name__)].values
+    elif message_type is not None:
+        msgtype = message_type
+        payload = data.values
+    else:
+        raise ValueError(f"message type must be specified either in the data or as argument")
+
+    time = data.index
+    is_timestamped = True
+    if epoch is not None:
+        if not isinstance(time, pd.DatetimeIndex):
+            raise ValueError(f"expected datetime index to encode with epoch but got {time.inferred_type}")
+        time = (time - epoch).total_seconds()
+    elif isinstance(time, pd.RangeIndex):
+        is_timestamped = False
+
+    if dtype is not None:
+        payload = payload.astype(dtype, copy=False)
+
+    ncols = payload.shape[1]
+    if length is not None and ncols != length:
+        raise ValueError(f"expected payload length {length} but got {ncols}")
+
+    if port is None:
+        port = 255
+
+    payloadtype = _payloadtypefromdtype[payload.dtype]
+    payloadlength = ncols * payload.dtype.itemsize
+    stride = payloadlength + 6
+    if is_timestamped:
+        payloadtype |= _PAYLOAD_TIMESTAMP_MASK
+        stride += 6
+
+    buffer = np.empty((nrows, stride), dtype=np.uint8)
+    buffer[:, 0] = msgtype
+    buffer[:, 1:5] = [stride - 2, address, port, payloadtype]
+
+    payloadoffset = 5
+    if is_timestamped:
+        seconds = time.astype(np.uint32)
+        micros = np.around(((time - seconds) / _SECONDS_PER_TICK).values).astype(np.uint16)
+        buffer[:, 5:9] = np.ndarray((nrows, 4), dtype=np.uint8, buffer=seconds.values)
+        buffer[:, 9:11] = np.ndarray((nrows, 2), dtype=np.uint8, buffer=micros)
+        payloadoffset += 6
+
+    payloadstop = payloadoffset + payloadlength
+    buffer[:, payloadoffset:payloadstop] = np.ndarray(
+        (nrows, payloadlength), dtype=np.uint8, buffer=np.ascontiguousarray(payload)
+    )
+    buffer[:, -1] = np.sum(buffer[:, 0:-1], axis=1, dtype=np.uint8)
+    return buffer.reshape(-1)

harp/model.py

@@ -7,14 +7,7 @@ from __future__ import annotations
 from enum import Enum
 from typing import Annotated, Dict, List, Optional, Union
 
-from pydantic import (
-    BaseModel,
-    BeforeValidator,
-    ConfigDict,
-    Field,
-    RootModel,
-    field_serializer,
-)
+from pydantic import BaseModel, BeforeValidator, ConfigDict, Field, RootModel, field_serializer
 
 
 class PayloadType(str, Enum):

harp/reader.py

@@ -6,20 +6,21 @@ from functools import partial
 from math import log2
 from os import PathLike
 from pathlib import Path
-from typing import Any, BinaryIO, Callable, Iterable, Mapping, Optional, Protocol, Union
+from typing import Callable, Iterable, Mapping, Optional, Protocol, Union
 
 from numpy import dtype
 from pandas import DataFrame, Series
-from pandas._typing import Axes
+from pandas._typing import Axes  # pyright: ignore[reportPrivateImportUsage]
 
 from harp.io import MessageType, read
 from harp.model import BitMask, GroupMask, Model, PayloadMember, Register
 from harp.schema import read_schema
+from harp.typing import _BufferLike, _FileLike
 
 
 @dataclass
 class _ReaderParams:
-    path: Path
+    base_path: Path
     epoch: Optional[datetime] = None
     keep_type: bool = False
 
@@ -27,7 +28,7 @@ class _ReaderParams:
 class _ReadRegister(Protocol):
     def __call__(
         self,
-        file: Optional[Union[str, bytes, PathLike[Any], BinaryIO]] = None,
+        file_or_buf: Optional[Union[_FileLike, _BufferLike]] = None,
         epoch: Optional[datetime] = None,
         keep_type: bool = False,
     ) -> DataFrame: ...
@@ -81,12 +82,12 @@ def _compose_parser(
     params: _ReaderParams,
 ) -> Callable[..., DataFrame]:
     def parser(
-        file: Optional[Union[str, bytes, PathLike[Any], BinaryIO]] = None,
+        data: Optional[Union[_FileLike, _BufferLike]] = None,
         columns: Optional[Axes] = None,
         epoch: Optional[datetime] = params.epoch,
         keep_type: bool = params.keep_type,
     ):
-        df = g(file, columns, epoch, keep_type)
+        df = g(data, columns, epoch, keep_type)
         result = f(df)
         type_col = df.get(MessageType.__name__)
         if type_col is not None:
@@ -167,16 +168,16 @@ def _create_payloadmember_parser(device: Model, member: PayloadMember):
 
 def _create_register_reader(register: Register, params: _ReaderParams):
     def reader(
-        file: Optional[Union[str, bytes, PathLike[Any], BinaryIO]] = None,
+        file_or_buf: Optional[Union[_FileLike, _BufferLike]] = None,
         columns: Optional[Axes] = None,
         epoch: Optional[datetime] = params.epoch,
         keep_type: bool = params.keep_type,
     ):
-        if file is None:
-            file = f"{params.path}_{register.address}.bin"
+        if file_or_buf is None:
+            file_or_buf = f"{params.base_path}_{register.address}.bin"
 
         data = read(
-            file,
+            file_or_buf,
             address=register.address,
             dtype=dtype(register.type),
             length=register.length,
@@ -189,7 +190,7 @@ def _create_register_reader(register: Register, params: _ReaderParams):
     return reader
 
 
-def _create_register_parser(device: Model, name: str, params: _ReaderParams):
+def _create_register_handler(device: Model, name: str, params: _ReaderParams):
     register = device.registers[name]
     reader = _create_register_reader(register, params)
 
@@ -197,29 +198,34 @@ def _create_register_parser(device: Model, name: str, params: _ReaderParams):
         key = register.maskType.root
         bitMask = None if device.bitMasks is None else device.bitMasks.get(key)
         if bitMask is not None:
-            parser = _create_bitmask_parser(bitMask)
-            reader = _compose_parser(parser, reader, params)
+            bitmask_parser = _create_bitmask_parser(bitMask)
+            reader = _compose_parser(bitmask_parser, reader, params)
             return RegisterReader(register, reader)
 
         groupMask = None if device.groupMasks is None else device.groupMasks.get(key)
         if groupMask is not None:
-            parser = _create_groupmask_parser(name, groupMask)
-            reader = _compose_parser(parser, reader, params)
+            groupmask_parser = _create_groupmask_parser(name, groupMask)
+            reader = _compose_parser(groupmask_parser, reader, params)
             return RegisterReader(register, reader)
 
     if register.payloadSpec is not None:
-        payload_parsers = [
+        member_parsers = [
             (key, _create_payloadmember_parser(device, member))
             for key, member in register.payloadSpec.items()
         ]
 
-        def parser(df: DataFrame):
-            return DataFrame({n: f(df) for n, f in payload_parsers}, index=df.index)
+        def payload_parser(df: DataFrame):
+            return DataFrame({n: f(df) for n, f in member_parsers}, index=df.index)
 
-        reader = _compose_parser(parser, reader, params)
+        reader = _compose_parser(payload_parser, reader, params)
         return RegisterReader(register, reader)
 
-    reader = partial(reader, columns=[name])
+    columns = (
+        [name]
+        if register.length is None or register.length == 1
+        else [f"{name}_{i}" for i in range(register.length)]
+    )
+    reader = partial(reader, columns=columns)
     return RegisterReader(register, reader)
 
 
@@ -265,7 +271,7 @@ def create_reader(
         base_path = path / device.device if is_dir else path.parent / device.device
 
     reg_readers = {
-        name: _create_register_parser(device, name, _ReaderParams(base_path, epoch, keep_type))
+        name: _create_register_handler(device, name, _ReaderParams(base_path, epoch, keep_type))
         for name in device.registers.keys()
     }
     return DeviceReader(device, reg_readers)

harp/schema.py

@@ -8,6 +8,9 @@ from harp.model import Model, Registers
 
 
 def _read_common_registers() -> Registers:
+    if __package__ is None:
+        raise ValueError("__package__ is None: unable to read common registers")
+
     file = resources.files(__package__) / "common.yml"
     with file.open("r") as fileIO:
         return parse_yaml_raw_as(Registers, fileIO.read())

harp/typing.py

@@ -0,0 +1,13 @@
+import mmap
+import sys
+from os import PathLike
+from typing import Any, BinaryIO, Union
+
+from numpy.typing import NDArray
+
+if sys.version_info >= (3, 12):
+    from collections.abc import Buffer as _BufferLike
+else:
+    _BufferLike = Union[bytes, bytearray, memoryview, mmap.mmap, NDArray[Any]]
+
+_FileLike = Union[str, PathLike[str], BinaryIO]

{harp_python-0.3.0.dist-info → harp_python-0.4.1.dist-info}/METADATA

@@ -1,9 +1,9 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: harp-python
-Version: 0.3.0
+Version: 0.4.1
 Summary: A low-level interface for loading binary Harp protocol data
 Author-email: Goncalo Lopes <g.lopes@neurogears.org>
-License: MIT License
+License-Expression: MIT
 Project-URL: Bug Tracker, https://github.com/harp-tech/harp-python/issues
 Project-URL: Source Code, https://github.com/harp-tech/harp-python
 Classifier: Development Status :: 4 - Beta
@@ -12,7 +12,6 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Operating System :: OS Independent
-Classifier: License :: OSI Approved :: MIT License
 Requires-Python: >=3.9.0
 Description-Content-Type: text/markdown
 License-File: LICENSE
@@ -29,6 +28,7 @@ Requires-Dist: codespell; extra == "dev"
 Provides-Extra: jupyter
 Requires-Dist: ipykernel; extra == "jupyter"
 Requires-Dist: matplotlib; extra == "jupyter"
+Dynamic: license-file
 
 # harp-python
 

harp_python-0.4.1.dist-info/RECORD

@@ -0,0 +1,12 @@
+harp/__init__.py,sha256=N6fclA4pCSdBwcJELnN15p5QioU_6F01gH-8F3wr_pI,264
+harp/common.yml,sha256=qcejMFu00JjI5rkubuEDdrUu_uFkN17f_Ljb6yBd-RM,4452
+harp/io.py,sha256=js_DDHOBxNIacnHQxua0i42ljOEYZFew28zPSTXSyOs,9828
+harp/model.py,sha256=djJ7Ah6GliwDh1rXhTYoqXSDMUJ3UI9PAKFL-F6ooAg,5769
+harp/reader.py,sha256=kdrNAG53yuWMgYvlv2uAfO-8CYLREI-1PLgsE-SnE_4,8887
+harp/schema.py,sha256=9tNwjnec8WtRkV8P_oblauW7btOi_y1vWFC6Tw4EWt0,1929
+harp/typing.py,sha256=MCjnRwj-K3vybLfp_dFFqf7MwmJ7PmYe0yD91bt8uEk,355
+harp_python-0.4.1.dist-info/licenses/LICENSE,sha256=KxnuV3gNCp4WmLUC8DYWJvCKPdxMpwmWIzAYBYdqPrM,1104
+harp_python-0.4.1.dist-info/METADATA,sha256=XjRu_m2eIB9ZVN649GnNRkinWiDmszh8W8fA3p6C96U,3164
+harp_python-0.4.1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+harp_python-0.4.1.dist-info/top_level.txt,sha256=ztK1iHjgwmfXqxHCpu-e2uHIwdmodywYEUaWSODMvyM,5
+harp_python-0.4.1.dist-info/RECORD,,

{harp_python-0.3.0.dist-info → harp_python-0.4.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.5.0)
+Generator: setuptools (78.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

harp_python-0.3.0.dist-info/RECORD

@@ -1,11 +0,0 @@
-harp/__init__.py,sha256=P2PHGnd9Oz3LwrscBHKXqdYZimF7k57HcAaIaksRKc0,220
-harp/common.yml,sha256=qcejMFu00JjI5rkubuEDdrUu_uFkN17f_Ljb6yBd-RM,4452
-harp/io.py,sha256=an802KRCU-eSBXKWB7nXsomUBWcfAwEOZs9U188_MUM,4179
-harp/model.py,sha256=oJim5Gc42H4MJwpLwd2Y7KHsoeI_Yw3CO0o6qn7pJbQ,5805
-harp/reader.py,sha256=u-Taf6gmkQ5_PcsDRNskPBhZ7UsCcYL0WPlH0sEmjk0,8588
-harp/schema.py,sha256=bUr4WS-en03mzisIc9eAYiG7kReMmHdEvVwrhEpTboo,1816
-harp_python-0.3.0.dist-info/LICENSE,sha256=KxnuV3gNCp4WmLUC8DYWJvCKPdxMpwmWIzAYBYdqPrM,1104
-harp_python-0.3.0.dist-info/METADATA,sha256=lWiDbZSxbm8NmtXXopNpmmUN_h0wN0nIYc_LOrXr0s4,3190
-harp_python-0.3.0.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
-harp_python-0.3.0.dist-info/top_level.txt,sha256=ztK1iHjgwmfXqxHCpu-e2uHIwdmodywYEUaWSODMvyM,5
-harp_python-0.3.0.dist-info/RECORD,,