pyForceDAQ 2.0.5.dev2__py3-none-any.whl → 2.0.5.dev3__py3-none-any.whl

pyforcedaq/_lib/data_recorder.py

@@ -6,34 +6,24 @@ See COPYING file distributed along with the pyForceDAQ copyright and license ter
 __author__ = "Oliver Lindemann"
 
 import atexit
-import gzip
 import logging
-from io import TextIOWrapper
 from pathlib import Path
 from time import asctime, localtime, strftime
 from typing import List
 
 from .. import __version__ as forceDAQVersion
 from .. import constants
-from .clock import wait_ms
+from .file_writer import FileWriter
 from .misc import set_logging
 from .process_priority_manager import ProcessPriorityManager
 from .sensor_process import SensorProcess
 from .settings import RecordingSettings, SensorSettings
-from .types import (
-    TAG_COMMENTS,
-    TAG_DAQEVENT,
-    TAG_UDPDATA,
-    DAQEvents,
-    ForceSensorData,
-    PollingPriority,
-    UDPData,
-)
+from .types import DAQEvents, ForceSensorData, PollingPriority
 from .udp_connection import UDPConnectionProcess
 
 set_logging(data_directory="data", log_file="recording.log")
 
-NEWLINE = "\n"
+# FIXME LSL marker event for all events
 
 
 class DataRecorder(object):
@@ -61,9 +51,15 @@ class DataRecorder(object):
             force_sensor_settings = [force_sensor_settings]
 
         self.recording_settings = recording_settings
+        if recording_settings.save_data:
+            self.file_writer = FileWriter(recording_settings)
+            queue = self.file_writer.queue
+        else:
+            self.file_writer = None
+            queue = None
 
         # create sensor processes
-        self._force_sensor_processes = []
+        self.force_sensor_processes: List[SensorProcess] = []
         event_trigger = []
         for fs in force_sensor_settings:
             if not isinstance(fs, SensorSettings):
@@ -72,13 +68,12 @@ class DataRecorder(object):
                 fst = SensorProcess(
                     sensor_settings=fs,
                     recording_settings=recording_settings,
+                    file_writer_queue=queue,
                     daq_type=constants.DAQ_TYPE,
-                    use_aiftt=constants.USE_AIFTT,
-                    pipe_buffered_data_after_pause=True,
-                )
+                    use_aiftt=constants.USE_AIFTT)
                 fst.start()
                 event_trigger.append(fst.event_trigger)
-                self._force_sensor_processes.append(fst)
+                self.force_sensor_processes.append(fst)
 
         # create udp connection process
         if poll_udp_connection:
@@ -90,7 +85,7 @@ class DataRecorder(object):
         # process managing FIYME needed?
         self._proc_manager = ProcessPriorityManager()
         self._proc_manager.add_subprocess(self.udp)
-        self._proc_manager.add_subprocess(self._force_sensor_processes)
+        self._proc_manager.add_subprocess(self.force_sensor_processes)
         if self.recording_settings.priority is not None:
             level = PollingPriority.NORMAL
         else:
@@ -102,37 +97,36 @@ class DataRecorder(object):
         )
         # logging.info("Subprocess priorities: {}".format(self._proc_manager.get_subprocess_priorities()))
 
-        self._is_recording = False
-        self._file = None
-        self._daq_event = []
-        self.path_open_file = Path("")
+        self._daq_event = [] # FIXME needed?
         atexit.register(self.quit)
 
     @property
-    def is_saving_data(self):
+    def has_file_writer(self):
         """Property indicates whether a data file is open"""
-        return self._file is not None
+        return isinstance(self.file_writer, FileWriter) and self.file_writer.is_alive()
 
     @property
     def is_alive(self):
         """Property indicates whether the recording processes are alive"""
         try:
-            return self._force_sensor_processes[0].is_alive()
-        except Exception:
+            return self.force_sensor_processes[0].is_alive()
+        except IndexError:
             return False
 
     @property
-    def is_recording(self):
+    def is_saving(self):
         """Property indicates whether the recording is started or paused"""
-        return self._is_recording
-
-    @property
-    def force_sensor_processes(self):
-        return self._force_sensor_processes
+        if  self.has_file_writer:
+            for fsp in self.force_sensor_processes:
+                if not fsp.is_saving:
+                    return False
+            return True # all sensor processes are saving, file writer is alive
+        else:
+            return False
 
     @property
     def sensor_settings_list(self):
-        return list(map(lambda x: x.sensor_settings, self._force_sensor_processes))
+        return list(map(lambda x: x.sensor_settings, self.force_sensor_processes))
 
     def quit(self) -> list | None:
         """Stop all recording processes, close data file and quit recording
@@ -146,20 +140,16 @@ class DataRecorder(object):
         if not self.is_alive:
             return
 
-        buffer = self.pause_recording()
+        self.pause_saving()
+        for fsp in self.force_sensor_processes:
+            fsp.join()
         self.close_data_file()
 
         if self.udp is not None:
             self.udp.quit()
 
-        # wait that all processes are quitted
-        for fsp in self._force_sensor_processes:
-            fsp.join()
-
         logging.info("Quit recording")
 
-        return buffer
-
     def process_and_write_udp_events(self) -> list:
         """process udp events and return them"""
         buffer = []
@@ -170,59 +160,11 @@ class DataRecorder(object):
                 # until queue empty or no udp connection
                 break
             buffer.append(data)
-        if len(buffer) > 0:
-            self._write_data(buffer)
-        return buffer
 
-    def _write_data(
-        self, data_buffer: list, recording_screen=None, float_decimal_places: int = 4
-    ) -> None:
-        """writes data to disk and set counters
-
-        ignores UDP remote control commands
-        """
-        # DOC output format
-
-        BLOCKSIZE = 10000  # for recording screen feedback only
-        write_forces = self.recording_settings.array_write_forces()
-        write_trigger = self.recording_settings.array_write_trigger()
-        write_deviceid = len(self.recording_settings.device_labels) > 1
-        float_format = "{0:." + str(float_decimal_places) + "f},"
-        buffer_len = len(data_buffer)
-        for c, d in enumerate(data_buffer):
-            if self._file is not None:
-                if isinstance(d, ForceSensorData):
-                    line = f"{d.time}, {d.acquisition_delay},"
-                    if write_deviceid:
-                        line += f"{d.sensor_id},"
-                    for x in d.forces[write_forces]:
-                        line += float_format.format(x)
-                    for x in d.trigger[write_trigger]:
-                        if isinstance(x, int):
-                            line += f"{x},"
-                        else:
-                            line += float_format.format(x)
-                    self._file_write(line[:-1] + NEWLINE)
-
-                elif isinstance(d, DAQEvents):
-                    self._file_write(f"{TAG_DAQEVENT},{d.time},{str(d.code)}" + NEWLINE)
-
-                elif isinstance(d, UDPData):
-                    self._file_write(f"{TAG_UDPDATA},{d.time},{d.unicode}" + NEWLINE)
-
-            if recording_screen is not None and c % BLOCKSIZE == 0:
-                recording_screen.stimulus(
-                    "Saving {0} of {1} blocks".format(
-                        c // BLOCKSIZE, buffer_len // BLOCKSIZE
-                    )
-                ).present()
-
-    def _file_write(self, s: str) -> None:
-
-        if isinstance(self._file, gzip.GzipFile):
-            self._file.write(s.encode("utf-8"))
-        elif isinstance(self._file, TextIOWrapper):
-            self._file.write(s)
+        if isinstance(self.file_writer, FileWriter):
+            for dat in buffer:
+                self.file_writer.queue.put(dat)
+        return buffer
 
     def store_daq_event(
         self, code: str | int | float, time: float | None = None
@@ -234,35 +176,20 @@ class DataRecorder(object):
         """
         self._daq_event.append(DAQEvents(time=time, code=code))
 
-    def start_recording(self, determine_bias: bool = False) -> None:
+    def start_saving(self) -> None:
         """Start polling process and record
 
         See Also
         --------
-        is_recording
+        is_saving
 
         """
 
-        if determine_bias:
-            self.determine_biases(n_samples=1000)
-
-        if len(
-            list(
-                filter(
-                    lambda x: x.event_bias_is_available.is_set(),
-                    self._force_sensor_processes,
-                )
-            )
-        ) != len(self._force_sensor_processes):
-            raise RuntimeError(
-                "Sensors can't be started before bias has been determined."
-            )
+        for fsp in self.force_sensor_processes:
+            fsp.flag_sensor_bias_is_determined.wait() # wait is no initial bias is set yet
+            fsp.start_saving()
 
-        # start polling
-        list(map(lambda x: x.start_polling(), self._force_sensor_processes))
-        self._is_recording = True
-
-    def pause_recording(self, recording_screen=None) -> list:
+    def pause_saving(self):
         """Pauses all polling processes and process data
 
         returns
@@ -270,59 +197,17 @@ class DataRecorder(object):
         data : all last data
 
         """
-        self._is_recording = False
-
-        data = []
-        if recording_screen is not None:
-            recording_screen.stimulus("").present()
 
         # pause polling
-        for fsp in self._force_sensor_processes:
-            fsp.pause_polling()
-
-        wait_ms(500)
-
-        if recording_screen is not None:
-            if self.is_saving_data:
-                recording_screen.stimulus("saving data ...").present()
-            else:
-                recording_screen.stimulus("pause recording").present()
+        for fsp in self.force_sensor_processes:
+            fsp.pause_saving()
 
-        # get data
-        for fsp in self._force_sensor_processes:
-            buffer = fsp.get_buffer()
-            self._write_data(buffer, recording_screen)
-            data.extend(buffer)
 
-        # udp event
-        data.extend(self.process_and_write_udp_events())
-
-        # soft trigger
-        self._write_data(self._daq_event)
-        data.extend(self._daq_event)
-        self._daq_event = []
-        return data
-
-    def determine_biases(self, n_samples: int) -> None:
-        """Record n data samples (n_samples) to determine bias.
-        Afterwards recording is in pause mode
-
-        Notes
-        -----
-        The function take some time to be processed
-
-        See Also
-        --------
-        Sensor.determine_bias()
-
-        """
-
-        self.pause_recording()
-        for x in self._force_sensor_processes:
-            x.determine_bias(n_samples=n_samples)
-
-        for x in self._force_sensor_processes:
-            x.event_bias_is_available.wait()
+    def determine_biases(self) -> None:
+        for x in self.force_sensor_processes:
+            x.determine_bias()
+        for x in self.force_sensor_processes:
+            x.flag_sensor_bias_is_determined.wait()
 
     def open_data_file(
         self,
@@ -330,7 +215,7 @@ class DataRecorder(object):
         subdirectory: str = "data",
         varnames: bool = True,
         comment_line: str = "",
-    ) -> Path:
+    ) -> Path | None:
         """Create a data file
 
         Only if data file has been opened, data will be saved!
@@ -355,20 +240,22 @@ class DataRecorder(object):
                 full path the actually used file (incl. timestamp)
 
         """
-        self.close_data_file()
+
+        if not isinstance(self.file_writer, FileWriter):
+            return
 
         # create filename
         data_dir = Path.cwd() / subdirectory
         data_dir.mkdir(exist_ok=True)
+
         if self.recording_settings.zip_data:
-            filename = Path(filename).with_suffix(".csv.gz")
+            filename = Path(filename).with_suffix(".csv.bz2")
         else:
             filename = Path(filename).with_suffix(".csv")
-
         while True:
-            self.path_open_file = data_dir / filename
-            if self.path_open_file.is_file():
-                # print "data file already exists, adding counter"
+            file_path = data_dir / filename
+            if file_path.is_file():
+                #get unique filename by adding timestamp if file already exists
                 filename = Path(
                     filename.stem
                     + "_"
@@ -378,32 +265,24 @@ class DataRecorder(object):
             else:
                 break
 
-        if self.recording_settings.zip_data:
-            self._file = gzip.open(self.path_open_file, "w")
-        else:
-            self._file = open(self.path_open_file, "w")
-        print("Data file: {}".format(self.path_open_file))
-
-        self._file_write(
-            TAG_COMMENTS
-            + "Recorded at {0} with pyForceDAQ {1}\n".format(
-                asctime(localtime()), forceDAQVersion
-            )
-        )
-        logging.info("new file: {}".format(self.path_open_file))
+        self.file_writer.start_recording(file_path=file_path, append_mode=False)
+        logging.info("new file: %s", file_path)
+
+        self.file_writer.queue.put(
+            f"Recorded at {asctime(localtime())} with pyForceDAQ {forceDAQVersion}\n")
 
         for s in self.sensor_settings_list:
             txt = f" Sensor: label={s.device_label}, cal-file={s.calibration_file}\n"
-            self._file_write(TAG_COMMENTS + txt)
+            self.file_writer.queue.put(txt)
 
         if len(comment_line) > 0:
-            self._file_write(TAG_COMMENTS + comment_line + "\n")
+            self.file_writer.queue.put(comment_line + "\n")
 
         if varnames:
             write_forces = self.recording_settings.array_write_forces()
             write_trigger = self.recording_settings.array_write_trigger()
             write_deviceid = len(self.recording_settings.device_labels) > 1
-            line = "time,delay,"
+            line = "time,"
             if write_deviceid:
                 line += "device_tag,"
             for x in range(6):
@@ -413,9 +292,9 @@ class DataRecorder(object):
                 line += "trigger1,"
             if write_trigger[1]:
                 line += "trigger2,"
-            self._file_write(line[:-1] + NEWLINE)
+            self.file_writer.queue.put(line[:-1] + "\n")
 
-        return self.path_open_file
+        return file_path
 
     def close_data_file(self) -> None:
         """Close the data file
@@ -423,8 +302,8 @@ class DataRecorder(object):
         Afterwards data will not be saved anymore.
 
         """
-
-        if self._file is not None:
-            self._file.close()
-            self._file = None
-            self.path_open_file = Path("")
+        if isinstance(self.file_writer, FileWriter):
+            self.file_writer.close_file()
+            if self.file_writer.is_alive():
+                self.file_writer.join()
+            self.file_writer.filepath = Path("")

pyforcedaq/_lib/file_writer.py

@@ -0,0 +1,110 @@
+import bz2
+from multiprocessing import Event, Process, Queue
+from pathlib import Path
+from queue import Empty
+
+from .settings import RecordingSettings
+from .types import (TAG_COMMENTS, TAG_DAQEVENT, TAG_UDPDATA, DAQEvents,
+                    ForceSensorData, UDPData)
+
+NEWLINE = "\n"
+ENCODING = "utf-8"
+
+class FileWriter(Process):
+    def __init__(
+        self, recording_settings: RecordingSettings, float_decimal_places: int = 4
+    ):
+        """To write to a file from multiple processes. Use FileWriter.queue.put(str) to write file"""
+
+        super().__init__()
+        self.filepath: Path  = Path("")
+        self.append_mode = False
+        self.queue = Queue()
+        self._force_quit = Event()
+        self._close_file = Event()
+        self._write_forces = recording_settings.array_write_forces()
+        self._write_trigger = recording_settings.array_write_trigger()
+        self._write_deviceid = len(recording_settings.device_labels) > 1
+        self._decimal_places = float_decimal_places
+
+    def close_file(self):
+        """closes file after all pending writes are done and no further write occurred
+        for close_timeout seconds
+        """
+        self._close_file.set()
+
+    def force_quit(self):
+        """forces the process to quit immediately, even if there are pending writes in the queue"""
+        self._force_quit.set()
+
+    def start_recording(self, file_path: Path, append_mode: bool = False):
+        """opens file for writing, if file already exists, it will be overwritten (or appended if append_mode is True)"""
+        self.filepath = file_path
+        self.append_mode = append_mode
+        self.start()
+
+    # def join(self, timeout=None):
+    #    super(FileWriter, self).join(timeout)
+
+    def run(self):
+
+        if self.filepath is None:
+            raise ValueError("File path is not set. Call start_recording() with a valid file path before running the process.")
+
+        float_format = "{0:." + str(self._decimal_places) + "f},"
+        if self.append_mode:
+            mode = "a"
+        else:
+            mode = "w"
+        if self.filepath.suffix.endswith("bz2"):
+            fl = bz2.open(self.filepath, mode)
+        else:
+            fl = open(self.filepath, mode, encoding=ENCODING)
+
+        self._close_file.clear()
+        self._force_quit.clear()
+
+        while not self._force_quit.is_set():
+
+            if self._close_file.is_set():
+                try:
+                    d = self.queue.get_nowait()
+                except Empty:
+                    break  # quit process
+            else:
+                try:
+                    d = self.queue.get(timeout=0.5)
+                except Empty:
+                    continue  # wait again for events
+
+            if isinstance(d, ForceSensorData):
+                txt = f"{d.time},"
+                if self._write_deviceid:
+                    txt += f"{d.sensor_id},"
+                for x in d.forces[self._write_forces]:
+                    txt += float_format.format(x)
+                for x in d.trigger[self._write_trigger]:
+                    if isinstance(x, int):
+                        txt += f"{x},"
+                    else:
+                        txt += float_format.format(x)
+                txt = txt[:-1] + NEWLINE
+
+            elif isinstance(d, DAQEvents):
+                txt = f"{TAG_DAQEVENT},{d.time},{str(d.code)}{NEWLINE}"
+
+            elif isinstance(d, UDPData):
+                txt = f"{TAG_UDPDATA},{d.time},{d.unicode}{NEWLINE}"
+            elif isinstance(d, str):
+                txt = f"{TAG_COMMENTS} {d}"
+            else:
+                continue  # ignore unknown data types or maybe raise error (TODO)
+
+            if isinstance(fl, bz2.BZ2File):
+                fl.write(txt.encode(ENCODING))
+            else:
+                fl.write(txt)
+
+        fl.flush()
+        fl.close()
+

pyforcedaq/_lib/sensor.py

@@ -6,6 +6,7 @@ See COPYING file distributed along with the pyForceDAQ copyright and license ter
 __author__ = "Oliver Lindemann"
 
 import numpy as np
+from numpy import typing as npt
 
 from .. import constants
 from .clock import local_clock
@@ -67,6 +68,15 @@ class Sensor(object):
                     continue
                 self._reverse_vector[idx] = -1
 
+    def set_bias(self, data: npt.NDArray):
+        """sets the bias"""
+
+        if np.shape(data)[1] != len(Sensor.SENSOR_CHANNELS):
+            raise ValueError(f"biasdata should have the shape (x, n_sensor_channels)")
+
+        if self._calib_converter is not None:
+            self._calib_converter.bias(np.mean(data, axis=0))
+
     def determine_bias(self, n_samples=100):
         """determines the bias"""
 
@@ -102,8 +112,8 @@ class Sensor(object):
 
         """
 
-        start = local_clock()
         data, _read_samples = self.daq.read_analog()
+        t = local_clock()
         if self.convert_to_FT and self._calib_converter is not None:
             forces = np.asarray(
                 self._calib_converter.convertToFT(voltages=data[Sensor.SENSOR_CHANNELS])
@@ -114,12 +124,10 @@ class Sensor(object):
 
         # reverse scaling if needed
         forces = forces * self._reverse_vector
-        t = local_clock()
 
         return ForceSensorData(
             time=t,
-            acquisition_delay=t - start,
             sensor_id=self.sensor_id,
             forces=forces,
-            trigger=data[Sensor.TRIGGER_CHANNELS],
+            trigger=data[Sensor.TRIGGER_CHANNELS]
         )