pyForceDAQ 2.0.4__tar.gz → 2.0.5.dev3__tar.gz

{pyforcedaq-2.0.4 → pyforcedaq-2.0.5.dev3}/PKG-INFO

@@ -1,18 +1,15 @@
 Metadata-Version: 2.3
 Name: pyForceDAQ
-Version: 2.0.4
+Version: 2.0.5.dev3
 Summary: A Python package for data acquisition and analysis in force-based experiments.
 Author: Oliver Lindemann
 Author-email: Oliver Lindemann <lindemann@essb.eur.nl>
 Requires-Dist: atiiaftt>=0.1.1
 Requires-Dist: expyriment>=1.0.1
-Requires-Dist: icecream>=2.2.0
 Requires-Dist: nidaqmx>=1.5.0
 Requires-Dist: numpy>=2.4.4
 Requires-Dist: psutil>=7.2.2
-Requires-Dist: pydaqmx>=1.4.7
 Requires-Dist: pylsl>=1.18.2
 Requires-Dist: pysimplegui>=6.0
-Requires-Dist: pyxdf>=1.17.4
 Requires-Dist: tomlkit>=0.15.0
 Requires-Python: >=3.12, <3.14

{pyforcedaq-2.0.4 → pyforcedaq-2.0.5.dev3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pyForceDAQ"
-version = "2.0.4"
+version = "2.0.5-dev3"
 description = "A Python package for data acquisition and analysis in force-based experiments."
 authors = [
     { name = "Oliver Lindemann", email = "lindemann@essb.eur.nl" },
@@ -9,14 +9,11 @@ requires-python = ">=3.12, <3.14"
 dependencies = [
     "atiiaftt>=0.1.1",
     "expyriment>=1.0.1",
-    "icecream>=2.2.0",
     "nidaqmx>=1.5.0",
     "numpy>=2.4.4",
     "psutil>=7.2.2",
-    "pydaqmx>=1.4.7",
     "pylsl>=1.18.2",
     "pysimplegui>=6.0",
-    "pyxdf>=1.17.4",
     "tomlkit>=0.15.0",
 ]
 
@@ -27,5 +24,25 @@ dependencies = [
 requires = ["uv_build>=0.11.15,<0.12.0"]
 build-backend = "uv_build"
 
+[dependency-groups]
+dev = [
+    "icecream>=2.2.0",
+    "ipython>=9.14.0",
+    "ruff>=0.15.15",
+]
+pydaqmx = [
+    "pydaqmx>=1.4.7",
+]
+test = [
+    "pytest>=9.0.3",
+]
+
 [project.scripts]
 pyforcedaq = "pyforcedaq.__main__:cli"
+
+[tool.ruff.lint]
+ignore = ["F401", "F405"]
+
+[tool.ruff.format]
+line-ending = "lf"
+indent-style = "space"

{pyforcedaq-2.0.4 → pyforcedaq-2.0.5.dev3}/src/pyforcedaq/__main__.py

@@ -1,8 +1,13 @@
 import argparse
 
-from . import __author__, __version__
+from . import __author__, __version__, constants
 
 
+def print_version():
+    print("+" + "-" * 23 + "+")
+    print(f"| pyforceDAQ {__version__}".ljust(24) + "|")
+    print("+" + "-" * 23 + "+")
+
 def cli():
 
     parser = argparse.ArgumentParser(
@@ -31,12 +36,29 @@ def cli():
         default=False,
         help="Omit launcher GUI and start recording directly",
     )
-    print("+" + "-" * 23 + "+")
-    print(f"| pyforceDAQ {__version__}".ljust(24) + "|")
-    print("+" + "-" * 23 + "+")
+
+    parser.add_argument(
+        "--mock",
+        action="store_true",
+        default=False,
+        help="Use mock sensor",
+    )
+
+    parser.add_argument(
+        "--dll",
+        action="store_true",
+        default=False,
+        help="Use self compiled ATI DLL",
+    )
 
     args = parser.parse_args()
+    if args.mock:
+        constants.DAQ_TYPE = constants.MOCK_SENSOR
+    else:
+        constants.DAQ_TYPE = constants.NIDAQMX # use NI-DAQmx
+    constants.USE_AIFTT = not args.dll
 
+    print_version()
     if not args.omit_launcher:
         if len(args.SETTINGS_FILE) > 0:
             print("Can't use launcher and settings file")
@@ -53,3 +75,4 @@ def cli():
 
 if __name__ == "__main__":  # required because of threading
     cli()
+    cli()

{pyforcedaq-2.0.4 → pyforcedaq-2.0.5.dev3}/src/pyforcedaq/_lib/data_recorder.py

@@ -6,33 +6,24 @@ See COPYING file distributed along with the pyForceDAQ copyright and license ter
 __author__ = "Oliver Lindemann"
 
 import atexit
-import gzip
 import logging
-from io import TextIOWrapper
 from pathlib import Path
 from time import asctime, localtime, strftime
 from typing import List
 
 from .. import __version__ as forceDAQVersion
-from .clock import wait_ms
+from .. import constants
+from .file_writer import FileWriter
 from .misc import set_logging
 from .process_priority_manager import ProcessPriorityManager
 from .sensor_process import SensorProcess
 from .settings import RecordingSettings, SensorSettings
-from .types import (
-    TAG_COMMENTS,
-    TAG_DAQEVENT,
-    TAG_UDPDATA,
-    DAQEvents,
-    ForceSensorData,
-    PollingPriority,
-    UDPData,
-)
+from .types import DAQEvents, ForceSensorData, PollingPriority
 from .udp_connection import UDPConnectionProcess
 
 set_logging(data_directory="data", log_file="recording.log")
 
-NEWLINE = "\n"
+# FIXME LSL marker event for all events
 
 
 class DataRecorder(object):
@@ -49,15 +40,26 @@ class DataRecorder(object):
 
         polling_priority has to be types.PollingPriority.{HIGH},
         {REALTIME} or {NORMAL} or None
+
+        You can change the used modules by settings the following constants before creating the
+        DataRecorder instance:
+            * set constants.DAQ_TYPE to constants.PYDAQMX, constants.NIDAQMX or constants.MOCK_SENSOR
+            * set constants.USE_AIFTT to True or False
         """
 
         if not isinstance(force_sensor_settings, list):
             force_sensor_settings = [force_sensor_settings]
 
         self.recording_settings = recording_settings
+        if recording_settings.save_data:
+            self.file_writer = FileWriter(recording_settings)
+            queue = self.file_writer.queue
+        else:
+            self.file_writer = None
+            queue = None
 
         # create sensor processes
-        self._force_sensor_processes = []
+        self.force_sensor_processes: List[SensorProcess] = []
         event_trigger = []
         for fs in force_sensor_settings:
             if not isinstance(fs, SensorSettings):
@@ -66,11 +68,12 @@ class DataRecorder(object):
                 fst = SensorProcess(
                     sensor_settings=fs,
                     recording_settings=recording_settings,
-                    pipe_buffered_data_after_pause=True,
-                )
+                    file_writer_queue=queue,
+                    daq_type=constants.DAQ_TYPE,
+                    use_aiftt=constants.USE_AIFTT)
                 fst.start()
                 event_trigger.append(fst.event_trigger)
-                self._force_sensor_processes.append(fst)
+                self.force_sensor_processes.append(fst)
 
         # create udp connection process
         if poll_udp_connection:
@@ -82,7 +85,7 @@ class DataRecorder(object):
         # process managing FIYME needed?
         self._proc_manager = ProcessPriorityManager()
         self._proc_manager.add_subprocess(self.udp)
-        self._proc_manager.add_subprocess(self._force_sensor_processes)
+        self._proc_manager.add_subprocess(self.force_sensor_processes)
         if self.recording_settings.priority is not None:
             level = PollingPriority.NORMAL
         else:
@@ -94,37 +97,36 @@ class DataRecorder(object):
         )
         # logging.info("Subprocess priorities: {}".format(self._proc_manager.get_subprocess_priorities()))
 
-        self._is_recording = False
-        self._file = None
-        self._daq_event = []
-        self.path_open_file = Path("")
+        self._daq_event = [] # FIXME needed?
         atexit.register(self.quit)
 
     @property
-    def is_saving_data(self):
+    def has_file_writer(self):
         """Property indicates whether a data file is open"""
-        return self._file is not None
+        return isinstance(self.file_writer, FileWriter) and self.file_writer.is_alive()
 
     @property
     def is_alive(self):
         """Property indicates whether the recording processes are alive"""
         try:
-            return self._force_sensor_processes[0].is_alive()
-        except Exception:
+            return self.force_sensor_processes[0].is_alive()
+        except IndexError:
             return False
 
     @property
-    def is_recording(self):
+    def is_saving(self):
         """Property indicates whether the recording is started or paused"""
-        return self._is_recording
-
-    @property
-    def force_sensor_processes(self):
-        return self._force_sensor_processes
+        if  self.has_file_writer:
+            for fsp in self.force_sensor_processes:
+                if not fsp.is_saving:
+                    return False
+            return True # all sensor processes are saving, file writer is alive
+        else:
+            return False
 
     @property
     def sensor_settings_list(self):
-        return list(map(lambda x: x.sensor_settings, self._force_sensor_processes))
+        return list(map(lambda x: x.sensor_settings, self.force_sensor_processes))
 
     def quit(self) -> list | None:
         """Stop all recording processes, close data file and quit recording
@@ -138,20 +140,16 @@ class DataRecorder(object):
         if not self.is_alive:
             return
 
-        buffer = self.pause_recording()
+        self.pause_saving()
+        for fsp in self.force_sensor_processes:
+            fsp.join()
         self.close_data_file()
 
         if self.udp is not None:
             self.udp.quit()
 
-        # wait that all processes are quitted
-        for fsp in self._force_sensor_processes:
-            fsp.join()
-
         logging.info("Quit recording")
 
-        return buffer
-
     def process_and_write_udp_events(self) -> list:
         """process udp events and return them"""
         buffer = []
@@ -162,59 +160,11 @@ class DataRecorder(object):
                 # until queue empty or no udp connection
                 break
             buffer.append(data)
-        if len(buffer) > 0:
-            self._write_data(buffer)
-        return buffer
-
-    def _write_data(
-        self, data_buffer: list, recording_screen=None, float_decimal_places: int = 4
-    ) -> None:
-        """writes data to disk and set counters
 
-        ignores UDP remote control commands
-        """
-        # DOC output format
-
-        BLOCKSIZE = 10000  # for recording screen feedback only
-        write_forces = self.recording_settings.array_write_forces()
-        write_trigger = self.recording_settings.array_write_trigger()
-        write_deviceid = len(self.recording_settings.device_labels) > 1
-        float_format = "{0:." + str(float_decimal_places) + "f},"
-        buffer_len = len(data_buffer)
-        for c, d in enumerate(data_buffer):
-            if self._file is not None:
-                if isinstance(d, ForceSensorData):
-                    line = f"{d.time}, {d.acquisition_delay},"
-                    if write_deviceid:
-                        line += f"{d.sensor_id},"
-                    for x in d.forces[write_forces]:
-                        line += float_format.format(x)
-                    for x in d.trigger[write_trigger]:
-                        if isinstance(x, int):
-                            line += f"{x},"
-                        else:
-                            line += float_format.format(x)
-                    self._file_write(line[:-1] + NEWLINE)
-
-                elif isinstance(d, DAQEvents):
-                    self._file_write(f"{TAG_DAQEVENT},{d.time},{str(d.code)}" + NEWLINE)
-
-                elif isinstance(d, UDPData):
-                    self._file_write(f"{TAG_UDPDATA},{d.time},{d.unicode}" + NEWLINE)
-
-            if recording_screen is not None and c % BLOCKSIZE == 0:
-                recording_screen.stimulus(
-                    "Saving {0} of {1} blocks".format(
-                        c // BLOCKSIZE, buffer_len // BLOCKSIZE
-                    )
-                ).present()
-
-    def _file_write(self, s: str) -> None:
-
-        if isinstance(self._file, gzip.GzipFile):
-            self._file.write(s.encode("utf-8"))
-        elif isinstance(self._file, TextIOWrapper):
-            self._file.write(s)
+        if isinstance(self.file_writer, FileWriter):
+            for dat in buffer:
+                self.file_writer.queue.put(dat)
+        return buffer
 
     def store_daq_event(
         self, code: str | int | float, time: float | None = None
@@ -226,35 +176,20 @@ class DataRecorder(object):
         """
         self._daq_event.append(DAQEvents(time=time, code=code))
 
-    def start_recording(self, determine_bias: bool = False) -> None:
+    def start_saving(self) -> None:
         """Start polling process and record
 
         See Also
         --------
-        is_recording
+        is_saving
 
         """
 
-        if determine_bias:
-            self.determine_biases(n_samples=1000)
+        for fsp in self.force_sensor_processes:
+            fsp.flag_sensor_bias_is_determined.wait() # wait is no initial bias is set yet
+            fsp.start_saving()
 
-        if len(
-            list(
-                filter(
-                    lambda x: x.event_bias_is_available.is_set(),
-                    self._force_sensor_processes,
-                )
-            )
-        ) != len(self._force_sensor_processes):
-            raise RuntimeError(
-                "Sensors can't be started before bias has been determined."
-            )
-
-        # start polling
-        list(map(lambda x: x.start_polling(), self._force_sensor_processes))
-        self._is_recording = True
-
-    def pause_recording(self, recording_screen=None) -> list:
+    def pause_saving(self):
         """Pauses all polling processes and process data
 
         returns
@@ -262,59 +197,17 @@ class DataRecorder(object):
         data : all last data
 
         """
-        self._is_recording = False
-
-        data = []
-        if recording_screen is not None:
-            recording_screen.stimulus("").present()
 
         # pause polling
-        for fsp in self._force_sensor_processes:
-            fsp.pause_polling()
-
-        wait_ms(500)
-
-        if recording_screen is not None:
-            if self.is_saving_data:
-                recording_screen.stimulus("saving data ...").present()
-            else:
-                recording_screen.stimulus("pause recording").present()
-
-        # get data
-        for fsp in self._force_sensor_processes:
-            buffer = fsp.get_buffer()
-            self._write_data(buffer, recording_screen)
-            data.extend(buffer)
-
-        # udp event
-        data.extend(self.process_and_write_udp_events())
-
-        # soft trigger
-        self._write_data(self._daq_event)
-        data.extend(self._daq_event)
-        self._daq_event = []
-        return data
-
-    def determine_biases(self, n_samples: int) -> None:
-        """Record n data samples (n_samples) to determine bias.
-        Afterwards recording is in pause mode
-
-        Notes
-        -----
-        The function take some time to be processed
-
-        See Also
-        --------
-        Sensor.determine_bias()
+        for fsp in self.force_sensor_processes:
+            fsp.pause_saving()
 
-        """
 
-        self.pause_recording()
-        for x in self._force_sensor_processes:
-            x.determine_bias(n_samples=n_samples)
-
-        for x in self._force_sensor_processes:
-            x.event_bias_is_available.wait()
+    def determine_biases(self) -> None:
+        for x in self.force_sensor_processes:
+            x.determine_bias()
+        for x in self.force_sensor_processes:
+            x.flag_sensor_bias_is_determined.wait()
 
     def open_data_file(
         self,
@@ -322,7 +215,7 @@ class DataRecorder(object):
         subdirectory: str = "data",
         varnames: bool = True,
         comment_line: str = "",
-    ) -> Path:
+    ) -> Path | None:
         """Create a data file
 
         Only if data file has been opened, data will be saved!
@@ -347,20 +240,22 @@ class DataRecorder(object):
                 full path the actually used file (incl. timestamp)
 
         """
-        self.close_data_file()
+
+        if not isinstance(self.file_writer, FileWriter):
+            return
 
         # create filename
         data_dir = Path.cwd() / subdirectory
         data_dir.mkdir(exist_ok=True)
+
         if self.recording_settings.zip_data:
-            filename = Path(filename).with_suffix(".csv.gz")
+            filename = Path(filename).with_suffix(".csv.bz2")
         else:
             filename = Path(filename).with_suffix(".csv")
-
         while True:
-            self.path_open_file = data_dir / filename
-            if self.path_open_file.is_file():
-                # print "data file already exists, adding counter"
+            file_path = data_dir / filename
+            if file_path.is_file():
+                #get unique filename by adding timestamp if file already exists
                 filename = Path(
                     filename.stem
                     + "_"
@@ -370,32 +265,24 @@ class DataRecorder(object):
             else:
                 break
 
-        if self.recording_settings.zip_data:
-            self._file = gzip.open(self.path_open_file, "w")
-        else:
-            self._file = open(self.path_open_file, "w")
-        print("Data file: {}".format(self.path_open_file))
-
-        self._file_write(
-            TAG_COMMENTS
-            + "Recorded at {0} with pyForceDAQ {1}\n".format(
-                asctime(localtime()), forceDAQVersion
-            )
-        )
-        logging.info("new file: {}".format(self.path_open_file))
+        self.file_writer.start_recording(file_path=file_path, append_mode=False)
+        logging.info("new file: %s", file_path)
+
+        self.file_writer.queue.put(
+            f"Recorded at {asctime(localtime())} with pyForceDAQ {forceDAQVersion}\n")
 
         for s in self.sensor_settings_list:
             txt = f" Sensor: label={s.device_label}, cal-file={s.calibration_file}\n"
-            self._file_write(TAG_COMMENTS + txt)
+            self.file_writer.queue.put(txt)
 
         if len(comment_line) > 0:
-            self._file_write(TAG_COMMENTS + comment_line + "\n")
+            self.file_writer.queue.put(comment_line + "\n")
 
         if varnames:
             write_forces = self.recording_settings.array_write_forces()
             write_trigger = self.recording_settings.array_write_trigger()
             write_deviceid = len(self.recording_settings.device_labels) > 1
-            line = "time,delay,"
+            line = "time,"
             if write_deviceid:
                 line += "device_tag,"
             for x in range(6):
@@ -405,9 +292,9 @@ class DataRecorder(object):
                 line += "trigger1,"
             if write_trigger[1]:
                 line += "trigger2,"
-            self._file_write(line[:-1] + NEWLINE)
+            self.file_writer.queue.put(line[:-1] + "\n")
 
-        return self.path_open_file
+        return file_path
 
     def close_data_file(self) -> None:
         """Close the data file
@@ -415,8 +302,8 @@ class DataRecorder(object):
         Afterwards data will not be saved anymore.
 
         """
-
-        if self._file is not None:
-            self._file.close()
-            self._file = None
-            self.path_open_file = Path("")
+        if isinstance(self.file_writer, FileWriter):
+            self.file_writer.close_file()
+            if self.file_writer.is_alive():
+                self.file_writer.join()
+            self.file_writer.filepath = Path("")

pyforcedaq-2.0.5.dev3/src/pyforcedaq/_lib/file_writer.py

@@ -0,0 +1,110 @@
+import bz2
+from multiprocessing import Event, Process, Queue
+from pathlib import Path
+from queue import Empty
+
+from .settings import RecordingSettings
+from .types import (TAG_COMMENTS, TAG_DAQEVENT, TAG_UDPDATA, DAQEvents,
+                    ForceSensorData, UDPData)
+
+NEWLINE = "\n"
+ENCODING = "utf-8"
+
+class FileWriter(Process):
+    def __init__(
+        self, recording_settings: RecordingSettings, float_decimal_places: int = 4
+    ):
+        """To write to a file from multiple processes. Use FileWriter.queue.put(str) to write file"""
+
+        super().__init__()
+        self.filepath: Path  = Path("")
+        self.append_mode = False
+        self.queue = Queue()
+        self._force_quit = Event()
+        self._close_file = Event()
+        self._write_forces = recording_settings.array_write_forces()
+        self._write_trigger = recording_settings.array_write_trigger()
+        self._write_deviceid = len(recording_settings.device_labels) > 1
+        self._decimal_places = float_decimal_places
+
+    def close_file(self):
+        """closes file after all pending writes are done and no further write occurred
+        for close_timeout seconds
+        """
+        self._close_file.set()
+
+    def force_quit(self):
+        """forces the process to quit immediately, even if there are pending writes in the queue"""
+        self._force_quit.set()
+
+    def start_recording(self, file_path: Path, append_mode: bool = False):
+        """opens file for writing, if file already exists, it will be overwritten (or appended if append_mode is True)"""
+        self.filepath = file_path
+        self.append_mode = append_mode
+        self.start()
+
+    # def join(self, timeout=None):
+    #    super(FileWriter, self).join(timeout)
+
+    def run(self):
+
+        if self.filepath is None:
+            raise ValueError("File path is not set. Call start_recording() with a valid file path before running the process.")
+
+        float_format = "{0:." + str(self._decimal_places) + "f},"
+        if self.append_mode:
+            mode = "a"
+        else:
+            mode = "w"
+        if self.filepath.suffix.endswith("bz2"):
+            fl = bz2.open(self.filepath, mode)
+        else:
+            fl = open(self.filepath, mode, encoding=ENCODING)
+
+        self._close_file.clear()
+        self._force_quit.clear()
+
+        while not self._force_quit.is_set():
+
+            if self._close_file.is_set():
+                try:
+                    d = self.queue.get_nowait()
+                except Empty:
+                    break  # quit process
+            else:
+                try:
+                    d = self.queue.get(timeout=0.5)
+                except Empty:
+                    continue  # wait again for events
+
+            if isinstance(d, ForceSensorData):
+                txt = f"{d.time},"
+                if self._write_deviceid:
+                    txt += f"{d.sensor_id},"
+                for x in d.forces[self._write_forces]:
+                    txt += float_format.format(x)
+                for x in d.trigger[self._write_trigger]:
+                    if isinstance(x, int):
+                        txt += f"{x},"
+                    else:
+                        txt += float_format.format(x)
+                txt = txt[:-1] + NEWLINE
+
+            elif isinstance(d, DAQEvents):
+                txt = f"{TAG_DAQEVENT},{d.time},{str(d.code)}{NEWLINE}"
+
+            elif isinstance(d, UDPData):
+                txt = f"{TAG_UDPDATA},{d.time},{d.unicode}{NEWLINE}"
+            elif isinstance(d, str):
+                txt = f"{TAG_COMMENTS} {d}"
+            else:
+                continue  # ignore unknown data types or maybe raise error (TODO)
+
+            if isinstance(fl, bz2.BZ2File):
+                fl.write(txt.encode(ENCODING))
+            else:
+                fl.write(txt)
+
+        fl.flush()
+        fl.close()
+

{pyforcedaq-2.0.4 → pyforcedaq-2.0.5.dev3}/src/pyforcedaq/_lib/process_priority_manager.py

@@ -16,7 +16,7 @@ class ProcessPriorityManager(object):
     platform = sys.platform
     pybits = 32 + int(sys.maxsize > 2**32) * 32
     main_process_id = psutil.Process().pid
-    _normal_nice_value = psutil.Process().nice()  # usied on Linux
+    _normal_nice_value = psutil.Process().nice()  # used on Linux
 
     def __init__(self):
         self._subprocs = []