sl-shared-assets 1.0.0rc13__py3-none-any.whl → 1.0.0rc15__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of sl-shared-assets might be problematic. Click here for more details.

Files changed (42) hide show
  1. sl_shared_assets/__init__.py +27 -9
  2. sl_shared_assets/__init__.pyi +71 -0
  3. sl_shared_assets/cli.py +13 -14
  4. sl_shared_assets/cli.pyi +28 -0
  5. sl_shared_assets/data_classes/__init__.py +63 -0
  6. sl_shared_assets/data_classes/__init__.pyi +61 -0
  7. sl_shared_assets/data_classes/configuration_data.py +64 -0
  8. sl_shared_assets/data_classes/configuration_data.pyi +37 -0
  9. sl_shared_assets/data_classes/runtime_data.py +233 -0
  10. sl_shared_assets/data_classes/runtime_data.pyi +145 -0
  11. sl_shared_assets/data_classes/session_data.py +1275 -0
  12. sl_shared_assets/data_classes/session_data.pyi +527 -0
  13. sl_shared_assets/data_classes/surgery_data.py +152 -0
  14. sl_shared_assets/data_classes/surgery_data.pyi +89 -0
  15. sl_shared_assets/server/__init__.py +8 -0
  16. sl_shared_assets/server/__init__.pyi +8 -0
  17. sl_shared_assets/server/job.py +140 -0
  18. sl_shared_assets/server/job.pyi +94 -0
  19. sl_shared_assets/server/server.py +213 -0
  20. sl_shared_assets/server/server.pyi +95 -0
  21. sl_shared_assets/suite2p/__init__.py +8 -0
  22. sl_shared_assets/suite2p/__init__.pyi +4 -0
  23. sl_shared_assets/suite2p/multi_day.py +193 -0
  24. sl_shared_assets/suite2p/multi_day.pyi +99 -0
  25. sl_shared_assets/{suite2p.py → suite2p/single_day.py} +55 -32
  26. sl_shared_assets/suite2p/single_day.pyi +192 -0
  27. sl_shared_assets/tools/__init__.py +8 -0
  28. sl_shared_assets/tools/__init__.pyi +5 -0
  29. sl_shared_assets/{ascension_tools.py → tools/ascension_tools.py} +3 -6
  30. sl_shared_assets/tools/ascension_tools.pyi +68 -0
  31. sl_shared_assets/tools/packaging_tools.pyi +52 -0
  32. sl_shared_assets/tools/transfer_tools.pyi +53 -0
  33. {sl_shared_assets-1.0.0rc13.dist-info → sl_shared_assets-1.0.0rc15.dist-info}/METADATA +1 -1
  34. sl_shared_assets-1.0.0rc15.dist-info/RECORD +40 -0
  35. sl_shared_assets/data_classes.py +0 -1656
  36. sl_shared_assets/server.py +0 -293
  37. sl_shared_assets-1.0.0rc13.dist-info/RECORD +0 -14
  38. /sl_shared_assets/{packaging_tools.py → tools/packaging_tools.py} +0 -0
  39. /sl_shared_assets/{transfer_tools.py → tools/transfer_tools.py} +0 -0
  40. {sl_shared_assets-1.0.0rc13.dist-info → sl_shared_assets-1.0.0rc15.dist-info}/WHEEL +0 -0
  41. {sl_shared_assets-1.0.0rc13.dist-info → sl_shared_assets-1.0.0rc15.dist-info}/entry_points.txt +0 -0
  42. {sl_shared_assets-1.0.0rc13.dist-info → sl_shared_assets-1.0.0rc15.dist-info}/licenses/LICENSE +0 -0
@@ -1,1656 +0,0 @@
1
- """This module provides classes used to store various data used by other Sun lab data acquisition and processing
2
- libraries.This includes classes used to store the data generated during acquisition and preprocessing and classes used
3
- to manage the runtime of other libraries (configuration data classes). Most classes from these modules are used by the
4
- major libraries 'sl-experiment' and 'sl-forgery'."""
5
-
6
- import re
7
- import copy
8
- import shutil as sh
9
- from pathlib import Path
10
- import warnings
11
- from dataclasses import field, dataclass
12
-
13
- import appdirs
14
- from ataraxis_base_utilities import LogLevel, console, ensure_directory_exists
15
- from ataraxis_data_structures import YamlConfig
16
- from ataraxis_time.time_helpers import get_timestamp
17
-
18
- # Ensures console is enabled when this file is imported
19
- if not console.enabled:
20
- console.enable()
21
-
22
-
23
- def replace_root_path(path: Path) -> None:
24
- """Replaces the path to the local root directory used to store all Sun lab projects with the provided path.
25
-
26
- The first time ProjectConfiguration class is instantiated to create a new project on a new machine,
27
- it asks the user to provide the path to the local directory where to save all Sun lab projects. This path is then
28
- stored inside the default user data directory as a .yaml file to be reused for all future projects. To support
29
- replacing this path without searching for the user data directory, which is usually hidden, this function finds and
30
- updates the contents of the file that stores the local root path.
31
-
32
- Args:
33
- path: The path to the new local root directory.
34
- """
35
- # Resolves the path to the static .txt file used to store the local path to the root directory
36
- app_dir = Path(appdirs.user_data_dir(appname="sun_lab_data", appauthor="sun_lab"))
37
- path_file = app_dir.joinpath("root_path.txt")
38
-
39
- # In case this function is called before the app directory is created, ensures the app directory exists
40
- ensure_directory_exists(path_file)
41
-
42
- # Ensures that the input root directory exists
43
- ensure_directory_exists(path)
44
-
45
- # Replaces the contents of the root_path.txt file with the provided path
46
- with open(path_file, "w") as f:
47
- f.write(str(path))
48
-
49
-
50
- @dataclass()
51
- class ProjectConfiguration(YamlConfig):
52
- """Stores the project-specific configuration parameters that do not change between different animals and runtime
53
- sessions.
54
-
55
- An instance of this class is generated and saved as a .yaml file in the 'configuration' directory of each project
56
- when it is created. After that, the stored data is reused for every runtime (training or experiment session) carried
57
- out for each animal of the project. Additionally, a copy of the most actual configuration file is saved inside each
58
- runtime session's 'raw_data' folder, providing seamless integration between the managed data and various Sun lab
59
- (sl-) libraries.
60
-
61
- Notes:
62
- Together with SessionData, this class forms the entry point for all interactions with the data acquired in the
63
- Sun lab. The fields of this class are used to flexibly configure the runtime behavior of major data acquisition
64
- (sl-experiment) and processing (sl-forgery) libraries, adapting them for any project in the lab.
65
-
66
- Most lab projects only need to adjust the "surgery_sheet_id" and "water_log_sheet_id" fields of the class. Most
67
- fields in this class are used by the sl-experiment library to generate the SessionData class instance for each
68
- session and during experiment data acquisition and preprocessing. Data processing pipelines use specialized
69
- configuration files stored in other modules of this library.
70
-
71
- Although all path fields use str | Path datatype, they are always stored as Path objects. These fields are
72
- converted to strings only when the data is dumped as a .yaml file.
73
- """
74
-
75
- project_name: str = ""
76
- """Stores the descriptive name of the project. This name is used to create the root directory for the project and
77
- to discover and load project's data during runtime."""
78
- surgery_sheet_id: str = ""
79
- """The ID of the Google Sheet file that stores information about surgical interventions performed on all animals
80
- participating in the managed project. This log sheet is used to parse and write the surgical intervention data for
81
- each animal into every runtime session raw_data folder, so that the surgery data is always kept together with the
82
- rest of the training and experiment data."""
83
- water_log_sheet_id: str = ""
84
- """The ID of the Google Sheet file that stores information about water restriction (and behavior tracker)
85
- information for all animals participating in the managed project. This is used to synchronize the information
86
- inside the water restriction log with the state of the animal at the end of each training or experiment session.
87
- """
88
- google_credentials_path: str | Path = Path("/media/Data/Experiments/sl-surgery-log-0f651e492767.json")
89
- """
90
- The path to the locally stored .JSON file that contains the service account credentials used to read and write
91
- Google Sheet data. This is used to access and work with the surgery log and the water restriction log files.
92
- Usually, the same service account is used across all projects.
93
- """
94
- server_credentials_path: str | Path = Path("/media/Data/Experiments/server_credentials.yaml")
95
- """
96
- The path to the locally stored .YAML file that contains the credentials for accessing the BioHPC server machine.
97
- While the filesystem of the server machine should already be mounted to the local machine via SMB or equivalent
98
- protocol, this data is used to establish SSH connection to the server and start newly acquired data processing
99
- after it is transferred to the server. This allows data acquisition, preprocessing, and processing to be controlled
100
- by the same runtime and prevents unprocessed data from piling up on the server.
101
- """
102
- local_root_directory: str | Path = Path("/media/Data/Experiments")
103
- """The absolute path to the directory where all projects are stored on the local host-machine (VRPC). Note,
104
- this field is configured automatically each time the class is instantiated through any method, so overwriting it
105
- manually will not be respected."""
106
- local_server_directory: str | Path = Path("/home/cybermouse/server/storage/sun_data")
107
- """The absolute path to the directory where the raw data portion of all projects is stored on the BioHPC server.
108
- This directory should be locally accessible (mounted) using a network sharing protocol, such as SMB."""
109
- local_nas_directory: str | Path = Path("/home/cybermouse/nas/rawdata")
110
- """The absolute path to the directory where all projects are stored on the Synology NAS. This directory should be
111
- locally accessible (mounted) using a network sharing protocol, such as SMB."""
112
- local_mesoscope_directory: str | Path = Path("/home/cybermouse/scanimage/mesodata")
113
- """The absolute path to the root mesoscope (ScanImagePC) directory where all mesoscope-acquired data is aggregated
114
- during acquisition runtime. This directory should be locally accessible (mounted) using a network sharing
115
- protocol, such as SMB."""
116
- local_server_working_directory: str | Path = Path("/home/cybermouse/server/workdir/sun_data")
117
- """The absolute path to the directory where the processed data portion of all projects is stored on the BioHPC
118
- server. This directory should be locally accessible (mounted) using a network sharing protocol, such as SMB."""
119
- remote_storage_directory: str | Path = Path("/storage/sun_data")
120
- """The absolute path, relative to the BioHPC server root, to the directory where all projects are stored on the
121
- slow (SSD) volume of the server. This path is used when running remote (server-side) jobs and, therefore, has to
122
- be relative to the server root."""
123
- remote_working_directory: str | Path = Path("/workdir/sun_data")
124
- """The absolute path, relative to the BioHPC server root, to the directory where all projects are stored on the
125
- fast (NVME) volume of the server. This path is used when running remote (server-side) jobs and, therefore, has to
126
- be relative to the server root."""
127
- face_camera_index: int = 0
128
- """The index of the face camera in the list of all available Harvester-managed cameras."""
129
- left_camera_index: int = 0
130
- """The index of the left body camera in the list of all available OpenCV-managed cameras."""
131
- right_camera_index: int = 2
132
- """The index of the right body camera in the list of all available OpenCV-managed cameras."""
133
- harvesters_cti_path: str | Path = Path("/opt/mvIMPACT_Acquire/lib/x86_64/mvGenTLProducer.cti")
134
- """The path to the GeniCam CTI file used to connect to Harvesters-managed cameras."""
135
- actor_port: str = "/dev/ttyACM0"
136
- """The USB port used by the Actor Microcontroller."""
137
- sensor_port: str = "/dev/ttyACM1"
138
- """The USB port used by the Sensor Microcontroller."""
139
- encoder_port: str = "/dev/ttyACM2"
140
- """The USB port used by the Encoder Microcontroller."""
141
- headbar_port: str = "/dev/ttyUSB0"
142
- """The USB port used by the HeadBar Zaber motor controllers (devices)."""
143
- lickport_port: str = "/dev/ttyUSB1"
144
- """The USB port used by the LickPort Zaber motor controllers (devices)."""
145
- unity_ip: str = "127.0.0.1"
146
- """The IP address of the MQTT broker used to communicate with the Unity game engine. This is only used during
147
- experiment runtimes. Training runtimes ignore this parameter."""
148
- unity_port: int = 1883
149
- """The port number of the MQTT broker used to communicate with the Unity game engine. This is only used during
150
- experiment runtimes. Training runtimes ignore this parameter."""
151
- valve_calibration_data: dict[int | float, int | float] | tuple[tuple[int | float, int | float], ...] = (
152
- (15000, 1.8556),
153
- (30000, 3.4844),
154
- (45000, 7.1846),
155
- (60000, 10.0854),
156
- )
157
- """A tuple of tuples that maps water delivery solenoid valve open times, in microseconds, to the dispensed volume
158
- of water, in microliters. During training and experiment runtimes, this data is used by the ValveModule to translate
159
- the requested reward volumes into times the valve needs to be open to deliver the desired volume of water.
160
- """
161
-
162
- @classmethod
163
- def load(cls, project_name: str, configuration_path: None | Path = None) -> "ProjectConfiguration":
164
- """Loads the project configuration parameters from a project_configuration.yaml file.
165
-
166
- This method is called during each interaction with any runtime session's data, including the creation of a new
167
- session. When this method is called for a non-existent (new) project name, it generates the default
168
- configuration file and prompts the user to update the configuration before proceeding with the runtime. All
169
- future interactions with the sessions from this project reuse the existing configuration file.
170
-
171
- Notes:
172
- As part of its runtime, the method may prompt the user to provide the path to the local root directory.
173
- This directory stores all project subdirectories and acts as the top level of the Sun lab data hierarchy.
174
- The path to the directory is then saved inside user's default data directory, so that it can be reused for
175
- all future projects. Use sl-replace-root CLI to replace the saved root directory path.
176
-
177
- Since this class is used for all Sun lab data structure interactions, this method supports multiple ways of
178
- loading class data. If this method is called as part of the sl-experiment new session creation pipeline, use
179
- 'project_name' argument. If this method is called as part of the sl-forgery data processing pipeline(s), use
180
- 'configuration_path' argument.
181
-
182
- Args:
183
- project_name: The name of the project whose configuration file needs to be discovered and loaded or, if the
184
- project does not exist, created.
185
- configuration_path: Optional. The path to the project_configuration.yaml file from which to load the data.
186
- This way of resolving the configuration data source always takes precedence over the project_name when
187
- both are provided.
188
-
189
- Returns:
190
- The initialized ProjectConfiguration instance that stores the configuration data for the target project.
191
- """
192
-
193
- # If the configuration path is not provided, uses the 'default' resolution strategy that involves reading the
194
- # user's data directory
195
- if configuration_path is None:
196
- # Uses appdirs to locate the user data directory and resolve the path to the storage file
197
- app_dir = Path(appdirs.user_data_dir(appname="sl_assets", appauthor="sun_lab"))
198
- path_file = app_dir.joinpath("root_path.txt")
199
-
200
- # If the .txt file that stores the local root path does not exist, prompts the user to provide the path to
201
- # the local root directory and creates the root_path.txt file
202
- if not path_file.exists():
203
- # Gets the path to the local root directory from the user via command line input
204
- message = (
205
- "Unable to resolve the local root directory automatically. Provide the absolute path to the local "
206
- "directory that stores all project-specific directories. This is required when resolving project "
207
- "configuration based on project's name."
208
- )
209
- # noinspection PyTypeChecker
210
- console.echo(message=message, level=LogLevel.WARNING)
211
- root_path_str = input("Local root path: ")
212
- root_path = Path(root_path_str)
213
-
214
- # If necessary, generates the local root directory
215
- ensure_directory_exists(root_path)
216
-
217
- # Also ensures that the app directory exists, so that the path_file can be created below.
218
- ensure_directory_exists(path_file)
219
-
220
- # Saves the root path to the file
221
- with open(path_file, "w") as f:
222
- f.write(str(root_path))
223
-
224
- # Once the location of the path storage file is resolved, reads the root path from the file
225
- with open(path_file, "r") as f:
226
- root_path = Path(f.read().strip())
227
-
228
- # Uses the root experiment directory path to generate the path to the target project's configuration file.
229
- configuration_path = root_path.joinpath(project_name, "configuration", "project_configuration.yaml")
230
- ensure_directory_exists(configuration_path) # Ensures the directory tree for the config path exists.
231
-
232
- # If the configuration file does not exist (this is the first time this class is initialized for a given
233
- # project), generates a precursor (default) configuration file and prompts the user to update the configuration.
234
- if not configuration_path.exists():
235
- message = (
236
- f"Unable to load project configuration data from disk as no 'project_configuration.yaml' file "
237
- f"found at the provided project path. Generating a precursor (default) configuration file under "
238
- f"{project_name}/configuration directory. Edit the file to specify project configuration before "
239
- f"proceeding further to avoid runtime errors. Also, edit other configuration precursors saved to the "
240
- f"same directory to control other aspects of data acquisition and processing."
241
- )
242
- # noinspection PyTypeChecker
243
- console.echo(message=message, level=LogLevel.WARNING)
244
-
245
- # Generates the default project configuration instance and dumps it as a .yaml file. Note, as part of
246
- # this process, the class generates the correct 'local_root_path' based on the path provided by the
247
- # user.
248
- precursor = ProjectConfiguration(local_root_directory=Path(str(configuration_path.parents[2])))
249
- precursor.project_name = project_name
250
- precursor.save(path=configuration_path)
251
-
252
- # Waits for the user to manually configure the newly created file.
253
- input(f"Enter anything to continue: ")
254
-
255
- # Loads the data from the YAML file and initializes the class instance. This now uses either the automatically
256
- # resolved configuration path or the manually provided path
257
- instance: ProjectConfiguration = cls.from_yaml(file_path=configuration_path) # type: ignore
258
-
259
- # Converts all paths loaded as strings to Path objects used inside the library
260
- instance.local_mesoscope_directory = Path(instance.local_mesoscope_directory)
261
- instance.local_nas_directory = Path(instance.local_nas_directory)
262
- instance.local_server_directory = Path(instance.local_server_directory)
263
- instance.local_server_working_directory = Path(instance.local_server_working_directory)
264
- instance.remote_storage_directory = Path(instance.remote_storage_directory)
265
- instance.remote_working_directory = Path(instance.remote_working_directory)
266
- instance.google_credentials_path = Path(instance.google_credentials_path)
267
- instance.server_credentials_path = Path(instance.server_credentials_path)
268
- instance.harvesters_cti_path = Path(instance.harvesters_cti_path)
269
-
270
- # Local root path is always re-computed from the resolved configuration file's location
271
- instance.local_root_directory = Path(str(configuration_path.parents[2]))
272
-
273
- # Converts valve_calibration data from dictionary to a tuple of tuples format
274
- if not isinstance(instance.valve_calibration_data, tuple):
275
- instance.valve_calibration_data = tuple((k, v) for k, v in instance.valve_calibration_data.items())
276
-
277
- # Partially verifies the loaded data. Most importantly, this step does not allow proceeding if the user did not
278
- # replace the surgery log and water restriction log placeholders with valid ID values.
279
- instance._verify_data()
280
-
281
- # Returns the initialized class instance to caller
282
- return instance
283
-
284
- def save(self, path: Path) -> None:
285
- """Saves class instance data to disk as a project_configuration.yaml file.
286
-
287
- This method is automatically called when a new project is created. After this method's runtime, all future
288
- calls to the load() method will reuse the configuration data saved to the .yaml file.
289
-
290
- Notes:
291
- When this method is used to generate the configuration .yaml file for a new project, it also generates the
292
- example 'default_experiment.yaml'. This file is designed to showcase how to write ExperimentConfiguration
293
- data files that are used to control Mesoscope-VR system states during experiment session runtimes.
294
-
295
- Args:
296
- path: The path to the .yaml file to save the data to.
297
- """
298
-
299
- # Copies instance data to prevent it from being modified by reference when executing the steps below
300
- original = copy.deepcopy(self)
301
-
302
- # Converts all Path objects to strings before dumping the data, as .yaml encoder does not properly recognize
303
- # Path objects
304
- original.local_root_directory = str(original.local_root_directory)
305
- original.local_mesoscope_directory = str(original.local_mesoscope_directory)
306
- original.local_nas_directory = str(original.local_nas_directory)
307
- original.local_server_directory = str(original.local_server_directory)
308
- original.local_server_working_directory = str(original.local_server_working_directory)
309
- original.remote_storage_directory = str(original.remote_storage_directory)
310
- original.remote_working_directory = str(original.remote_working_directory)
311
- original.google_credentials_path = str(original.google_credentials_path)
312
- original.server_credentials_path = str(original.server_credentials_path)
313
- original.harvesters_cti_path = str(original.harvesters_cti_path)
314
-
315
- # Converts valve calibration data into dictionary format
316
- if isinstance(original.valve_calibration_data, tuple):
317
- original.valve_calibration_data = {k: v for k, v in original.valve_calibration_data}
318
-
319
- # Saves the data to the YAML file
320
- original.to_yaml(file_path=path)
321
-
322
- # As part of this runtime, also generates and dumps the 'precursor' experiment configuration file.
323
- experiment_configuration_path = path.parent.joinpath("default_experiment.yaml")
324
- if not experiment_configuration_path.exists():
325
- example_experiment = ExperimentConfiguration()
326
- example_experiment.to_yaml(experiment_configuration_path)
327
-
328
- def _verify_data(self) -> None:
329
- """Verifies the user-modified data loaded from the project_configuration.yaml file.
330
-
331
- Since this class is explicitly designed to be modified by the user, this verification step is carried out to
332
- ensure that the loaded data matches expectations. This reduces the potential for user errors to impact the
333
- runtime behavior of the libraries using this class. This internal method is automatically called by the load()
334
- method.
335
-
336
- Notes:
337
- The method does not verify all fields loaded from the configuration file and instead focuses on fields that
338
- do not have valid default values. Since these fields are expected to be frequently modified by users, they
339
- are the ones that require additional validation.
340
-
341
- Raises:
342
- ValueError: If the loaded data does not match expected formats or values.
343
- """
344
-
345
- # Verifies Google Sheet ID formatting. Google Sheet IDs are usually 44 characters long, containing letters,
346
- # numbers, hyphens, and underscores
347
- pattern = r"^[a-zA-Z0-9_-]{44}$"
348
- if not re.match(pattern, self.surgery_sheet_id):
349
- message = (
350
- f"Unable to verify the surgery_sheet_id field loaded from the 'project_configuration.yaml' file. "
351
- f"Expected a string with 44 characters, using letters, numbers, hyphens, and underscores, but found: "
352
- f"{self.surgery_sheet_id}."
353
- )
354
- console.error(message=message, error=ValueError)
355
- if not re.match(pattern, self.water_log_sheet_id):
356
- message = (
357
- f"Unable to verify the surgery_sheet_id field loaded from the 'project_configuration.yaml' file. "
358
- f"Expected a string with 44 characters, using letters, numbers, hyphens, and underscores, but found: "
359
- f"{self.water_log_sheet_id}."
360
- )
361
- console.error(message=message, error=ValueError)
362
-
363
-
364
- @dataclass()
365
- class RawData:
366
- """Stores the paths to the directories and files that make up the 'raw_data' session directory.
367
-
368
- The raw_data directory stores the data acquired during the session runtime before and after preprocessing. Since
369
- preprocessing does not alter the data, any data in that folder is considered 'raw'. The raw_data folder is initially
370
- created on the VRPC and, after preprocessing, is copied to the BioHPC server and the Synology NAS for long-term
371
- storage and further processing.
372
-
373
- Notes:
374
- The overall structure of the raw_data directory remains fixed for the entire lifetime of the data. It is reused
375
- across all destinations.
376
- """
377
-
378
- raw_data_path: str | Path
379
- """Stores the path to the root raw_data directory of the session. This directory stores all raw data during
380
- acquisition and preprocessing. Note, preprocessing does not alter raw data, so at any point in time all data inside
381
- the folder is considered 'raw'."""
382
- camera_data_path: str | Path = "null"
383
- """Stores the path to the directory that contains all camera data acquired during the session. Primarily, this
384
- includes .mp4 video files from each recorded camera."""
385
- mesoscope_data_path: str | Path = "null"
386
- """Stores the path to the directory that contains all Mesoscope data acquired during the session. Primarily, this
387
- includes the mesoscope-acquired .tiff files (brain activity data) and the motion estimation data."""
388
- behavior_data_path: str | Path = "null"
389
- """Stores the path to the directory that contains all behavior data acquired during the session. Primarily, this
390
- includes the .npz log files used by data-acquisition libraries to store all acquired data. The data stored in this
391
- way includes the camera and mesoscope frame timestamps and the states of Mesoscope-VR components, such as lick
392
- sensors, rotary encoders, and other modules."""
393
- zaber_positions_path: str | Path = "null"
394
- """Stores the path to the zaber_positions.yaml file. This file contains the snapshot of all Zaber motor positions
395
- at the end of the session. Zaber motors are used to position the LickPort and the HeadBar manipulators, which is
396
- essential for supporting proper brain imaging and animal's running behavior during the session."""
397
- session_descriptor_path: str | Path = "null"
398
- """Stores the path to the session_descriptor.yaml file. This file is partially filled by the system during runtime
399
- and partially by the experimenter after the runtime. It contains session-specific information, such as the specific
400
- training parameters, the positions of the Mesoscope objective and the notes made by the experimenter during
401
- runtime."""
402
- hardware_configuration_path: str | Path = "null"
403
- """Stores the path to the hardware_configuration.yaml file. This file contains the partial snapshot of the
404
- calibration parameters used by the Mesoscope-VR system components during runtime. Primarily, this is used during
405
- data processing to read the .npz data log files generated during runtime."""
406
- surgery_metadata_path: str | Path = "null"
407
- """Stores the path to the surgery_metadata.yaml file. This file contains the most actual information about the
408
- surgical intervention(s) performed on the animal prior to the session."""
409
- project_configuration_path: str | Path = "null"
410
- """Stores the path to the project_configuration.yaml file. This file contains the snapshot of the configuration
411
- parameters for the session's project."""
412
- session_data_path: str | Path = "null"
413
- """Stores the path to the session_data.yaml file. This path is used b y the SessionData instance to save itself to
414
- disk as a .yaml file. The file contains all paths used during data acquisition and processing on both the VRPC and
415
- the BioHPC server."""
416
- experiment_configuration_path: str | Path = "null"
417
- """Stores the path to the experiment_configuration.yaml file. This file contains the snapshot of the
418
- experiment runtime configuration used by the session. This file is only created for experiment session. It does not
419
- exist for behavior training sessions."""
420
- mesoscope_positions_path: str | Path = "null"
421
- """Stores the path to the mesoscope_positions.yaml file. This file contains the snapshot of the positions used
422
- by the Mesoscope at the end of the session. This includes both the physical position of the mesoscope objective and
423
- the 'virtual' tip, tilt, and fastZ positions set via ScanImage software. This file is only created for experiment
424
- sessions that use the mesoscope, it is omitted for behavior training sessions."""
425
- window_screenshot_path: str | Path = "null"
426
- """Stores the path to the .png screenshot of the ScanImagePC screen. The screenshot should contain the image of the
427
- cranial window and the red-dot alignment windows. This is used to generate a visual snapshot of the cranial window
428
- alignment and appearance for each experiment session. This file is only created for experiment sessions that use
429
- the mesoscope, it is omitted for behavior training sessions."""
430
- telomere_path: str | Path = "null"
431
- """Stores the path to the telomere.bin file. This file is created by the data processing pipelines running on the
432
- BioHPC server to confirm that the raw_data transferred to the server was not altered or damage in transmission. This
433
- path is used by the BioHPC server pipeline. The VRPC uses the path stored in the 'destinations' section to check
434
- for the existence of the file when it purges redundant data."""
435
- checksum_path: str | Path = "null"
436
- """Stores the path to the ax_checksum.txt file. This file is generated as part of packaging the data for
437
- transmission and stores the xxHash-128 checksum of the data. It is used to verify that the transmission did not
438
- damage or otherwise alter the data.."""
439
-
440
- def __post_init__(self) -> None:
441
- """This method is automatically called after class instantiation and ensures that all path fields of the class
442
- are converted to Path objects.
443
- """
444
-
445
- # Translates the root raw_data path to the Path object before using it to determine other Path objects.
446
- self.raw_data_path = Path(self.raw_data_path)
447
-
448
- # When the class is instantiated for the first time, resolves all 'null' path placeholders
449
- if self.camera_data_path == "null":
450
- self.camera_data_path = self.raw_data_path.joinpath("camera_data")
451
- if self.mesoscope_data_path == "null":
452
- self.mesoscope_data_path = self.raw_data_path.joinpath("mesoscope_data")
453
- if self.behavior_data_path == "null":
454
- self.behavior_data_path = self.raw_data_path.joinpath("behavior_data")
455
- if self.zaber_positions_path == "null":
456
- self.zaber_positions_path = self.raw_data_path.joinpath("zaber_positions.yaml")
457
- if self.session_descriptor_path == "null":
458
- self.session_descriptor_path = self.raw_data_path.joinpath("session_descriptor.yaml")
459
- if self.hardware_configuration_path == "null":
460
- self.hardware_configuration_path = self.raw_data_path.joinpath("hardware_configuration.yaml")
461
- if self.surgery_metadata_path == "null":
462
- self.surgery_metadata_path = self.raw_data_path.joinpath("surgery_metadata.yaml")
463
- if self.project_configuration_path == "null":
464
- self.project_configuration_path = self.raw_data_path.joinpath("project_configuration.yaml")
465
- if self.session_data_path == "null":
466
- self.session_data_path = self.raw_data_path.joinpath("session_data.yaml")
467
- if self.experiment_configuration_path == "null":
468
- self.experiment_configuration_path = self.raw_data_path.joinpath("experiment_configuration.yaml")
469
- if self.mesoscope_positions_path == "null":
470
- self.mesoscope_positions_path = self.raw_data_path.joinpath("mesoscope_positions.yaml")
471
- if self.window_screenshot_path == "null":
472
- self.window_screenshot_path = self.raw_data_path.joinpath("window_screenshot.png")
473
- if self.telomere_path == "null":
474
- self.telomere_path = self.raw_data_path.joinpath("telomere.bin")
475
- if self.checksum_path == "null":
476
- self.checksum_path = self.raw_data_path.joinpath("ax_checksum.txt")
477
-
478
- def make_string(self) -> None:
479
- """Converts all Path objects stored inside the class to strings.
480
-
481
- This transformation is required to support dumping class data into a .YAML file so that the data can be stored
482
- on disk.
483
- """
484
- self.raw_data_path = str(self.raw_data_path)
485
- self.camera_data_path = str(self.camera_data_path)
486
- self.mesoscope_data_path = str(self.mesoscope_data_path)
487
- self.behavior_data_path = str(self.behavior_data_path)
488
- self.zaber_positions_path = str(self.zaber_positions_path)
489
- self.session_descriptor_path = str(self.session_descriptor_path)
490
- self.hardware_configuration_path = str(self.hardware_configuration_path)
491
- self.surgery_metadata_path = str(self.surgery_metadata_path)
492
- self.project_configuration_path = str(self.project_configuration_path)
493
- self.session_data_path = str(self.session_data_path)
494
- self.experiment_configuration_path = str(self.experiment_configuration_path)
495
- self.mesoscope_positions_path = str(self.mesoscope_positions_path)
496
- self.window_screenshot_path = str(self.window_screenshot_path)
497
- self.telomere_path = str(self.telomere_path)
498
- self.checksum_path = str(self.checksum_path)
499
-
500
- def make_dirs(self) -> None:
501
- """Ensures that all major subdirectories and the root raw_data directory exist.
502
-
503
- This method is used by the VRPC to generate the raw_data directory when it creates a new session.
504
- """
505
- ensure_directory_exists(Path(self.raw_data_path))
506
- ensure_directory_exists(Path(self.camera_data_path))
507
- ensure_directory_exists(Path(self.mesoscope_data_path))
508
- ensure_directory_exists(Path(self.behavior_data_path))
509
-
510
- def switch_root(self, new_root: Path) -> None:
511
- """Changes the root of the managed raw_data directory to the provided root path.
512
-
513
- This service method is used by the SessionData class to convert all paths in this class to be relative to the
514
- new root. This is used to adjust the SessionData instance to work for the VRPC (one root) or the BioHPC server
515
- (another root). This method is only implemented for subclasses intended to be used both locally and on the
516
- BioHPC server.
517
-
518
- Note:
519
- This is the only class that contains this method. Other classes are re-initialized using the data loaded
520
- from the RawData section, if necessary.
521
-
522
- Args:
523
- new_root: The new root directory to use for all paths inside the instance. This has to be the path to the
524
- directory that stores all Sun lab projects on the target machine.
525
- """
526
- # Gets current root from the raw_data_path.
527
- old_root = Path(self.raw_data_path).parents[3]
528
-
529
- # Updates all paths by replacing old_root with new_root
530
- self.raw_data_path = new_root.joinpath(Path(self.raw_data_path).relative_to(old_root))
531
- self.camera_data_path = new_root.joinpath(Path(self.camera_data_path).relative_to(old_root))
532
- self.mesoscope_data_path = new_root.joinpath(Path(self.mesoscope_data_path).relative_to(old_root))
533
- self.behavior_data_path = new_root.joinpath(Path(self.behavior_data_path).relative_to(old_root))
534
- self.zaber_positions_path = new_root.joinpath(Path(self.zaber_positions_path).relative_to(old_root))
535
- self.session_descriptor_path = new_root.joinpath(Path(self.session_descriptor_path).relative_to(old_root))
536
- self.hardware_configuration_path = new_root.joinpath(
537
- Path(self.hardware_configuration_path).relative_to(old_root)
538
- )
539
- self.surgery_metadata_path = new_root.joinpath(Path(self.surgery_metadata_path).relative_to(old_root))
540
- self.project_configuration_path = new_root.joinpath(Path(self.project_configuration_path).relative_to(old_root))
541
- self.session_data_path = new_root.joinpath(Path(self.session_data_path).relative_to(old_root))
542
- self.experiment_configuration_path = new_root.joinpath(
543
- Path(self.experiment_configuration_path).relative_to(old_root)
544
- )
545
- self.mesoscope_positions_path = new_root.joinpath(Path(self.mesoscope_positions_path).relative_to(old_root))
546
- self.window_screenshot_path = new_root.joinpath(Path(self.window_screenshot_path).relative_to(old_root))
547
- self.telomere_path = new_root.joinpath(Path(self.telomere_path).relative_to(old_root))
548
- self.checksum_path = new_root.joinpath(Path(self.checksum_path).relative_to(old_root))
549
-
550
-
551
- @dataclass()
552
- class ProcessedData:
553
- """Stores the paths to the directories and files that make up the 'processed_data' session directory.
554
-
555
- The processed_data directory stores the data generated by various processing pipelines from the raw data. Processed
556
- data represents an intermediate step between raw data and the dataset used in the data analysis, but is not itself
557
- designed to be analyzed.
558
-
559
- Notes:
560
- The paths from this section are typically used only on the BioHPC server. This is because most data processing
561
- in the lab is performed using the processing server's resources. On the server, processed data is stored on
562
- the fast volume, in contrast to raw data, which is stored on the slow volume. However, to support local testing,
563
- the class resolves the paths in this section both locally and globally (on the server).
564
-
565
- """
566
-
567
- processed_data_path: str | Path
568
- """Stores the path to the root processed_data directory of the session. This directory stores the processed data
569
- as it is generated by various data processing pipelines."""
570
- deeplabcut_path: str | Path
571
- """Stores the path to the root DeepLabCut project directory. Since DeepLabCut adopts a project-based directory
572
- management hierarchy, it is easier to have a single DLC folder shared by all animals and sessions of a given
573
- project. This root folder is typically stored under the main project directory on the fast BioHPC server volume."""
574
- configuration_path: str | Path
575
- """Stores the path to the project's processed data configuration directory. This directory is used by all animals
576
- and sessions of the project to store data processing configuration files. Since processing configuration is
577
- typically reused for all data in the project, this allows updating a single file to control the processing for all
578
- data. This directory is typically stored on the fast BioHPC server volume."""
579
- camera_data_path: str | Path = "null"
580
- """Stores the output of the DeepLabCut pose estimation pipeline."""
581
- mesoscope_data_path: str | Path = "null"
582
- """Stores the output of the suite2p cell registration pipeline."""
583
- behavior_data_path: str | Path = "null"
584
- """Stores the output of the Sun lab behavior data extraction pipeline."""
585
- suite2p_configuration_path: str | Path = "null"
586
- """Stores the path to the suite2p_configuration.yaml file stored inside the project's 'configuration' directory on
587
- the fast BioHPC server volume. This configuration file specifies the parameters for the 'single day' suite2p
588
- registration pipeline, which is applied to each session that generates brain activity data."""
589
- processing_tracker_path: str | Path = "null"
590
- """Stores the path to the processing_tracker.yaml file stored inside the sessions' root processed_data directory on
591
- the fast BioHPC server volume. This file is used to track which processing pipelines to apply to the target
592
- session and the status (success / failure) of each processing step.
593
- """
594
- multiday_configuration_path: str | Path = "null"
595
- """Stores the path to the multiday_configuration.yaml file stored inside the project's 'configuration' directory
596
- on the fast BioHPC server volume. This configuration file specifies the parameters for the 'multiday'
597
- sl-suite2p-based registration pipelines used tot rack brain cells across multiple sessions."""
598
-
599
- def __post_init__(self) -> None:
600
- """This method is automatically called after class instantiation and ensures that all path fields of the class
601
- are converted to Path objects.
602
- """
603
-
604
- # Ensures all objects that have to be provided externally are transformed to Path objects. Unlike 'null' fields,
605
- # these paths cannot be resolved automatically
606
- self.processed_data_path = Path(self.processed_data_path)
607
- self.deeplabcut_path = Path(self.deeplabcut_path)
608
- self.configuration_path = Path(self.configuration_path)
609
-
610
- # When the class is instantiated for the first time, resolves all 'null' path placeholders
611
- if self.camera_data_path == "null":
612
- self.camera_data_path = self.processed_data_path.joinpath("camera_data")
613
- if self.mesoscope_data_path == "null":
614
- self.mesoscope_data_path = self.processed_data_path.joinpath("mesoscope_data")
615
- if self.behavior_data_path == "null":
616
- self.behavior_data_path = self.processed_data_path.joinpath("behavior_data")
617
- if self.suite2p_configuration_path == "null":
618
- self.suite2p_configuration_path = self.processed_data_path.joinpath("suite2p_configuration.yaml")
619
- if self.processing_tracker_path == "null":
620
- self.processing_tracker_path = self.processed_data_path.joinpath("processing_tracker.yaml")
621
- if self.multiday_configuration_path == "null":
622
- self.multiday_configuration_path = self.processed_data_path.joinpath("multiday_configuration.yaml")
623
-
624
- def make_string(self) -> None:
625
- """Converts all Path objects stored inside the class to strings.
626
-
627
- This transformation is required to support dumping class data into a .YAML file so that the data can be stored
628
- on disk.
629
- """
630
- self.processed_data_path = str(self.processed_data_path)
631
- self.deeplabcut_path = str(self.deeplabcut_path)
632
- self.configuration_path = str(self.configuration_path)
633
- self.camera_data_path = str(self.camera_data_path)
634
- self.mesoscope_data_path = str(self.mesoscope_data_path)
635
- self.behavior_data_path = str(self.behavior_data_path)
636
- self.suite2p_configuration_path = str(self.suite2p_configuration_path)
637
- self.processing_tracker_path = str(self.processing_tracker_path)
638
- self.multiday_configuration_path = str(self.multiday_configuration_path)
639
-
640
- def make_dirs(self) -> None:
641
- """Ensures that all major subdirectories of the processed_data directory exist.
642
-
643
- This method is used by the BioHPC server to generate the processed_data directory as part of the sl-forgery
644
- library runtime.
645
- """
646
- ensure_directory_exists(Path(self.processed_data_path))
647
- ensure_directory_exists(Path(self.deeplabcut_path))
648
- ensure_directory_exists(Path(self.configuration_path))
649
- ensure_directory_exists(Path(self.camera_data_path))
650
- ensure_directory_exists(Path(self.mesoscope_data_path))
651
- ensure_directory_exists(Path(self.behavior_data_path))
652
-
653
-
654
- @dataclass()
655
- class PersistentData:
656
- """Stores the paths to the directories and files that make up the 'persistent_data' directories of the VRPC and
657
- the ScanImagePC.
658
-
659
- Persistent data directories are used to keep certain files on the VRPC and the ScanImagePC. Typically, this data
660
- is reused during the following sessions. For example, a copy of Zaber motor positions is persisted on the VRPC for
661
- each animal after every session to support automatically restoring Zaber motors to the positions used during the
662
- previous session.
663
-
664
- Notes:
665
- Persistent data includes the project and experiment configuration data. Some persistent data is overwritten
666
- after each session, other data is generated once and kept through the animal's lifetime. Primarily, this data is
667
- only used internally by the sl-experiment or sl-forgery libraries and is not intended for end-users.
668
- """
669
-
670
- scanimage_persistent_path: str | Path
671
- """Stores the path to the project and animal specific 'persistent_data' directory to which the managed session
672
- belongs, relative to the ScanImagePC root. This directory is primarily used to store reference motion registration
673
- data acquired during the first experiment session for each animal."""
674
- vrpc_persistent_path: str | Path
675
- """Stores the path to the project and animal specific 'persistent_data' directory to which the managed session
676
- belongs, relative to the VRPC root. This directory is primarily used to back up the Zaber and Mesoscope positions,
677
- so that they can be restored between sessions of the same animals."""
678
- zaber_positions_path: str | Path = "null"
679
- """Stores the path to the Zaber motor positions snapshot generated at the end of the previous session runtime. This
680
- is used to automatically restore all Zaber motors to the same position across all sessions."""
681
- mesoscope_positions_path: str | Path = "null"
682
- """Stores the path to the Mesoscope positions snapshot generated at the end of the previous session runtime. This
683
- is used to help the user to (manually) restore the Mesoscope to the same position across all sessions."""
684
- motion_estimator_path: str | Path = "null"
685
- """Stores the 'reference' motion estimator file generated during the first experiment session of each animal. This
686
- file is kept on the ScanImagePC to image the same population of cells across all experiment sessions."""
687
-
688
- def __post_init__(self) -> None:
689
- """This method is automatically called after class instantiation and ensures that all path fields of the class
690
- are converted to Path objects.
691
- """
692
-
693
- # Converts 'anchor' paths that have to be provided at class instantiation
694
- self.scanimage_persistent_path = Path(self.scanimage_persistent_path)
695
- self.vrpc_persistent_path = Path(self.vrpc_persistent_path)
696
-
697
- # Replaces 'null' defaults with automatically resolved paths
698
- if self.zaber_positions_path == "null":
699
- self.zaber_positions_path = self.vrpc_persistent_path.joinpath("zaber_positions.yaml")
700
- if self.mesoscope_positions_path == "null":
701
- self.mesoscope_positions_path = self.vrpc_persistent_path.joinpath("mesoscope_positions.yaml")
702
- if self.motion_estimator_path == "null":
703
- self.motion_estimator_path = self.scanimage_persistent_path.joinpath("MotionEstimator.me")
704
-
705
- def make_string(self) -> None:
706
- """Converts all Path objects stored inside the class to strings.
707
-
708
- This transformation is required to support dumping class data into a .YAML file so that the data can be stored
709
- on disk.
710
- """
711
- self.scanimage_persistent_path = str(self.scanimage_persistent_path)
712
- self.vrpc_persistent_path = str(self.vrpc_persistent_path)
713
- self.zaber_positions_path = str(self.zaber_positions_path)
714
- self.mesoscope_positions_path = str(self.mesoscope_positions_path)
715
- self.motion_estimator_path = str(self.motion_estimator_path)
716
-
717
- def make_dirs(self) -> None:
718
- """Ensures that the VRPC and the ScanImagePC persistent_data directories exist."""
719
-
720
- # We need to call ensure_directory_exists one for each unique directory tree
721
- ensure_directory_exists(Path(self.vrpc_persistent_path)) # vrpc_root/project/animal/persistent_data
722
- ensure_directory_exists(Path(self.scanimage_persistent_path)) # scanimagepc_root/project/animal/persistent_data
723
-
724
-
725
- @dataclass()
726
- class MesoscopeData:
727
- """Stores the paths to the directories used by the ScanImagePC to save mesoscope-generated data during session
728
- runtime.
729
-
730
- The ScanImagePC is largely isolated from the VRPC during runtime. For the VRPC to pull the data acquired by the
731
- ScanImagePC, it has to use the predefined directory structure to save the data. This class stores the predefined
732
- path to various directories where ScanImagePC is expected to save the data and store it after acquisition.sers.
733
- """
734
-
735
- root_data_path: str | Path
736
- """Stores the path to the root ScanImagePC data directory, mounted to the VRPC filesystem via the SMB or equivalent
737
- protocol. This path is used during experiment session runtimes to discover the cranial window screenshots
738
- taken by the user before starting the experiment."""
739
- session_specific_path: str | Path
740
- """Stores the path to the session-specific data directory. This directory is generated at the end of each experiment
741
- runtime to prepare mesoscope data for further processing and to reset the 'shared' folder for the next session's
742
- runtime."""
743
- mesoscope_data_path: str | Path = "null"
744
- """Stores the path to the 'general' mesoscope_data directory. All experiment sessions (across all animals and
745
- projects) use the same mesoscope_data directory to save the data generated by the mesoscope via ScanImage
746
- software. This simplifies ScanImagePC configuration process during runtime. The data is moved into a
747
- session-specific directory during preprocessing."""
748
- ubiquitin_path: str | Path = "null"
749
- """Stores the path to the 'ubiquitin.bin' file. This file is automatically generated inside the session-specific
750
- data folder after its contents are safely transferred to the VRPC as part of preprocessing. During redundant data
751
- removal, the VRPC searches for directories marked with ubiquitin.bin, so that they can be safely removed from the
752
- ScanImagePC filesystem."""
753
-
754
- def __post_init__(self) -> None:
755
- """This method is automatically called after class instantiation and ensures that all path fields of the class
756
- are converted to Path objects.
757
- """
758
- # Converts the paths that have to be provided at class instantiation to the Path objects.
759
- self.root_data_path = Path(self.root_data_path)
760
- self.session_specific_path = Path(self.session_specific_path)
761
-
762
- # Resolves the 'null' placeholders using the 'anchor' paths above
763
- if self.mesoscope_data_path == "null":
764
- self.mesoscope_data_path = self.root_data_path.joinpath("mesoscope_data")
765
- if self.ubiquitin_path == "null":
766
- self.ubiquitin_path = self.session_specific_path.joinpath("ubiquitin.bin")
767
-
768
- def make_string(self) -> None:
769
- """Converts all Path objects stored inside the class to strings.
770
-
771
- This transformation is required to support dumping class data into a .YAML file so that the data can be stored
772
- on disk.
773
- """
774
- self.root_data_path = str(self.root_data_path)
775
- self.session_specific_path = str(self.session_specific_path)
776
- self.mesoscope_data_path = str(self.mesoscope_data_path)
777
- self.ubiquitin_path = str(self.ubiquitin_path)
778
-
779
- def make_dirs(self) -> None:
780
- """Ensures that the ScanImagePC data acquisition directories exist."""
781
-
782
- # Does not create the session-specific directory. This is on purpose, as the session-specific directory
783
- # is generated during runtime by renaming the 'general' mesoscope_data directory. The 'general' directory is
784
- # then recreated from scratch. This ensures that the general directory is empty (ready for the next session)
785
- # with minimal I/O overhead.
786
- ensure_directory_exists(Path(self.mesoscope_data_path))
787
-
788
-
789
- @dataclass()
790
- class Destinations:
791
- """Stores the paths to the VRPC filesystem-mounted Synology NAS and BioHPC server directories.
792
-
793
- These directories are used during data preprocessing to transfer the preprocessed raw_data directory from the
794
- VRPC to the long-term storage destinations.
795
- """
796
-
797
- nas_raw_data_path: str | Path
798
- """Stores the path to the session's raw_data directory on the Synology NAS, which is mounted to the VRPC via the
799
- SMB or equivalent protocol."""
800
- server_raw_data_path: str | Path
801
- """Stores the path to the session's raw_data directory on the BioHPC server, which is mounted to the VRPC via the
802
- SMB or equivalent protocol."""
803
- server_processed_data_path: str | Path
804
- """Stores the path to the session's processed_data directory on the BioHPC server, which is mounted to the VRPC via
805
- the SMB or equivalent protocol."""
806
- server_configuration_path: str | Path
807
- """Stores the path to the project-specific 'configuration' directory on the BioHPC server, which is mounted to the
808
- VRPC via the SMB or equivalent protocol."""
809
- server_telomere_path: str | Path = "null"
810
- """Stores the path to the session's telomere.bin marker. This marker is generated as part of BioHPC-side data
811
- processing pipeline to notify he VRPC that the server received the data intact. it is used to determine which
812
- data can be safely removed from the VRPC filesystem."""
813
- server_suite2p_configuration_path: str | Path = "null"
814
- """Stores the path to the suite2p_configuration.yaml file stored inside the project's 'configuration' directory on
815
- the fast BioHPC server volume. This configuration file specifies the parameters for the 'single day' suite2p
816
- registration pipeline, which is applied to each session that generates brain activity data."""
817
- server_processing_tracker_path: str | Path = "null"
818
- """Stores the path to the processing_tracker.yaml file stored inside the sessions' root processed_data directory on
819
- the fast BioHPC server volume. This file is used to track which processing pipelines to apply to the target
820
- session and the status (success / failure) of each processing step.
821
- """
822
- server_multiday_configuration_path: str | Path = "null"
823
- """Stores the path to the multiday_configuration.yaml file stored inside the project's 'configuration' directory
824
- on the fast BioHPC server volume. This configuration file specifies the parameters for the 'multiday'
825
- sl-suite2p-based registration pipelines used tot rack brain cells across multiple sessions."""
826
-
827
- def __post_init__(self) -> None:
828
- """This method is automatically called after class instantiation and ensures that all path fields of the class
829
- are converted to Path objects.
830
- """
831
- self.nas_raw_data_path = Path(self.nas_raw_data_path)
832
- self.server_raw_data_path = Path(self.server_raw_data_path)
833
- self.server_configuration_path = Path(self.server_configuration_path)
834
- self.server_processed_data_path = Path(self.server_processed_data_path)
835
-
836
- if self.server_telomere_path == "null":
837
- self.server_telomere_path = self.server_raw_data_path.joinpath("telomere.bin")
838
- if self.server_suite2p_configuration_path == "null":
839
- self.server_suite2p_configuration_path = self.server_configuration_path.joinpath(
840
- "suite2p_configuration.yaml"
841
- )
842
- if self.server_processing_tracker_path == "null":
843
- self.server_processing_tracker_path = self.server_processed_data_path.joinpath("processing_tracker.yaml")
844
- if self.server_multiday_configuration_path == "null":
845
- self.server_multiday_configuration_path = self.server_configuration_path.joinpath(
846
- "multiday_configuration.yaml"
847
- )
848
-
849
- def make_string(self) -> None:
850
- """Converts all Path objects stored inside the class to strings.
851
-
852
- This transformation is required to support dumping class data into a .YAML file so that the data can be stored
853
- on disk.
854
- """
855
- self.nas_raw_data_path = str(self.nas_raw_data_path)
856
- self.server_raw_data_path = str(self.server_raw_data_path)
857
- self.server_telomere_path = str(self.server_telomere_path)
858
- self.server_configuration_path = str(self.server_configuration_path)
859
- self.server_processed_data_path = str(self.server_processed_data_path)
860
- self.server_suite2p_configuration_path = str(self.server_suite2p_configuration_path)
861
- self.server_processing_tracker_path = str(self.server_processing_tracker_path)
862
- self.server_multiday_configuration_path = str(self.server_multiday_configuration_path)
863
-
864
- def make_dirs(self) -> None:
865
- """Ensures that all destination directories exist."""
866
- ensure_directory_exists(Path(self.nas_raw_data_path))
867
- ensure_directory_exists(Path(self.server_raw_data_path))
868
- ensure_directory_exists(Path(self.server_configuration_path))
869
- ensure_directory_exists(Path(self.server_processed_data_path))
870
-
871
-
872
- @dataclass
873
- class SessionData(YamlConfig):
874
- """Stores and manages the data layout of a single training or experiment session acquired using the Sun lab
875
- Mesoscope-VR system.
876
-
877
- The primary purpose of this class is to maintain the session data structure across all supported destinations and
878
- during all processing stages. It generates the paths used by all other classes from all Sun lab libraries that
879
- interact with the session's data from the point of its creation and until the data is integrated into an
880
- analysis dataset.
881
-
882
- When necessary, the class can be used to either generate a new session or load the layout of an already existing
883
- session. When the class is used to create a new session, it generates the new session's name using the current
884
- UTC timestamp, accurate to microseconds. This ensures that each session name is unique and preserves the overall
885
- session order.
886
-
887
- Notes:
888
- If this class is instantiated on the VRPC, it is expected that the BioHPC server, Synology NAS, and ScanImagePC
889
- data directories are mounted on the local filesystem via the SMB or equivalent protocol. All manipulations
890
- with these destinations are carried out with the assumption that the local OS has full access to these
891
- directories and filesystems.
892
-
893
- This class is specifically designed for working with the data from a single session, performed by a single
894
- animal under the specific experiment. The class is used to manage both raw and processed data. It follows the
895
- data through acquisition, preprocessing and processing stages of the Sun lab data workflow. Together with
896
- ProjectConfiguration class, this class serves as an entry point for all interactions with the managed session's
897
- data.
898
- """
899
-
900
- project_name: str
901
- """Stores the name of the managed session's project."""
902
- animal_id: str
903
- """Stores the unique identifier of the animal that participates in the managed session."""
904
- session_name: str
905
- """Stores the name (timestamp-based ID) of the managed session."""
906
- session_type: str
907
- """Stores the type of the session. Primarily, this determines how to read the session_descriptor.yaml file. Has
908
- to be set to one of the four supported types: 'Lick training', 'Run training', 'Window checking' or 'Experiment'.
909
- """
910
- experiment_name: str | None
911
- """Stores the name of the experiment configuration file. If the session_type field is set to 'Experiment' and this
912
- field is not None (null), it communicates the specific experiment configuration used by the session. During runtime,
913
- the name stored here is used to load the specific experiment configuration data stored in a .yaml file with the
914
- same name. If the session is not an experiment session, this field is ignored."""
915
- raw_data: RawData
916
- """This section stores the paths to various directories and files that make up the raw_data subfolder. This
917
- subfolder stores all data acquired during training or experiment runtimes before and after preprocessing. Note, the
918
- preprocessing does not change the raw data in any way other than lossless compression and minor format
919
- reorganization. Therefore, the data is considered 'raw' both before and after preprocessing."""
920
- processed_data: ProcessedData
921
- """This section stores the paths to various directories used to store processed session data. Processed data is
922
- generated from raw data by running various processing pipelines, such as suite2p, DeepLabCut and Sun lab's behavior
923
- parsing pipelines. Typically, all data is processed on the BioHPC server and may be stored on a filesystem volume
924
- different from the one that stores the raw data."""
925
- persistent_data: PersistentData
926
- """This section stores the paths to various files and directories that are held back on the VRPC and ScanImagePC
927
- after the session data is transferred to long-term storage destinations as part of preprocessing. Typically, this
928
- data is reused during the acquisition of future runtime session data. This section is not used during data
929
- processing."""
930
- mesoscope_data: MesoscopeData
931
- """This section stores the paths to various directories used by the ScanImagePC when acquiring mesoscope-related
932
- data. During runtime, the VRPC (behavior data and experiment control) and the ScanImagePC (brain activity data and
933
- Mesoscope control) operate mostly independently of each-other. During preprocessing, the VRPC pulls the data from
934
- the ScanImagePC, using the paths in this section to find the data to be transferred. This section is not used
935
- during data processing."""
936
- destinations: Destinations
937
- """This section stores the paths to the destination directories on the BioHPC server and Synology NAS, to which the
938
- data is copied as part of preprocessing for long-term storage and further processing. Both of these directories
939
- should be mapped (mounted) to the VRPC's filesystem via the SMB or equivalent protocol. This section is not used
940
- during data processing."""
941
-
942
- @classmethod
943
- def create(
944
- cls,
945
- animal_id: str,
946
- session_type: str,
947
- project_configuration: ProjectConfiguration,
948
- experiment_name: str | None = None,
949
- session_name: str | None = None,
950
- ) -> "SessionData":
951
- """Creates a new SessionData object and generates the new session's data structure.
952
-
953
- This method is called by sl-experiment runtimes that create new training or experiment sessions to generate the
954
- session data directory tree. It always assumes it is called on the VRPC and, as part of its runtime, resolves
955
- and generates the necessary local and ScanImagePC directories to support acquiring and preprocessing session's
956
- data.
957
-
958
- Notes:
959
- To load an already existing session data structure, use the load() method instead.
960
-
961
- This method automatically dumps the data of the created SessionData instance into the session_data.yaml file
962
- inside the root raw_data directory of the created hierarchy. It also finds and dumps other configuration
963
- files, such as project_configuration.yaml and experiment_configuration.yaml, into the same raw_data
964
- directory. This ensures that if the session's runtime is interrupted unexpectedly, the acquired data can
965
- still be processed.
966
-
967
- Args:
968
- animal_id: The ID code of the animal for which the data is acquired.
969
- session_type: The type of the session. Primarily, this determines how to read the session_descriptor.yaml
970
- file. Valid options are 'Lick training', 'Run training', 'Window checking', or 'Experiment'.
971
- experiment_name: The name of the experiment executed during managed session. This optional argument is only
972
- used for 'Experiment' session types. It is used to find the experiment configuration .YAML file.
973
- project_configuration: The initialized ProjectConfiguration instance that stores the session's project
974
- configuration data. This is used to determine the root directory paths for all lab machines used during
975
- data acquisition and processing.
976
- session_name: An optional session_name override. Generally, this argument should not be provided for most
977
- sessions. When provided, the method uses this name instead of generating a new timestamp-based name.
978
- This is only used during the 'ascension' runtime to convert old data structures to the modern
979
- lab standards.
980
-
981
- Returns:
982
- An initialized SessionData instance that stores the layout of the newly created session's data.
983
- """
984
-
985
- # Acquires the UTC timestamp to use as the session name
986
- if session_name is None:
987
- session_name = str(get_timestamp(time_separator="-"))
988
-
989
- # Extracts the root directory paths stored inside the project configuration file. All roots are expected to be
990
- # mounted on the local (VRPC) via SMB or equivalent protocol and be relative to the VRPC root.
991
- vrpc_root = Path(project_configuration.local_root_directory)
992
- mesoscope_root = Path(project_configuration.local_mesoscope_directory)
993
- biohpc_root = Path(project_configuration.local_server_directory)
994
- biohpc_workdir = Path(project_configuration.local_server_working_directory)
995
- nas_root = Path(project_configuration.local_nas_directory)
996
-
997
- # Extracts the name of the project stored inside the project configuration file.
998
- project_name = project_configuration.project_name
999
-
1000
- # Constructs the session directory path and generates the directory
1001
- session_path = vrpc_root.joinpath(project_name, animal_id, session_name)
1002
-
1003
- # Handles potential session name conflicts
1004
- counter = 0
1005
- while session_path.exists():
1006
- counter += 1
1007
- new_session_name = f"{session_name}_{counter}"
1008
- session_path = vrpc_root.joinpath(project_name, animal_id, new_session_name)
1009
-
1010
- # If a conflict is detected and resolved, warns the user about the resolved conflict.
1011
- if counter > 0:
1012
- message = (
1013
- f"Session name conflict occurred for animal '{animal_id}' of project '{project_name}' "
1014
- f"when adding the new session with timestamp {session_name}. The session with identical name "
1015
- f"already exists. The newly created session directory uses a '_{counter}' postfix to distinguish "
1016
- f"itself from the already existing session directory."
1017
- )
1018
- warnings.warn(message=message)
1019
-
1020
- # Generates subclasses stored inside the main class instance based on the data resolved above. Note; most fields
1021
- # of these classes are initialized automatically, using the 'root' path arguments.
1022
- raw_data = RawData(
1023
- raw_data_path=session_path.joinpath("raw_data"),
1024
- )
1025
- raw_data.make_dirs() # Generates the local directory tree
1026
-
1027
- processed_data = ProcessedData(
1028
- processed_data_path=session_path.joinpath("processed_data"),
1029
- deeplabcut_path=vrpc_root.joinpath(project_name, "deeplabcut"),
1030
- configuration_path=vrpc_root.joinpath(project_name, "configuration"),
1031
- )
1032
-
1033
- vrpc_persistent_path = vrpc_root.joinpath(project_name, animal_id, "persistent_data")
1034
- scanimagepc_persistent_path = mesoscope_root.joinpath(project_name, animal_id, "persistent_data")
1035
- persistent_data = PersistentData(
1036
- vrpc_persistent_path=vrpc_persistent_path, scanimage_persistent_path=scanimagepc_persistent_path
1037
- )
1038
- persistent_data.make_dirs() # Generates all persistent directory trees
1039
-
1040
- mesoscope_data = MesoscopeData(
1041
- root_data_path=mesoscope_root, session_specific_path=mesoscope_root.joinpath(session_name)
1042
- )
1043
- mesoscope_data.make_dirs() # Generates all Mesoscope directory trees
1044
-
1045
- destinations = Destinations(
1046
- nas_raw_data_path=nas_root.joinpath(project_name, animal_id, session_name, "raw_data"),
1047
- server_raw_data_path=biohpc_root.joinpath(project_name, animal_id, session_name, "raw_data"),
1048
- server_configuration_path=biohpc_workdir.joinpath(project_name, "configuration"),
1049
- server_processed_data_path=biohpc_workdir.joinpath(project_name, "processed_data"),
1050
- )
1051
- destinations.make_dirs() # Generates all destination directory trees
1052
-
1053
- # Packages the sections generated above into a SessionData instance
1054
- instance = SessionData(
1055
- project_name=project_configuration.project_name,
1056
- animal_id=animal_id,
1057
- session_name=session_name,
1058
- session_type=session_type,
1059
- raw_data=raw_data,
1060
- processed_data=processed_data,
1061
- persistent_data=persistent_data,
1062
- mesoscope_data=mesoscope_data,
1063
- destinations=destinations,
1064
- experiment_name=experiment_name,
1065
- )
1066
-
1067
- # Saves the configured instance data to the session's folder, so that it can be reused during processing or
1068
- # preprocessing
1069
- instance._save()
1070
-
1071
- # Extracts and saves the necessary configuration classes to the session raw_data folder. Note, this list of
1072
- # classes is not exhaustive. More classes are saved as part of the session runtime management class start() and
1073
- # __init__() method runtimes:
1074
-
1075
- # Resolves the path to the project configuration folder
1076
- vrpc_configuration_path = vrpc_root.joinpath(project_name, "configuration")
1077
-
1078
- # Discovers and saves the necessary configuration class instances to the raw_data folder of the managed session:
1079
- # Project Configuration
1080
- sh.copy2(
1081
- src=vrpc_configuration_path.joinpath("project_configuration.yaml"),
1082
- dst=instance.raw_data.project_configuration_path,
1083
- )
1084
- # Experiment Configuration, if the session type is Experiment.
1085
- if experiment_name is not None:
1086
- sh.copy2(
1087
- src=vrpc_configuration_path.joinpath(f"{experiment_name}.yaml"),
1088
- dst=instance.raw_data.experiment_configuration_path,
1089
- )
1090
-
1091
- # Returns the initialized SessionData instance to caller
1092
- return instance
1093
-
1094
- @classmethod
1095
- def load(
1096
- cls,
1097
- session_path: Path,
1098
- on_server: bool,
1099
- ) -> "SessionData":
1100
- """Loads the SessionData instance from the target session's session_data.yaml file.
1101
-
1102
- This method is used to load the data layout information of an already existing session. Primarily, this is used
1103
- when preprocessing or processing session data. Depending on the call location (machine), the method
1104
- automatically resolves all necessary paths and creates the necessary directories.
1105
-
1106
- Notes:
1107
- To create a new session, use the create() method instead.
1108
-
1109
- Args:
1110
- session_path: The path to the root directory of an existing session, e.g.: vrpc_root/project/animal/session.
1111
- on_server: Determines whether the method is used to initialize an existing session on the VRPC or the
1112
- BioHPC server. Note, VRPC runtimes use the same 'root' directory to store raw_data and processed_data
1113
- subfolders. BioHPC server runtimes use different volumes (drives) to store these subfolders.
1114
-
1115
- Returns:
1116
- An initialized SessionData instance for the session whose data is stored at the provided path.
1117
-
1118
- Raises:
1119
- FileNotFoundError: If the 'session_data.yaml' file is not found under the session_path/raw_data/ subfolder.
1120
- """
1121
- # To properly initialize the SessionData instance, the provided path should contain the raw_data directory
1122
- # with session_data.yaml file.
1123
- session_data_path = session_path.joinpath("raw_data", "session_data.yaml")
1124
- if not session_data_path.exists():
1125
- message = (
1126
- f"Unable to load the SessionData class for the target session: {session_path.stem}. No "
1127
- f"session_data.yaml file was found inside the raw_data folder of the session. This likely "
1128
- f"indicates that the session runtime was interrupted before recording any data, or that the "
1129
- f"session path does not point to a valid session."
1130
- )
1131
- console.error(message=message, error=FileNotFoundError)
1132
-
1133
- # Loads class data from .yaml
1134
- instance: SessionData = cls.from_yaml(file_path=session_data_path) # type: ignore
1135
-
1136
- # The method assumes that the 'donor' .yaml file is always stored inside the raw_data directory of the session
1137
- # to be processed. Since the directory itself might have moved (between or even within the same PC) relative to
1138
- # where it was when the SessionData snapshot was generated, reconfigures the paths to all raw_data files using
1139
- # the root from above.
1140
- instance.raw_data.switch_root(new_root=session_path.parents[2])
1141
-
1142
- # Resolves the paths to the processed_data directories. The resolution strategy depends on whether the method is
1143
- # called on the VRPC (locally) or the BioHPC server (remotely).
1144
- new_root: Path
1145
- if not on_server:
1146
- # Local runtimes use the same root session directory for both raw_data and processed_data. This stems from
1147
- # the assumption that most local machines in the lab only use NVME (fast) volumes and, therefore, do not
1148
- # need to separate 'storage' and 'working' data directories.
1149
-
1150
- # Regenerates the processed_data paths using the information loaded from the session_data.yaml file and
1151
- # static path definition rules used in the lab.
1152
- new_root = session_path.parents[2] # Reuses the local root for non-server runtimes
1153
-
1154
- else:
1155
- # The BioHPC server stores raw_data on slow volume and processed_data on fast (NVME) volume. Therefore, to
1156
- # configure processed_data paths, the method first needs to load the fast volume root path from the
1157
- # project_configuration.yaml file stored in the raw_data folder.
1158
- project_configuration: ProjectConfiguration = ProjectConfiguration.load(
1159
- project_name=instance.project_name,
1160
- configuration_path=Path(instance.raw_data.project_configuration_path),
1161
- )
1162
- new_root = Path(project_configuration.remote_working_directory)
1163
-
1164
- # Regenerates the processed_data path depending on the root resolution above
1165
- instance.processed_data = ProcessedData(
1166
- processed_data_path=new_root.joinpath(
1167
- instance.project_name, instance.animal_id, instance.session_name, "processed_data"
1168
- ),
1169
- configuration_path=new_root.joinpath(instance.project_name, instance.animal_id, "configuration"),
1170
- deeplabcut_path=new_root.joinpath(instance.project_name, "deeplabcut"),
1171
- )
1172
-
1173
- # Generates processed_data directories
1174
- instance.processed_data.make_dirs()
1175
-
1176
- # Returns the initialized SessionData instance to caller
1177
- return instance
1178
-
1179
- def _save(self) -> None:
1180
- """Saves the instance data to the 'raw_data' directory of the managed session as a 'session_data.yaml' file.
1181
-
1182
- This is used to save the data stored in the instance to disk, so that it can be reused during preprocessing or
1183
- data processing. The method is intended to only be used by the SessionData instance itself during its
1184
- create() method runtime.
1185
- """
1186
-
1187
- # Copies instance data to prevent it from being modified by reference when executing the steps below
1188
- original = copy.deepcopy(self)
1189
-
1190
- # Extracts the target file path before it is converted to a string.
1191
- file_path: Path = copy.copy(self.raw_data.session_data_path) # type: ignore
1192
-
1193
- # Converts all Paths objects to strings before dumping the data to YAML.
1194
- original.raw_data.make_string()
1195
- original.processed_data.make_string()
1196
- original.persistent_data.make_string()
1197
- original.mesoscope_data.make_string()
1198
- original.destinations.make_string()
1199
-
1200
- # Saves instance data as a .YAML file
1201
- original.to_yaml(file_path=file_path)
1202
-
1203
-
1204
- @dataclass()
1205
- class ExperimentState:
1206
- """Encapsulates the information used to set and maintain the desired experiment and Mesoscope-VR system state.
1207
-
1208
- Primarily, experiment runtime logic (task logic) is resolved by the Unity game engine. However, the Mesoscope-VR
1209
- system configuration may also need to change throughout the experiment to optimize the runtime by disabling or
1210
- reconfiguring specific hardware modules. For example, some experiment stages may require the running wheel to be
1211
- locked to prevent the animal from running, and other may require the VR screens to be turned off.
1212
- """
1213
-
1214
- experiment_state_code: int
1215
- """The integer code of the experiment state. Experiment states do not have a predefined meaning, Instead, each
1216
- project is expected to define and follow its own experiment state code mapping. Typically, the experiment state
1217
- code is used to denote major experiment stages, such as 'baseline', 'task', 'cooldown', etc. Note, the same
1218
- experiment state code can be used by multiple sequential ExperimentState instances to change the VR system states
1219
- while maintaining the same experiment state."""
1220
- vr_state_code: int
1221
- """One of the supported VR system state-codes. Currently, the Mesoscope-VR system supports two state codes. State
1222
- code '1' denotes 'REST' state and code '2' denotes 'RUN' state. Note, multiple consecutive ExperimentState
1223
- instances with different experiment state codes can reuse the same VR state code."""
1224
- state_duration_s: float
1225
- """The time, in seconds, to maintain the current combination of the experiment and VR states."""
1226
-
1227
-
1228
- @dataclass()
1229
- class ExperimentConfiguration(YamlConfig):
1230
- """Stores the configuration of a single experiment runtime.
1231
-
1232
- Primarily, this includes the sequence of experiment and Virtual Reality (Mesoscope-VR) states that defines the flow
1233
- of the experiment runtime. During runtime, the main runtime control function traverses the sequence of states
1234
- stored in this class instance start-to-end in the exact order specified by the user. Together with custom Unity
1235
- projects that define the task logic (how the system responds to animal interactions with the VR system) this class
1236
- allows flexibly implementing a wide range of experiments.
1237
-
1238
- Each project should define one or more experiment configurations and save them as .yaml files inside the project
1239
- 'configuration' folder. The name for each configuration file is defined by the user and is used to identify and load
1240
- the experiment configuration when 'sl-run-experiment' CLI command exposed by the sl-experiment library is executed.
1241
- """
1242
-
1243
- cue_map: dict[int, float] = field(default_factory=lambda: {0: 30.0, 1: 30.0, 2: 30.0, 3: 30.0, 4: 30.0})
1244
- """A dictionary that maps each integer-code associated with a wall cue used in the Virtual Reality experiment
1245
- environment to its length in real-world centimeters. It is used to map each VR cue to the distance the animal needs
1246
- to travel to fully traverse the wall cue region from start to end."""
1247
- experiment_states: dict[str, ExperimentState] = field(
1248
- default_factory=lambda: {
1249
- "baseline": ExperimentState(experiment_state_code=1, vr_state_code=1, state_duration_s=30),
1250
- "experiment": ExperimentState(experiment_state_code=2, vr_state_code=2, state_duration_s=120),
1251
- "cooldown": ExperimentState(experiment_state_code=3, vr_state_code=1, state_duration_s=15),
1252
- }
1253
- )
1254
- """A dictionary that uses human-readable state-names as keys and ExperimentState instances as values. Each
1255
- ExperimentState instance represents a phase of the experiment."""
1256
-
1257
-
1258
- @dataclass()
1259
- class HardwareConfiguration(YamlConfig):
1260
- """This class is used to save the runtime hardware configuration parameters as a .yaml file.
1261
-
1262
- This information is used to read and decode the data saved to the .npz log files during runtime as part of data
1263
- processing.
1264
-
1265
- Notes:
1266
- All fields in this dataclass initialize to None. During log processing, any log associated with a hardware
1267
- module that provides the data stored in a field will be processed, unless that field is None. Therefore, setting
1268
- any field in this dataclass to None also functions as a flag for whether to parse the log associated with the
1269
- module that provides this field's information.
1270
-
1271
- This class is automatically configured by MesoscopeExperiment and BehaviorTraining classes from sl-experiment
1272
- library to facilitate log parsing.
1273
- """
1274
-
1275
- cue_map: dict[int, float] | None = None
1276
- """MesoscopeExperiment instance property. Stores the dictionary that maps the integer id-codes associated with each
1277
- wall cue in the Virtual Reality task environment with distances in real-world centimeters animals should run on the
1278
- wheel to fully traverse the cue region on a linearized track."""
1279
- cm_per_pulse: float | None = None
1280
- """EncoderInterface instance property. Stores the conversion factor used to translate encoder pulses into
1281
- real-world centimeters."""
1282
- maximum_break_strength: float | None = None
1283
- """BreakInterface instance property. Stores the breaking torque, in Newton centimeters, applied by the break to
1284
- the edge of the running wheel when it is engaged at 100% strength."""
1285
- minimum_break_strength: float | None = None
1286
- """BreakInterface instance property. Stores the breaking torque, in Newton centimeters, applied by the break to
1287
- the edge of the running wheel when it is engaged at 0% strength (completely disengaged)."""
1288
- lick_threshold: int | None = None
1289
- """LickInterface instance property. Determines the threshold, in 12-bit Analog to Digital Converter (ADC) units,
1290
- above which an interaction value reported by the lick sensor is considered a lick (compared to noise or non-lick
1291
- touch)."""
1292
- valve_scale_coefficient: float | None = None
1293
- """ValveInterface instance property. To dispense precise water volumes during runtime, ValveInterface uses power
1294
- law equation applied to valve calibration data to determine how long to keep the valve open. This stores the
1295
- scale_coefficient of the power law equation that describes the relationship between valve open time and dispensed
1296
- water volume, derived from calibration data."""
1297
- valve_nonlinearity_exponent: float | None = None
1298
- """ValveInterface instance property. To dispense precise water volumes during runtime, ValveInterface uses power
1299
- law equation applied to valve calibration data to determine how long to keep the valve open. This stores the
1300
- nonlinearity_exponent of the power law equation that describes the relationship between valve open time and
1301
- dispensed water volume, derived from calibration data."""
1302
- torque_per_adc_unit: float | None = None
1303
- """TorqueInterface instance property. Stores the conversion factor used to translate torque values reported by the
1304
- sensor as 12-bit Analog to Digital Converter (ADC) units, into real-world Newton centimeters (N·cm) of torque that
1305
- had to be applied to the edge of the running wheel to produce the observed ADC value."""
1306
- screens_initially_on: bool | None = None
1307
- """ScreenInterface instance property. Stores the initial state of the Virtual Reality screens at the beginning of
1308
- the session runtime."""
1309
- recorded_mesoscope_ttl: bool | None = None
1310
- """TTLInterface instance property. A boolean flag that determines whether the processed session recorded brain
1311
- activity data with the mesoscope."""
1312
-
1313
-
1314
- @dataclass()
1315
- class LickTrainingDescriptor(YamlConfig):
1316
- """This class is used to save the description information specific to lick training sessions as a .yaml file.
1317
-
1318
- The information stored in this class instance is filled in two steps. The main runtime function fills most fields
1319
- of the class, before it is saved as a .yaml file. After runtime, the experimenter manually fills leftover fields,
1320
- such as 'experimenter_notes,' before the class instance is transferred to the long-term storage destination.
1321
-
1322
- The fully filled instance data is also used during preprocessing to write the water restriction log entry for the
1323
- trained animal.
1324
- """
1325
-
1326
- experimenter: str
1327
- """The ID of the experimenter running the session."""
1328
- mouse_weight_g: float
1329
- """The weight of the animal, in grams, at the beginning of the session."""
1330
- dispensed_water_volume_ml: float
1331
- """Stores the total water volume, in milliliters, dispensed during runtime."""
1332
- minimum_reward_delay: int
1333
- """Stores the minimum delay, in seconds, that can separate the delivery of two consecutive water rewards."""
1334
- maximum_reward_delay_s: int
1335
- """Stores the maximum delay, in seconds, that can separate the delivery of two consecutive water rewards."""
1336
- maximum_water_volume_ml: float
1337
- """Stores the maximum volume of water the system is allowed to dispense during training."""
1338
- maximum_training_time_m: int
1339
- """Stores the maximum time, in minutes, the system is allowed to run the training for."""
1340
- experimenter_notes: str = "Replace this with your notes."
1341
- """This field is not set during runtime. It is expected that each experimenter replaces this field with their
1342
- notes made during runtime."""
1343
- experimenter_given_water_volume_ml: float = 0.0
1344
- """The additional volume of water, in milliliters, administered by the experimenter to the animal after the session.
1345
- """
1346
-
1347
-
1348
- @dataclass()
1349
- class RunTrainingDescriptor(YamlConfig):
1350
- """This class is used to save the description information specific to run training sessions as a .yaml file.
1351
-
1352
- The information stored in this class instance is filled in two steps. The main runtime function fills most fields
1353
- of the class, before it is saved as a .yaml file. After runtime, the experimenter manually fills leftover fields,
1354
- such as 'experimenter_notes,' before the class instance is transferred to the long-term storage destination.
1355
-
1356
- The fully filled instance data is also used during preprocessing to write the water restriction log entry for the
1357
- trained animal.
1358
- """
1359
-
1360
- experimenter: str
1361
- """The ID of the experimenter running the session."""
1362
- mouse_weight_g: float
1363
- """The weight of the animal, in grams, at the beginning of the session."""
1364
- dispensed_water_volume_ml: float
1365
- """Stores the total water volume, in milliliters, dispensed during runtime."""
1366
- final_run_speed_threshold_cm_s: float
1367
- """Stores the final running speed threshold, in centimeters per second, that was active at the end of training."""
1368
- final_run_duration_threshold_s: float
1369
- """Stores the final running duration threshold, in seconds, that was active at the end of training."""
1370
- initial_run_speed_threshold_cm_s: float
1371
- """Stores the initial running speed threshold, in centimeters per second, used during training."""
1372
- initial_run_duration_threshold_s: float
1373
- """Stores the initial running duration threshold, in seconds, used during training."""
1374
- increase_threshold_ml: float
1375
- """Stores the volume of water delivered to the animal, in milliliters, that triggers the increase in the running
1376
- speed and duration thresholds."""
1377
- run_speed_increase_step_cm_s: float
1378
- """Stores the value, in centimeters per second, used by the system to increment the running speed threshold each
1379
- time the animal receives 'increase_threshold' volume of water."""
1380
- run_duration_increase_step_s: float
1381
- """Stores the value, in seconds, used by the system to increment the duration threshold each time the animal
1382
- receives 'increase_threshold' volume of water."""
1383
- maximum_water_volume_ml: float
1384
- """Stores the maximum volume of water the system is allowed to dispense during training."""
1385
- maximum_training_time_m: int
1386
- """Stores the maximum time, in minutes, the system is allowed to run the training for."""
1387
- experimenter_notes: str = "Replace this with your notes."
1388
- """This field is not set during runtime. It is expected that each experimenter will replace this field with their
1389
- notes made during runtime."""
1390
- experimenter_given_water_volume_ml: float = 0.0
1391
- """The additional volume of water, in milliliters, administered by the experimenter to the animal after the session.
1392
- """
1393
-
1394
-
1395
- @dataclass()
1396
- class MesoscopeExperimentDescriptor(YamlConfig):
1397
- """This class is used to save the description information specific to experiment sessions as a .yaml file.
1398
-
1399
- The information stored in this class instance is filled in two steps. The main runtime function fills most fields
1400
- of the class, before it is saved as a .yaml file. After runtime, the experimenter manually fills leftover fields,
1401
- such as 'experimenter_notes,' before the class instance is transferred to the long-term storage destination.
1402
-
1403
- The fully filled instance data is also used during preprocessing to write the water restriction log entry for the
1404
- animal participating in the experiment runtime.
1405
- """
1406
-
1407
- experimenter: str
1408
- """The ID of the experimenter running the session."""
1409
- mouse_weight_g: float
1410
- """The weight of the animal, in grams, at the beginning of the session."""
1411
- dispensed_water_volume_ml: float
1412
- """Stores the total water volume, in milliliters, dispensed during runtime."""
1413
- experimenter_notes: str = "Replace this with your notes."
1414
- """This field is not set during runtime. It is expected that each experimenter will replace this field with their
1415
- notes made during runtime."""
1416
- experimenter_given_water_volume_ml: float = 0.0
1417
- """The additional volume of water, in milliliters, administered by the experimenter to the animal after the session.
1418
- """
1419
-
1420
-
1421
- @dataclass()
1422
- class ZaberPositions(YamlConfig):
1423
- """This class is used to save Zaber motor positions as a .yaml file to reuse them between sessions.
1424
-
1425
- The class is specifically designed to store, save, and load the positions of the LickPort and HeadBar motors
1426
- (axes). It is used to both store Zaber motor positions for each session for future analysis and to restore the same
1427
- Zaber motor positions across consecutive runtimes for the same project and animal combination.
1428
-
1429
- Notes:
1430
- All positions are saved using native motor units. All class fields initialize to default placeholders that are
1431
- likely NOT safe to apply to the VR system. Do not apply the positions loaded from the file unless you are
1432
- certain they are safe to use.
1433
-
1434
- Exercise caution when working with Zaber motors. The motors are powerful enough to damage the surrounding
1435
- equipment and manipulated objects. Do not modify the data stored inside the .yaml file unless you know what you
1436
- are doing.
1437
- """
1438
-
1439
- headbar_z: int = 0
1440
- """The absolute position, in native motor units, of the HeadBar z-axis motor."""
1441
- headbar_pitch: int = 0
1442
- """The absolute position, in native motor units, of the HeadBar pitch-axis motor."""
1443
- headbar_roll: int = 0
1444
- """The absolute position, in native motor units, of the HeadBar roll-axis motor."""
1445
- lickport_z: int = 0
1446
- """The absolute position, in native motor units, of the LickPort z-axis motor."""
1447
- lickport_x: int = 0
1448
- """The absolute position, in native motor units, of the LickPort x-axis motor."""
1449
- lickport_y: int = 0
1450
- """The absolute position, in native motor units, of the LickPort y-axis motor."""
1451
-
1452
-
1453
- @dataclass()
1454
- class MesoscopePositions(YamlConfig):
1455
- """This class is used to save the real and virtual Mesoscope objective positions as a .yaml file to reuse it
1456
- between experiment sessions.
1457
-
1458
- Primarily, the class is used to help the experimenter to position the Mesoscope at the same position across
1459
- multiple imaging sessions. It stores both the physical (real) position of the objective along the motorized
1460
- X, Y, Z, and Roll axes and the virtual (ScanImage software) tip, tilt, and fastZ focus axes.
1461
-
1462
- Notes:
1463
- Since the API to read and write these positions automatically is currently not available, this class relies on
1464
- the experimenter manually entering all positions and setting the mesoscope to these positions when necessary.
1465
- """
1466
-
1467
- mesoscope_x_position: float = 0.0
1468
- """The X-axis position, in centimeters, of the Mesoscope objective used during session runtime."""
1469
- mesoscope_y_position: float = 0.0
1470
- """The Y-axis position, in centimeters, of the Mesoscope objective used during session runtime."""
1471
- mesoscope_roll_position: float = 0.0
1472
- """The Roll-axis position, in degrees, of the Mesoscope objective used during session runtime."""
1473
- mesoscope_z_position: float = 0.0
1474
- """The Z-axis position, in centimeters, of the Mesoscope objective used during session runtime."""
1475
- mesoscope_fast_z_position: float = 0.0
1476
- """The Fast-Z-axis position, in micrometers, of the Mesoscope objective used during session runtime."""
1477
- mesoscope_tip_position: float = 0.0
1478
- """The Tilt-axis position, in degrees, of the Mesoscope objective used during session runtime."""
1479
- mesoscope_tilt_position: float = 0.0
1480
- """The Tip-axis position, in degrees, of the Mesoscope objective used during session runtime."""
1481
-
1482
-
1483
- @dataclass()
1484
- class SubjectData:
1485
- """Stores the ID information of the surgical intervention's subject (animal)."""
1486
-
1487
- id: int
1488
- """Stores the unique ID (name) of the subject. Assumes all animals are given a numeric ID, rather than a string
1489
- name."""
1490
- ear_punch: str
1491
- """Stores the ear tag location of the subject."""
1492
- sex: str
1493
- """Stores the gender of the subject."""
1494
- genotype: str
1495
- """Stores the genotype of the subject."""
1496
- date_of_birth_us: int
1497
- """Stores the date of birth of the subject as the number of microseconds elapsed since UTC epoch onset."""
1498
- weight_g: float
1499
- """Stores the weight of the subject pre-surgery, in grams."""
1500
- cage: int
1501
- """Stores the number of the cage used to house the subject after surgery."""
1502
- location_housed: str
1503
- """Stores the location used to house the subject after the surgery."""
1504
- status: str
1505
- """Stores the current status of the subject (alive / deceased)."""
1506
-
1507
-
1508
- @dataclass()
1509
- class ProcedureData:
1510
- """Stores the general information about the surgical intervention."""
1511
-
1512
- surgery_start_us: int
1513
- """Stores the date and time when the surgery has started as microseconds elapsed since UTC epoch onset."""
1514
- surgery_end_us: int
1515
- """Stores the date and time when the surgery has ended as microseconds elapsed since UTC epoch onset."""
1516
- surgeon: str
1517
- """Stores the name or ID of the surgeon. If the intervention was carried out by multiple surgeons, all participating
1518
- surgeon names and IDs are stored as part of the same string."""
1519
- protocol: str
1520
- """Stores the experiment protocol number (ID) used during the surgery."""
1521
- surgery_notes: str
1522
- """Stores surgeon's notes taken during the surgery."""
1523
- post_op_notes: str
1524
- """Stores surgeon's notes taken during the post-surgery recovery period."""
1525
- surgery_quality: int = 0
1526
- """Stores the quality of the surgical intervention as a numeric level. 0 indicates unusable (bad) result, 1
1527
- indicates usable result that is not good enough to be included in a publication, 2 indicates publication-grade
1528
- result."""
1529
-
1530
-
1531
- @dataclass
1532
- class ImplantData:
1533
- """Stores the information about a single implantation performed during the surgical intervention.
1534
-
1535
- Multiple ImplantData instances are used at the same time if the surgery involved multiple implants.
1536
- """
1537
-
1538
- implant: str
1539
- """The descriptive name of the implant."""
1540
- implant_target: str
1541
- """The name of the brain region or cranium section targeted by the implant."""
1542
- implant_code: int
1543
- """The manufacturer code or internal reference code for the implant. This code is used to identify the implant in
1544
- additional datasheets and lab ordering documents."""
1545
- implant_ap_coordinate_mm: float
1546
- """Stores implant's antero-posterior stereotactic coordinate, in millimeters, relative to bregma."""
1547
- implant_ml_coordinate_mm: float
1548
- """Stores implant's medial-lateral stereotactic coordinate, in millimeters, relative to bregma."""
1549
- implant_dv_coordinate_mm: float
1550
- """Stores implant's dorsal-ventral stereotactic coordinate, in millimeters, relative to bregma."""
1551
-
1552
-
1553
- @dataclass
1554
- class InjectionData:
1555
- """Stores the information about a single injection performed during surgical intervention.
1556
-
1557
- Multiple InjectionData instances are used at the same time if the surgery involved multiple injections.
1558
- """
1559
-
1560
- injection: str
1561
- """The descriptive name of the injection."""
1562
- injection_target: str
1563
- """The name of the brain region targeted by the injection."""
1564
- injection_volume_nl: float
1565
- """The volume of substance, in nanoliters, delivered during the injection."""
1566
- injection_code: int
1567
- """The manufacturer code or internal reference code for the injected substance. This code is used to identify the
1568
- substance in additional datasheets and lab ordering documents."""
1569
- injection_ap_coordinate_mm: float
1570
- """Stores injection's antero-posterior stereotactic coordinate, in millimeters, relative to bregma."""
1571
- injection_ml_coordinate_mm: float
1572
- """Stores injection's medial-lateral stereotactic coordinate, in millimeters, relative to bregma."""
1573
- injection_dv_coordinate_mm: float
1574
- """Stores injection's dorsal-ventral stereotactic coordinate, in millimeters, relative to bregma."""
1575
-
1576
-
1577
- @dataclass
1578
- class DrugData:
1579
- """Stores the information about all drugs administered to the subject before, during, and immediately after the
1580
- surgical intervention.
1581
- """
1582
-
1583
- lactated_ringers_solution_volume_ml: float
1584
- """Stores the volume of Lactated Ringer's Solution (LRS) administered during surgery, in ml."""
1585
- lactated_ringers_solution_code: int
1586
- """Stores the manufacturer code or internal reference code for Lactated Ringer's Solution (LRS). This code is used
1587
- to identify the LRS batch in additional datasheets and lab ordering documents."""
1588
- ketoprofen_volume_ml: float
1589
- """Stores the volume of ketoprofen diluted with saline administered during surgery, in ml."""
1590
- ketoprofen_code: int
1591
- """Stores the manufacturer code or internal reference code for ketoprofen. This code is used to identify the
1592
- ketoprofen batch in additional datasheets and lab ordering documents."""
1593
- buprenorphine_volume_ml: float
1594
- """Stores the volume of buprenorphine diluted with saline administered during surgery, in ml."""
1595
- buprenorphine_code: int
1596
- """Stores the manufacturer code or internal reference code for buprenorphine. This code is used to identify the
1597
- buprenorphine batch in additional datasheets and lab ordering documents."""
1598
- dexamethasone_volume_ml: float
1599
- """Stores the volume of dexamethasone diluted with saline administered during surgery, in ml."""
1600
- dexamethasone_code: int
1601
- """Stores the manufacturer code or internal reference code for dexamethasone. This code is used to identify the
1602
- dexamethasone batch in additional datasheets and lab ordering documents."""
1603
-
1604
-
1605
- @dataclass
1606
- class SurgeryData(YamlConfig):
1607
- """Stores the data about a single mouse surgical intervention.
1608
-
1609
- This class aggregates other dataclass instances that store specific data about the surgical procedure. Primarily, it
1610
- is used to save the data as a .yaml file to every session's raw_data directory of each animal used in every lab
1611
- project. This way, the surgery data is always stored alongside the behavior and brain activity data collected
1612
- during the session.
1613
- """
1614
-
1615
- subject: SubjectData
1616
- """Stores the ID information about the subject (mouse)."""
1617
- procedure: ProcedureData
1618
- """Stores general data about the surgical intervention."""
1619
- drugs: DrugData
1620
- """Stores the data about the substances subcutaneously injected into the subject before, during and immediately
1621
- after the surgical intervention."""
1622
- implants: list[ImplantData]
1623
- """Stores the data for all cranial and transcranial implants introduced to the subject during the surgical
1624
- intervention."""
1625
- injections: list[InjectionData]
1626
- """Stores the data about all substances infused into the brain of the subject during the surgical intervention."""
1627
-
1628
-
1629
- @dataclass()
1630
- class ProcessingTracker(YamlConfig):
1631
- """Tracks the data processing status for a single session.
1632
-
1633
- This class is used during BioHPC-server data processing runtimes to track which processing steps are enabled and
1634
- have been successfully applied to a given session. This is used to optimize data processing and avoid unnecessary
1635
- processing step repetitions where possible.
1636
-
1637
- Notes:
1638
- This class uses a similar mechanism for determining whether a particular option is enabled as the
1639
- HardwareConfiguration class. Specifically, if any field of the class is set to None (null), the processing
1640
- associated with that field is disabled. Otherwise, if the field is False, that session has not been processed
1641
- and, if True, the session has been processed.
1642
- """
1643
-
1644
- checksum: bool | None = None
1645
- """Tracks whether session data integrity has been verified using checksum recalculation method. This step should
1646
- be enabled for all sessions to ensure their data was transmitted intact."""
1647
- log_extractions: bool | None = None
1648
- """Tracks whether session's behavior and runtime logs have been parsed to extract the relevant data. This step
1649
- should be enabled for all sessions other than the 'Window checking' session type, which does not generate any log
1650
- data."""
1651
- suite2p: bool | None = None
1652
- """Tracks whether the Mesoscope-acquired brain activity data has been processed (registered) using sl-suite2p.
1653
- This step should eb enabled for all experiment sessions that collect brain activity data."""
1654
- deeplabcut: bool | None = None
1655
- """Tracks whether session's videos have been processed using DeepLabCut to extract pose estimation and various
1656
- animal body part tracking. This step should only be enabled for projects that need to track this data."""