sl-shared-assets 1.0.0rc18__py3-none-any.whl → 1.0.0rc20__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of sl-shared-assets might be problematic. Click here for more details.
- sl_shared_assets/__init__.pyi +71 -0
- sl_shared_assets/cli.pyi +28 -0
- sl_shared_assets/data_classes/__init__.pyi +61 -0
- sl_shared_assets/data_classes/configuration_data.pyi +37 -0
- sl_shared_assets/data_classes/runtime_data.py +12 -0
- sl_shared_assets/data_classes/runtime_data.pyi +148 -0
- sl_shared_assets/data_classes/session_data.py +12 -9
- sl_shared_assets/data_classes/session_data.pyi +544 -0
- sl_shared_assets/data_classes/surgery_data.pyi +89 -0
- sl_shared_assets/server/__init__.pyi +8 -0
- sl_shared_assets/server/job.pyi +94 -0
- sl_shared_assets/server/server.pyi +95 -0
- sl_shared_assets/suite2p/__init__.pyi +4 -0
- sl_shared_assets/suite2p/multi_day.py +7 -8
- sl_shared_assets/suite2p/multi_day.pyi +104 -0
- sl_shared_assets/suite2p/single_day.py +5 -4
- sl_shared_assets/suite2p/single_day.pyi +220 -0
- sl_shared_assets/tools/__init__.pyi +5 -0
- sl_shared_assets/tools/ascension_tools.pyi +68 -0
- sl_shared_assets/tools/packaging_tools.pyi +52 -0
- sl_shared_assets/tools/transfer_tools.pyi +53 -0
- {sl_shared_assets-1.0.0rc18.dist-info → sl_shared_assets-1.0.0rc20.dist-info}/METADATA +1 -1
- sl_shared_assets-1.0.0rc20.dist-info/RECORD +40 -0
- sl_shared_assets-1.0.0rc18.dist-info/RECORD +0 -23
- {sl_shared_assets-1.0.0rc18.dist-info → sl_shared_assets-1.0.0rc20.dist-info}/WHEEL +0 -0
- {sl_shared_assets-1.0.0rc18.dist-info → sl_shared_assets-1.0.0rc20.dist-info}/entry_points.txt +0 -0
- {sl_shared_assets-1.0.0rc18.dist-info → sl_shared_assets-1.0.0rc20.dist-info}/licenses/LICENSE +0 -0
|
@@ -0,0 +1,544 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
from dataclasses import field, dataclass
|
|
3
|
+
|
|
4
|
+
from _typeshed import Incomplete
|
|
5
|
+
from ataraxis_data_structures import YamlConfig
|
|
6
|
+
|
|
7
|
+
from .configuration_data import ExperimentConfiguration as ExperimentConfiguration
|
|
8
|
+
|
|
9
|
+
def replace_root_path(path: Path) -> None:
|
|
10
|
+
"""Replaces the path to the local root directory used to store all Sun lab projects with the provided path.
|
|
11
|
+
|
|
12
|
+
The first time ProjectConfiguration class is instantiated to create a new project on a new machine,
|
|
13
|
+
it asks the user to provide the path to the local directory where to save all Sun lab projects. This path is then
|
|
14
|
+
stored inside the default user data directory as a .yaml file to be reused for all future projects. To support
|
|
15
|
+
replacing this path without searching for the user data directory, which is usually hidden, this function finds and
|
|
16
|
+
updates the contents of the file that stores the local root path.
|
|
17
|
+
|
|
18
|
+
Args:
|
|
19
|
+
path: The path to the new local root directory.
|
|
20
|
+
"""
|
|
21
|
+
@dataclass()
|
|
22
|
+
class ProjectConfiguration(YamlConfig):
|
|
23
|
+
"""Stores the project-specific configuration parameters that do not change between different animals and runtime
|
|
24
|
+
sessions.
|
|
25
|
+
|
|
26
|
+
An instance of this class is generated and saved as a .yaml file in the \'configuration\' directory of each project
|
|
27
|
+
when it is created. After that, the stored data is reused for every runtime (training or experiment session) carried
|
|
28
|
+
out for each animal of the project. Additionally, a copy of the most actual configuration file is saved inside each
|
|
29
|
+
runtime session\'s \'raw_data\' folder, providing seamless integration between the managed data and various Sun lab
|
|
30
|
+
(sl-) libraries.
|
|
31
|
+
|
|
32
|
+
Notes:
|
|
33
|
+
Together with SessionData, this class forms the entry point for all interactions with the data acquired in the
|
|
34
|
+
Sun lab. The fields of this class are used to flexibly configure the runtime behavior of major data acquisition
|
|
35
|
+
(sl-experiment) and processing (sl-forgery) libraries, adapting them for any project in the lab.
|
|
36
|
+
|
|
37
|
+
Most lab projects only need to adjust the "surgery_sheet_id" and "water_log_sheet_id" fields of the class. Most
|
|
38
|
+
fields in this class are used by the sl-experiment library to generate the SessionData class instance for each
|
|
39
|
+
session and during experiment data acquisition and preprocessing. Data processing pipelines use specialized
|
|
40
|
+
configuration files stored in other modules of this library.
|
|
41
|
+
|
|
42
|
+
Although all path fields use str | Path datatype, they are always stored as Path objects. These fields are
|
|
43
|
+
converted to strings only when the data is dumped as a .yaml file.
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
project_name: str = ...
|
|
47
|
+
surgery_sheet_id: str = ...
|
|
48
|
+
water_log_sheet_id: str = ...
|
|
49
|
+
google_credentials_path: str | Path = ...
|
|
50
|
+
server_credentials_path: str | Path = ...
|
|
51
|
+
local_root_directory: str | Path = ...
|
|
52
|
+
local_server_directory: str | Path = ...
|
|
53
|
+
local_nas_directory: str | Path = ...
|
|
54
|
+
local_mesoscope_directory: str | Path = ...
|
|
55
|
+
local_server_working_directory: str | Path = ...
|
|
56
|
+
remote_storage_directory: str | Path = ...
|
|
57
|
+
remote_working_directory: str | Path = ...
|
|
58
|
+
face_camera_index: int = ...
|
|
59
|
+
left_camera_index: int = ...
|
|
60
|
+
right_camera_index: int = ...
|
|
61
|
+
harvesters_cti_path: str | Path = ...
|
|
62
|
+
actor_port: str = ...
|
|
63
|
+
sensor_port: str = ...
|
|
64
|
+
encoder_port: str = ...
|
|
65
|
+
headbar_port: str = ...
|
|
66
|
+
lickport_port: str = ...
|
|
67
|
+
unity_ip: str = ...
|
|
68
|
+
unity_port: int = ...
|
|
69
|
+
valve_calibration_data: dict[int | float, int | float] | tuple[tuple[int | float, int | float], ...] = ...
|
|
70
|
+
@classmethod
|
|
71
|
+
def load(cls, project_name: str, configuration_path: None | Path = None) -> ProjectConfiguration:
|
|
72
|
+
"""Loads the project configuration parameters from a project_configuration.yaml file.
|
|
73
|
+
|
|
74
|
+
This method is called during each interaction with any runtime session's data, including the creation of a new
|
|
75
|
+
session. When this method is called for a non-existent (new) project name, it generates the default
|
|
76
|
+
configuration file and prompts the user to update the configuration before proceeding with the runtime. All
|
|
77
|
+
future interactions with the sessions from this project reuse the existing configuration file.
|
|
78
|
+
|
|
79
|
+
Notes:
|
|
80
|
+
As part of its runtime, the method may prompt the user to provide the path to the local root directory.
|
|
81
|
+
This directory stores all project subdirectories and acts as the top level of the Sun lab data hierarchy.
|
|
82
|
+
The path to the directory is then saved inside user's default data directory, so that it can be reused for
|
|
83
|
+
all future projects. Use sl-replace-root CLI to replace the saved root directory path.
|
|
84
|
+
|
|
85
|
+
Since this class is used for all Sun lab data structure interactions, this method supports multiple ways of
|
|
86
|
+
loading class data. If this method is called as part of the sl-experiment new session creation pipeline, use
|
|
87
|
+
'project_name' argument. If this method is called as part of the sl-forgery data processing pipeline(s), use
|
|
88
|
+
'configuration_path' argument.
|
|
89
|
+
|
|
90
|
+
Args:
|
|
91
|
+
project_name: The name of the project whose configuration file needs to be discovered and loaded or, if the
|
|
92
|
+
project does not exist, created.
|
|
93
|
+
configuration_path: Optional. The path to the project_configuration.yaml file from which to load the data.
|
|
94
|
+
This way of resolving the configuration data source always takes precedence over the project_name when
|
|
95
|
+
both are provided.
|
|
96
|
+
|
|
97
|
+
Returns:
|
|
98
|
+
The initialized ProjectConfiguration instance that stores the configuration data for the target project.
|
|
99
|
+
"""
|
|
100
|
+
def save(self, path: Path) -> None:
|
|
101
|
+
"""Saves class instance data to disk as a project_configuration.yaml file.
|
|
102
|
+
|
|
103
|
+
This method is automatically called when a new project is created. After this method's runtime, all future
|
|
104
|
+
calls to the load() method will reuse the configuration data saved to the .yaml file.
|
|
105
|
+
|
|
106
|
+
Notes:
|
|
107
|
+
When this method is used to generate the configuration .yaml file for a new project, it also generates the
|
|
108
|
+
example 'default_experiment.yaml'. This file is designed to showcase how to write ExperimentConfiguration
|
|
109
|
+
data files that are used to control Mesoscope-VR system states during experiment session runtimes.
|
|
110
|
+
|
|
111
|
+
Args:
|
|
112
|
+
path: The path to the .yaml file to save the data to.
|
|
113
|
+
"""
|
|
114
|
+
def _verify_data(self) -> None:
|
|
115
|
+
"""Verifies the user-modified data loaded from the project_configuration.yaml file.
|
|
116
|
+
|
|
117
|
+
Since this class is explicitly designed to be modified by the user, this verification step is carried out to
|
|
118
|
+
ensure that the loaded data matches expectations. This reduces the potential for user errors to impact the
|
|
119
|
+
runtime behavior of the libraries using this class. This internal method is automatically called by the load()
|
|
120
|
+
method.
|
|
121
|
+
|
|
122
|
+
Notes:
|
|
123
|
+
The method does not verify all fields loaded from the configuration file and instead focuses on fields that
|
|
124
|
+
do not have valid default values. Since these fields are expected to be frequently modified by users, they
|
|
125
|
+
are the ones that require additional validation.
|
|
126
|
+
|
|
127
|
+
Raises:
|
|
128
|
+
ValueError: If the loaded data does not match expected formats or values.
|
|
129
|
+
"""
|
|
130
|
+
|
|
131
|
+
@dataclass()
|
|
132
|
+
class RawData:
|
|
133
|
+
"""Stores the paths to the directories and files that make up the 'raw_data' session-specific directory.
|
|
134
|
+
|
|
135
|
+
The raw_data directory stores the data acquired during the session runtime before and after preprocessing. Since
|
|
136
|
+
preprocessing does not alter the data, any data in that folder is considered 'raw'. The raw_data folder is initially
|
|
137
|
+
created on the VRPC and, after preprocessing, is copied to the BioHPC server and the Synology NAS for long-term
|
|
138
|
+
storage and further processing.
|
|
139
|
+
"""
|
|
140
|
+
|
|
141
|
+
raw_data_path: Path = ...
|
|
142
|
+
camera_data_path: Path = ...
|
|
143
|
+
mesoscope_data_path: Path = ...
|
|
144
|
+
behavior_data_path: Path = ...
|
|
145
|
+
zaber_positions_path: Path = ...
|
|
146
|
+
session_descriptor_path: Path = ...
|
|
147
|
+
hardware_configuration_path: Path = ...
|
|
148
|
+
surgery_metadata_path: Path = ...
|
|
149
|
+
project_configuration_path: Path = ...
|
|
150
|
+
session_data_path: Path = ...
|
|
151
|
+
experiment_configuration_path: Path = ...
|
|
152
|
+
mesoscope_positions_path: Path = ...
|
|
153
|
+
window_screenshot_path: Path = ...
|
|
154
|
+
telomere_path: Path = ...
|
|
155
|
+
checksum_path: Path = ...
|
|
156
|
+
def resolve_paths(self, root_directory_path: Path) -> None:
|
|
157
|
+
"""Resolves all paths managed by the class instance based on the input root directory path.
|
|
158
|
+
|
|
159
|
+
This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
|
|
160
|
+
machine that instantiates the class.
|
|
161
|
+
|
|
162
|
+
Args:
|
|
163
|
+
root_directory_path: The path to the top-level directory of the local hierarchy. Depending on the managed
|
|
164
|
+
hierarchy, this has to point to a directory under the main /session, /animal, or /project directory of
|
|
165
|
+
the managed session.
|
|
166
|
+
"""
|
|
167
|
+
def make_directories(self) -> None:
|
|
168
|
+
"""Ensures that all major subdirectories and the root directory exist."""
|
|
169
|
+
|
|
170
|
+
@dataclass()
|
|
171
|
+
class DeepLabCutData:
|
|
172
|
+
"""Stores the paths to the directories and files that make up the 'deeplabcut' project-specific directory.
|
|
173
|
+
|
|
174
|
+
DeepLabCut (DLC) is used to track animal body parts and poses in video data acquired during experiment and training
|
|
175
|
+
sessions. Since DLC is designed to work with projects, rather than single animals or sessions, each Sun lab
|
|
176
|
+
project data hierarchy contains a dedicated 'deeplabcut' directory under the root project directory. The contents of
|
|
177
|
+
that directory are largely managed by the DLC itself. Therefore, each session of a given project refers to and
|
|
178
|
+
uses the same 'deeplabcut' directory.
|
|
179
|
+
"""
|
|
180
|
+
|
|
181
|
+
deeplabcut_path: Path = ...
|
|
182
|
+
def resolve_paths(self, root_directory_path: Path) -> None:
|
|
183
|
+
"""Resolves all paths managed by the class instance based on the input root directory path.
|
|
184
|
+
|
|
185
|
+
This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
|
|
186
|
+
machine that instantiates the class.
|
|
187
|
+
|
|
188
|
+
Args:
|
|
189
|
+
root_directory_path: The path to the top-level directory of the local hierarchy. Depending on the managed
|
|
190
|
+
hierarchy, this has to point to a directory under the main /session, /animal, or /project directory of
|
|
191
|
+
the managed session.
|
|
192
|
+
"""
|
|
193
|
+
def make_directories(self) -> None:
|
|
194
|
+
"""Ensures that all major subdirectories and the root directory exist."""
|
|
195
|
+
|
|
196
|
+
@dataclass()
|
|
197
|
+
class ConfigurationData:
|
|
198
|
+
"""Stores the paths to the directories and files that make up the 'configuration' project-specific directory.
|
|
199
|
+
|
|
200
|
+
The configuration directory contains various configuration files and settings used by data acquisition,
|
|
201
|
+
preprocessing, and processing pipelines in the lab. Generally, all configuration settings are defined once for each
|
|
202
|
+
project and are reused for every session within the project. Therefore, this directory is created under each main
|
|
203
|
+
project directory.
|
|
204
|
+
|
|
205
|
+
Notes:
|
|
206
|
+
Some attribute names inside this section match the names in the RawData section. This is intentional, as some
|
|
207
|
+
configuration files are copied into the raw_data session directories to allow reinstating the session data
|
|
208
|
+
hierarchy across machines.
|
|
209
|
+
"""
|
|
210
|
+
|
|
211
|
+
configuration_path: Path = ...
|
|
212
|
+
experiment_configuration_path: Path = ...
|
|
213
|
+
project_configuration_path: Path = ...
|
|
214
|
+
single_day_s2p_configuration_path: Path = ...
|
|
215
|
+
multi_day_s2p_configuration_path: Path = ...
|
|
216
|
+
def resolve_paths(self, root_directory_path: Path, experiment_name: str | None = None) -> None:
|
|
217
|
+
"""Resolves all paths managed by the class instance based on the input root directory path.
|
|
218
|
+
|
|
219
|
+
This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
|
|
220
|
+
machine that instantiates the class.
|
|
221
|
+
|
|
222
|
+
Args:
|
|
223
|
+
root_directory_path: The path to the top-level directory of the local hierarchy. Depending on the managed
|
|
224
|
+
hierarchy, this has to point to a directory under the main /session, /animal, or /project directory of
|
|
225
|
+
the managed session.
|
|
226
|
+
experiment_name: Optionally specifies the name of the experiment executed as part of the managed session's
|
|
227
|
+
runtime. This is used to correctly configure the path to the specific ExperimentConfiguration data file.
|
|
228
|
+
If the managed session is not an Experiment session, this parameter should be set to None.
|
|
229
|
+
"""
|
|
230
|
+
def make_directories(self) -> None:
|
|
231
|
+
"""Ensures that all major subdirectories and the root directory exist."""
|
|
232
|
+
|
|
233
|
+
@dataclass()
|
|
234
|
+
class ProcessedData:
|
|
235
|
+
"""Stores the paths to the directories and files that make up the 'processed_data' session-specific directory.
|
|
236
|
+
|
|
237
|
+
The processed_data directory stores the data generated by various processing pipelines from the raw data (contents
|
|
238
|
+
of the raw_data directory). Processed data represents an intermediate step between raw data and the dataset used in
|
|
239
|
+
the data analysis, but is not itself designed to be analyzed.
|
|
240
|
+
|
|
241
|
+
Notes:
|
|
242
|
+
The paths from this section are typically used only on the BioHPC server. This is because most data processing
|
|
243
|
+
in the lab is performed using the processing server's resources. On the server, processed data is stored on
|
|
244
|
+
the fast (NVME) drive volume, in contrast to raw data, which is stored on the slow (SSD) drive volume.
|
|
245
|
+
|
|
246
|
+
When this class is instantiated on a machine other than BioHPC server, for example, to test processing
|
|
247
|
+
pipelines, it uses the same drive as the raw_data folder to create the processed_data folder. This relies on the
|
|
248
|
+
assumption that non-server machines in the lab only use fast NVME drives, so there is no need to separate
|
|
249
|
+
storage and processing volumes.
|
|
250
|
+
"""
|
|
251
|
+
|
|
252
|
+
processed_data_path: Path = ...
|
|
253
|
+
camera_data_path: Path = ...
|
|
254
|
+
mesoscope_data_path: Path = ...
|
|
255
|
+
behavior_data_path: Path = ...
|
|
256
|
+
job_logs_path: Path = ...
|
|
257
|
+
project_configuration_path: Path = ...
|
|
258
|
+
session_data_path: Path = ...
|
|
259
|
+
def resolve_paths(self, root_directory_path: Path) -> None:
|
|
260
|
+
"""Resolves all paths managed by the class instance based on the input root directory path.
|
|
261
|
+
|
|
262
|
+
This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
|
|
263
|
+
machine that instantiates the class.
|
|
264
|
+
|
|
265
|
+
Args:
|
|
266
|
+
root_directory_path: The path to the top-level directory of the local hierarchy. Depending on the managed
|
|
267
|
+
hierarchy, this has to point to a directory under the main /session, /animal, or /project directory of
|
|
268
|
+
the managed session.
|
|
269
|
+
"""
|
|
270
|
+
def make_directories(self) -> None:
|
|
271
|
+
"""Ensures that all major subdirectories and the root directory exist."""
|
|
272
|
+
|
|
273
|
+
@dataclass()
|
|
274
|
+
class VRPCPersistentData:
|
|
275
|
+
"""Stores the paths to the directories and files that make up the 'persistent_data' directory on the VRPC.
|
|
276
|
+
|
|
277
|
+
Persistent data directories are only used during data acquisition. Therefore, unlike most other directories, they
|
|
278
|
+
are purposefully designed for specific PCs that participate in data acquisition. This section manages the
|
|
279
|
+
animal-specific persistent_data directory stored on the VRPC.
|
|
280
|
+
|
|
281
|
+
VRPC persistent data directory is used to preserve configuration data, such as the positions of Zaber motors and
|
|
282
|
+
Meososcope objective, so that they can be reused across sessions of the same animals. The data in this directory
|
|
283
|
+
is read at the beginning of each session and replaced at the end of each session.
|
|
284
|
+
"""
|
|
285
|
+
|
|
286
|
+
persistent_data_path: Path = ...
|
|
287
|
+
zaber_positions_path: Path = ...
|
|
288
|
+
mesoscope_positions_path: Path = ...
|
|
289
|
+
def resolve_paths(self, root_directory_path: Path) -> None:
|
|
290
|
+
"""Resolves all paths managed by the class instance based on the input root directory path.
|
|
291
|
+
|
|
292
|
+
This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
|
|
293
|
+
machine that instantiates the class.
|
|
294
|
+
|
|
295
|
+
Args:
|
|
296
|
+
root_directory_path: The path to the top-level directory of the local hierarchy. Depending on the managed
|
|
297
|
+
hierarchy, this has to point to a directory under the main /session, /animal, or /project directory of
|
|
298
|
+
the managed session.
|
|
299
|
+
"""
|
|
300
|
+
def make_directories(self) -> None:
|
|
301
|
+
"""Ensures that all major subdirectories and the root directory exist."""
|
|
302
|
+
|
|
303
|
+
@dataclass()
|
|
304
|
+
class ScanImagePCPersistentData:
|
|
305
|
+
"""Stores the paths to the directories and files that make up the 'persistent_data' directory on the ScanImagePC.
|
|
306
|
+
|
|
307
|
+
Persistent data directories are only used during data acquisition. Therefore, unlike most other directories, they
|
|
308
|
+
are purposefully designed for specific PCs that participate in data acquisition. This section manages the
|
|
309
|
+
animal-specific persistent_data directory stored on the ScanImagePC (Mesoscope PC).
|
|
310
|
+
|
|
311
|
+
ScanImagePC persistent data directory is used to preserve the motion estimation snapshot, generated during the first
|
|
312
|
+
experiment session. This is necessary to align the brain recording field of view across sessions. In turn, this
|
|
313
|
+
is used to carry out 'online' motion and z-drift correction, improving the accuracy of across-day (multi-day)
|
|
314
|
+
cell tracking.
|
|
315
|
+
"""
|
|
316
|
+
|
|
317
|
+
persistent_data_path: Path = ...
|
|
318
|
+
motion_estimator_path: Path = ...
|
|
319
|
+
def resolve_paths(self, root_directory_path: Path) -> None:
|
|
320
|
+
"""Resolves all paths managed by the class instance based on the input root directory path.
|
|
321
|
+
|
|
322
|
+
This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
|
|
323
|
+
machine that instantiates the class.
|
|
324
|
+
|
|
325
|
+
Args:
|
|
326
|
+
root_directory_path: The path to the top-level directory of the local hierarchy. Depending on the managed
|
|
327
|
+
hierarchy, this has to point to a directory under the main /session, /animal, or /project directory of
|
|
328
|
+
the managed session.
|
|
329
|
+
"""
|
|
330
|
+
def make_directories(self) -> None:
|
|
331
|
+
"""Ensures that all major subdirectories and the root directory exist."""
|
|
332
|
+
|
|
333
|
+
@dataclass()
|
|
334
|
+
class MesoscopeData:
|
|
335
|
+
"""Stores the paths to the directories and files that make up the 'meso_data' directory on the ScanImagePC.
|
|
336
|
+
|
|
337
|
+
The meso_data directory is the root directory where all mesoscope-generated data is stored on the ScanImagePC. The
|
|
338
|
+
path to this directory should be given relative to the VRPC root and be mounted to the VRPC filesystem via the
|
|
339
|
+
SMB or equivalent protocol.
|
|
340
|
+
|
|
341
|
+
During runtime, the ScanImagePC should organize all collected data under this root directory. During preprocessing,
|
|
342
|
+
the VRPC uses SMB to access the data in this directory and merge it into the 'raw_data' session directory. The paths
|
|
343
|
+
in this section, therefore, are specific to the VRPC and are not used on other PCs.
|
|
344
|
+
"""
|
|
345
|
+
|
|
346
|
+
meso_data_path: Path = ...
|
|
347
|
+
mesoscope_data_path: Path = ...
|
|
348
|
+
session_specific_path: Path = ...
|
|
349
|
+
ubiquitin_path: Path = ...
|
|
350
|
+
def resolve_paths(self, root_mesoscope_path: Path, session_name: str) -> None:
|
|
351
|
+
"""Resolves all paths managed by the class instance based on the input root directory path.
|
|
352
|
+
|
|
353
|
+
This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
|
|
354
|
+
machine that instantiates the class.
|
|
355
|
+
|
|
356
|
+
Args:
|
|
357
|
+
root_mesoscope_path: The path to the top-level directory of the ScanImagePC data hierarchy mounted to the
|
|
358
|
+
VRPC via the SMB or equivalent protocol.
|
|
359
|
+
session_name: The name of the session for which this subclass is initialized.
|
|
360
|
+
"""
|
|
361
|
+
def make_directories(self) -> None:
|
|
362
|
+
"""Ensures that all major subdirectories and the root directory exist."""
|
|
363
|
+
|
|
364
|
+
@dataclass()
|
|
365
|
+
class VRPCDestinations:
|
|
366
|
+
"""Stores the paths to the VRPC filesystem-mounted directories of the Synology NAS and BioHPC server.
|
|
367
|
+
|
|
368
|
+
The paths from this section are primarily used to transfer preprocessed data to the long-term storage destinations.
|
|
369
|
+
Additionally, they allow VRPC to interface with the configuration directory of the BioHPC server to start data
|
|
370
|
+
processing jobs and to read the data from the processed_data directory to remove redundant data from the VRPC
|
|
371
|
+
filesystem.
|
|
372
|
+
|
|
373
|
+
Overall, this section is intended solely for the VRPC and should not be used on other PCs.
|
|
374
|
+
"""
|
|
375
|
+
|
|
376
|
+
nas_raw_data_path: Path = ...
|
|
377
|
+
server_raw_data_path: Path = ...
|
|
378
|
+
server_processed_data_path: Path = ...
|
|
379
|
+
server_configuration_path: Path = ...
|
|
380
|
+
telomere_path: Path = ...
|
|
381
|
+
suite2p_configuration_path: Path = ...
|
|
382
|
+
processing_tracker_path: Path = ...
|
|
383
|
+
multiday_configuration_path: Path = ...
|
|
384
|
+
def resolve_paths(
|
|
385
|
+
self,
|
|
386
|
+
nas_raw_data_path: Path,
|
|
387
|
+
server_raw_data_path: Path,
|
|
388
|
+
server_processed_data_path: Path,
|
|
389
|
+
server_configuration_path: Path,
|
|
390
|
+
) -> None:
|
|
391
|
+
"""Resolves all paths managed by the class instance based on the input root directory paths.
|
|
392
|
+
|
|
393
|
+
This method is called each time the class is instantiated to regenerate the managed path hierarchy on any
|
|
394
|
+
machine that instantiates the class.
|
|
395
|
+
|
|
396
|
+
Args:
|
|
397
|
+
nas_raw_data_path: The path to the session's raw_data directory on the Synology NAS, relative to the VRPC
|
|
398
|
+
filesystem root.
|
|
399
|
+
server_raw_data_path: The path to the session's raw_data directory on the BioHPC server, relative to the
|
|
400
|
+
VRPC filesystem root.
|
|
401
|
+
server_processed_data_path: The path to the session's processed_data directory on the BioHPC server,
|
|
402
|
+
relative to the VRPC filesystem root.
|
|
403
|
+
server_configuration_path: The path to the project-specific 'configuration' directory on the BioHPC server,
|
|
404
|
+
relative to the VRPC filesystem root.
|
|
405
|
+
"""
|
|
406
|
+
def make_directories(self) -> None:
|
|
407
|
+
"""Ensures that all major subdirectories and the root directory exist."""
|
|
408
|
+
|
|
409
|
+
@dataclass
|
|
410
|
+
class SessionData(YamlConfig):
|
|
411
|
+
"""Stores and manages the data layout of a single training or experiment session acquired using the Sun lab
|
|
412
|
+
Mesoscope-VR system.
|
|
413
|
+
|
|
414
|
+
The primary purpose of this class is to maintain the session data structure across all supported destinations and
|
|
415
|
+
during all processing stages. It generates the paths used by all other classes from all Sun lab libraries that
|
|
416
|
+
interact with the session's data from the point of its creation and until the data is integrated into an
|
|
417
|
+
analysis dataset.
|
|
418
|
+
|
|
419
|
+
When necessary, the class can be used to either generate a new session or load the layout of an already existing
|
|
420
|
+
session. When the class is used to create a new session, it generates the new session's name using the current
|
|
421
|
+
UTC timestamp, accurate to microseconds. This ensures that each session name is unique and preserves the overall
|
|
422
|
+
session order.
|
|
423
|
+
|
|
424
|
+
Notes:
|
|
425
|
+
If this class is instantiated on the VRPC, it is expected that the BioHPC server, Synology NAS, and ScanImagePC
|
|
426
|
+
data directories are mounted on the local filesystem via the SMB or equivalent protocol. All manipulations
|
|
427
|
+
with these destinations are carried out with the assumption that the local OS has full access to these
|
|
428
|
+
directories and filesystems.
|
|
429
|
+
|
|
430
|
+
This class is specifically designed for working with the data from a single session, performed by a single
|
|
431
|
+
animal under the specific experiment. The class is used to manage both raw and processed data. It follows the
|
|
432
|
+
data through acquisition, preprocessing and processing stages of the Sun lab data workflow. Together with
|
|
433
|
+
ProjectConfiguration class, this class serves as an entry point for all interactions with the managed session's
|
|
434
|
+
data.
|
|
435
|
+
"""
|
|
436
|
+
|
|
437
|
+
project_name: str
|
|
438
|
+
animal_id: str
|
|
439
|
+
session_name: str
|
|
440
|
+
session_type: str
|
|
441
|
+
experiment_name: str | None
|
|
442
|
+
raw_data: RawData = field(default_factory=Incomplete)
|
|
443
|
+
processed_data: ProcessedData = field(default_factory=Incomplete)
|
|
444
|
+
deeplabcut_data: DeepLabCutData = field(default_factory=Incomplete)
|
|
445
|
+
configuration_data: ConfigurationData = field(default_factory=Incomplete)
|
|
446
|
+
vrpc_persistent_data: VRPCPersistentData = field(default_factory=Incomplete)
|
|
447
|
+
scanimagepc_persistent_data: ScanImagePCPersistentData = field(default_factory=Incomplete)
|
|
448
|
+
mesoscope_data: MesoscopeData = field(default_factory=Incomplete)
|
|
449
|
+
destinations: VRPCDestinations = field(default_factory=Incomplete)
|
|
450
|
+
@classmethod
|
|
451
|
+
def create(
|
|
452
|
+
cls,
|
|
453
|
+
animal_id: str,
|
|
454
|
+
session_type: str,
|
|
455
|
+
project_configuration: ProjectConfiguration,
|
|
456
|
+
experiment_name: str | None = None,
|
|
457
|
+
session_name: str | None = None,
|
|
458
|
+
) -> SessionData:
|
|
459
|
+
"""Creates a new SessionData object and generates the new session's data structure.
|
|
460
|
+
|
|
461
|
+
This method is called by sl-experiment runtimes that create new training or experiment sessions to generate the
|
|
462
|
+
session data directory tree. It always assumes it is called on the VRPC and, as part of its runtime, resolves
|
|
463
|
+
and generates the necessary local and ScanImagePC directories to support acquiring and preprocessing session's
|
|
464
|
+
data.
|
|
465
|
+
|
|
466
|
+
Notes:
|
|
467
|
+
To load an already existing session data structure, use the load() method instead.
|
|
468
|
+
|
|
469
|
+
This method automatically dumps the data of the created SessionData instance into the session_data.yaml file
|
|
470
|
+
inside the root raw_data directory of the created hierarchy. It also finds and dumps other configuration
|
|
471
|
+
files, such as project_configuration.yaml and experiment_configuration.yaml, into the same raw_data
|
|
472
|
+
directory. This ensures that if the session's runtime is interrupted unexpectedly, the acquired data can
|
|
473
|
+
still be processed.
|
|
474
|
+
|
|
475
|
+
Args:
|
|
476
|
+
animal_id: The ID code of the animal for which the data is acquired.
|
|
477
|
+
session_type: The type of the session. Primarily, this determines how to read the session_descriptor.yaml
|
|
478
|
+
file. Valid options are 'Lick training', 'Run training', 'Window checking', or 'Experiment'.
|
|
479
|
+
experiment_name: The name of the experiment executed during managed session. This optional argument is only
|
|
480
|
+
used for 'Experiment' session types. It is used to find the experiment configuration .YAML file.
|
|
481
|
+
project_configuration: The initialized ProjectConfiguration instance that stores the session's project
|
|
482
|
+
configuration data. This is used to determine the root directory paths for all lab machines used during
|
|
483
|
+
data acquisition and processing.
|
|
484
|
+
session_name: An optional session_name override. Generally, this argument should not be provided for most
|
|
485
|
+
sessions. When provided, the method uses this name instead of generating a new timestamp-based name.
|
|
486
|
+
This is only used during the 'ascension' runtime to convert old data structures to the modern
|
|
487
|
+
lab standards.
|
|
488
|
+
|
|
489
|
+
Returns:
|
|
490
|
+
An initialized SessionData instance that stores the layout of the newly created session's data.
|
|
491
|
+
"""
|
|
492
|
+
@classmethod
|
|
493
|
+
def load(cls, session_path: Path, on_server: bool, make_directories: bool = True) -> SessionData:
|
|
494
|
+
"""Loads the SessionData instance from the target session's session_data.yaml file.
|
|
495
|
+
|
|
496
|
+
This method is used to load the data layout information of an already existing session. Primarily, this is used
|
|
497
|
+
when preprocessing or processing session data. Depending on the call location (machine), the method
|
|
498
|
+
automatically resolves all necessary paths and creates the necessary directories.
|
|
499
|
+
|
|
500
|
+
Notes:
|
|
501
|
+
To create a new session, use the create() method instead.
|
|
502
|
+
|
|
503
|
+
Although session_data.yaml is stored both inside raw_data and processed_data subfolders, this method
|
|
504
|
+
always searches only inside the raw_data folder. Storing session data in both folders is only used to ensure
|
|
505
|
+
human experimenters can always trace all data in the lab back to the proper project, animal, and session.
|
|
506
|
+
|
|
507
|
+
Args:
|
|
508
|
+
session_path: The path to the root directory of an existing session, e.g.: vrpc_root/project/animal/session.
|
|
509
|
+
on_server: Determines whether the method is used to initialize an existing session on the BioHPC server or
|
|
510
|
+
a non-server machine. Note, non-server runtimes use the same 'root' directory to store raw_data and
|
|
511
|
+
processed_data subfolders. BioHPC server runtimes use different volumes (drives) to store these
|
|
512
|
+
subfolders.
|
|
513
|
+
make_directories: Determines whether to attempt creating any missing directories. Generally, this option
|
|
514
|
+
is safe to be True for all destinations other than some specific BioHPC server runtimes, where some
|
|
515
|
+
data is 'owned' by a general lab account and not the user account. These cases are only present for the
|
|
516
|
+
sl-forgery library and are resolved by that library.
|
|
517
|
+
|
|
518
|
+
Returns:
|
|
519
|
+
An initialized SessionData instance for the session whose data is stored at the provided path.
|
|
520
|
+
|
|
521
|
+
Raises:
|
|
522
|
+
FileNotFoundError: If the 'session_data.yaml' file is not found under the session_path/raw_data/ subfolder.
|
|
523
|
+
"""
|
|
524
|
+
def _save(self) -> None:
|
|
525
|
+
"""Saves the instance data to the 'raw_data' directory and the 'processed_data' directory of the managed session
|
|
526
|
+
as a 'session_data.yaml' file.
|
|
527
|
+
|
|
528
|
+
This is used to save the data stored in the instance to disk, so that it can be reused during preprocessing or
|
|
529
|
+
data processing. The method is intended to only be used by the SessionData instance itself during its
|
|
530
|
+
create() method runtime.
|
|
531
|
+
"""
|
|
532
|
+
@classmethod
|
|
533
|
+
def _safe_load(cls, path: Path) -> SessionData:
|
|
534
|
+
"""Loads a SessionData class instance into memory in a way that avoids collisions with outdated SessionData
|
|
535
|
+
formats.
|
|
536
|
+
|
|
537
|
+
This method is used instead of the default method inherited from the YamlConfig class. Primarily, this is used
|
|
538
|
+
to avoid errors with old SessionData class formats that contain some data that is either no longer present or
|
|
539
|
+
cannot be loaded from YAML. Using this custom method ensures we can load any SessionData class, provided it
|
|
540
|
+
contains the required header fields.
|
|
541
|
+
|
|
542
|
+
Returns:
|
|
543
|
+
The SessionData instance initialized using the resolved header data.
|
|
544
|
+
"""
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
from ataraxis_data_structures import YamlConfig
|
|
4
|
+
|
|
5
|
+
@dataclass()
|
|
6
|
+
class SubjectData:
|
|
7
|
+
"""Stores the ID information of the surgical intervention's subject (animal)."""
|
|
8
|
+
|
|
9
|
+
id: int
|
|
10
|
+
ear_punch: str
|
|
11
|
+
sex: str
|
|
12
|
+
genotype: str
|
|
13
|
+
date_of_birth_us: int
|
|
14
|
+
weight_g: float
|
|
15
|
+
cage: int
|
|
16
|
+
location_housed: str
|
|
17
|
+
status: str
|
|
18
|
+
|
|
19
|
+
@dataclass()
|
|
20
|
+
class ProcedureData:
|
|
21
|
+
"""Stores the general information about the surgical intervention."""
|
|
22
|
+
|
|
23
|
+
surgery_start_us: int
|
|
24
|
+
surgery_end_us: int
|
|
25
|
+
surgeon: str
|
|
26
|
+
protocol: str
|
|
27
|
+
surgery_notes: str
|
|
28
|
+
post_op_notes: str
|
|
29
|
+
surgery_quality: int = ...
|
|
30
|
+
|
|
31
|
+
@dataclass
|
|
32
|
+
class ImplantData:
|
|
33
|
+
"""Stores the information about a single implantation performed during the surgical intervention.
|
|
34
|
+
|
|
35
|
+
Multiple ImplantData instances are used at the same time if the surgery involved multiple implants.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
implant: str
|
|
39
|
+
implant_target: str
|
|
40
|
+
implant_code: int
|
|
41
|
+
implant_ap_coordinate_mm: float
|
|
42
|
+
implant_ml_coordinate_mm: float
|
|
43
|
+
implant_dv_coordinate_mm: float
|
|
44
|
+
|
|
45
|
+
@dataclass
|
|
46
|
+
class InjectionData:
|
|
47
|
+
"""Stores the information about a single injection performed during surgical intervention.
|
|
48
|
+
|
|
49
|
+
Multiple InjectionData instances are used at the same time if the surgery involved multiple injections.
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
injection: str
|
|
53
|
+
injection_target: str
|
|
54
|
+
injection_volume_nl: float
|
|
55
|
+
injection_code: int
|
|
56
|
+
injection_ap_coordinate_mm: float
|
|
57
|
+
injection_ml_coordinate_mm: float
|
|
58
|
+
injection_dv_coordinate_mm: float
|
|
59
|
+
|
|
60
|
+
@dataclass
|
|
61
|
+
class DrugData:
|
|
62
|
+
"""Stores the information about all drugs administered to the subject before, during, and immediately after the
|
|
63
|
+
surgical intervention.
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
lactated_ringers_solution_volume_ml: float
|
|
67
|
+
lactated_ringers_solution_code: int
|
|
68
|
+
ketoprofen_volume_ml: float
|
|
69
|
+
ketoprofen_code: int
|
|
70
|
+
buprenorphine_volume_ml: float
|
|
71
|
+
buprenorphine_code: int
|
|
72
|
+
dexamethasone_volume_ml: float
|
|
73
|
+
dexamethasone_code: int
|
|
74
|
+
|
|
75
|
+
@dataclass
|
|
76
|
+
class SurgeryData(YamlConfig):
|
|
77
|
+
"""Stores the data about a single mouse surgical intervention.
|
|
78
|
+
|
|
79
|
+
This class aggregates other dataclass instances that store specific data about the surgical procedure. Primarily, it
|
|
80
|
+
is used to save the data as a .yaml file to every session's raw_data directory of each animal used in every lab
|
|
81
|
+
project. This way, the surgery data is always stored alongside the behavior and brain activity data collected
|
|
82
|
+
during the session.
|
|
83
|
+
"""
|
|
84
|
+
|
|
85
|
+
subject: SubjectData
|
|
86
|
+
procedure: ProcedureData
|
|
87
|
+
drugs: DrugData
|
|
88
|
+
implants: list[ImplantData]
|
|
89
|
+
injections: list[InjectionData]
|