sl-shared-assets 1.0.0rc19__py3-none-any.whl → 1.0.0rc21__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of sl-shared-assets might be problematic. Click here for more details.
- sl_shared_assets/__init__.py +27 -27
- sl_shared_assets/__init__.pyi +73 -0
- sl_shared_assets/cli.py +266 -40
- sl_shared_assets/cli.pyi +87 -0
- sl_shared_assets/data_classes/__init__.py +23 -20
- sl_shared_assets/data_classes/__init__.pyi +61 -0
- sl_shared_assets/data_classes/configuration_data.py +407 -26
- sl_shared_assets/data_classes/configuration_data.pyi +194 -0
- sl_shared_assets/data_classes/runtime_data.py +59 -41
- sl_shared_assets/data_classes/runtime_data.pyi +145 -0
- sl_shared_assets/data_classes/session_data.py +168 -914
- sl_shared_assets/data_classes/session_data.pyi +249 -0
- sl_shared_assets/data_classes/surgery_data.py +3 -3
- sl_shared_assets/data_classes/surgery_data.pyi +89 -0
- sl_shared_assets/server/__init__.pyi +8 -0
- sl_shared_assets/server/job.pyi +94 -0
- sl_shared_assets/server/server.pyi +95 -0
- sl_shared_assets/tools/__init__.py +8 -1
- sl_shared_assets/tools/__init__.pyi +15 -0
- sl_shared_assets/tools/ascension_tools.py +27 -26
- sl_shared_assets/tools/ascension_tools.pyi +68 -0
- sl_shared_assets/tools/packaging_tools.py +14 -1
- sl_shared_assets/tools/packaging_tools.pyi +56 -0
- sl_shared_assets/tools/project_management_tools.py +164 -0
- sl_shared_assets/tools/project_management_tools.pyi +48 -0
- sl_shared_assets/tools/transfer_tools.pyi +53 -0
- {sl_shared_assets-1.0.0rc19.dist-info → sl_shared_assets-1.0.0rc21.dist-info}/METADATA +21 -4
- sl_shared_assets-1.0.0rc21.dist-info/RECORD +36 -0
- sl_shared_assets-1.0.0rc21.dist-info/entry_points.txt +8 -0
- sl_shared_assets/suite2p/__init__.py +0 -8
- sl_shared_assets/suite2p/multi_day.py +0 -225
- sl_shared_assets/suite2p/single_day.py +0 -563
- sl_shared_assets-1.0.0rc19.dist-info/RECORD +0 -23
- sl_shared_assets-1.0.0rc19.dist-info/entry_points.txt +0 -4
- {sl_shared_assets-1.0.0rc19.dist-info → sl_shared_assets-1.0.0rc21.dist-info}/WHEEL +0 -0
- {sl_shared_assets-1.0.0rc19.dist-info → sl_shared_assets-1.0.0rc21.dist-info}/licenses/LICENSE +0 -0
sl_shared_assets/__init__.py
CHANGED
|
@@ -2,14 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
See https://github.com/Sun-Lab-NBB/sl-shared-assets for more details.
|
|
4
4
|
API documentation: https://sl-shared-assets-api-docs.netlify.app/
|
|
5
|
-
Authors: Ivan Kondratyev (Inkaros), Kushaan Gupta, Yuantao Deng
|
|
5
|
+
Authors: Ivan Kondratyev (Inkaros), Kushaan Gupta, Yuantao Deng, Natalie Yeung
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
8
|
from ataraxis_base_utilities import console
|
|
9
9
|
|
|
10
|
-
from .tools import transfer_directory, calculate_directory_checksum
|
|
11
|
-
from .server import Server, ServerCredentials
|
|
12
|
-
from .suite2p import MultiDayS2PConfiguration, SingleDayS2PConfiguration
|
|
10
|
+
from .tools import transfer_directory, verify_session_checksum, generate_project_manifest, calculate_directory_checksum
|
|
11
|
+
from .server import Job, Server, ServerCredentials
|
|
13
12
|
from .data_classes import (
|
|
14
13
|
RawData,
|
|
15
14
|
DrugData,
|
|
@@ -18,23 +17,24 @@ from .data_classes import (
|
|
|
18
17
|
SubjectData,
|
|
19
18
|
SurgeryData,
|
|
20
19
|
InjectionData,
|
|
21
|
-
MesoscopeData,
|
|
22
20
|
ProcedureData,
|
|
23
21
|
ProcessedData,
|
|
24
|
-
|
|
22
|
+
MesoscopePaths,
|
|
25
23
|
ZaberPositions,
|
|
26
24
|
ExperimentState,
|
|
27
|
-
|
|
28
|
-
ConfigurationData,
|
|
25
|
+
MesoscopeCameras,
|
|
29
26
|
MesoscopePositions,
|
|
30
|
-
VRPCPersistentData,
|
|
31
27
|
ProjectConfiguration,
|
|
32
|
-
HardwareConfiguration,
|
|
33
28
|
RunTrainingDescriptor,
|
|
34
29
|
LickTrainingDescriptor,
|
|
35
|
-
|
|
36
|
-
|
|
30
|
+
MesoscopeHardwareState,
|
|
31
|
+
MesoscopeMicroControllers,
|
|
32
|
+
MesoscopeAdditionalFirmware,
|
|
33
|
+
MesoscopeSystemConfiguration,
|
|
37
34
|
MesoscopeExperimentDescriptor,
|
|
35
|
+
MesoscopeExperimentConfiguration,
|
|
36
|
+
get_system_configuration_data,
|
|
37
|
+
set_system_configuration_file,
|
|
38
38
|
)
|
|
39
39
|
|
|
40
40
|
# Ensures console is enabled when this library is imported
|
|
@@ -42,24 +42,16 @@ if not console.enabled:
|
|
|
42
42
|
console.enable()
|
|
43
43
|
|
|
44
44
|
__all__ = [
|
|
45
|
-
# Server
|
|
45
|
+
# Server package
|
|
46
46
|
"Server",
|
|
47
47
|
"ServerCredentials",
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
"MultiDayS2PConfiguration",
|
|
51
|
-
# Data classes module
|
|
48
|
+
"Job",
|
|
49
|
+
# Data classes package
|
|
52
50
|
"DrugData",
|
|
53
51
|
"ImplantData",
|
|
54
52
|
"SessionData",
|
|
55
53
|
"RawData",
|
|
56
54
|
"ProcessedData",
|
|
57
|
-
"ConfigurationData",
|
|
58
|
-
"DeepLabCutData",
|
|
59
|
-
"VRPCPersistentData",
|
|
60
|
-
"ScanImagePCPersistentData",
|
|
61
|
-
"MesoscopeData",
|
|
62
|
-
"VRPCDestinations",
|
|
63
55
|
"SubjectData",
|
|
64
56
|
"SurgeryData",
|
|
65
57
|
"InjectionData",
|
|
@@ -68,13 +60,21 @@ __all__ = [
|
|
|
68
60
|
"ExperimentState",
|
|
69
61
|
"MesoscopePositions",
|
|
70
62
|
"ProjectConfiguration",
|
|
71
|
-
"
|
|
63
|
+
"MesoscopeHardwareState",
|
|
72
64
|
"RunTrainingDescriptor",
|
|
73
65
|
"LickTrainingDescriptor",
|
|
74
|
-
"
|
|
66
|
+
"MesoscopeExperimentConfiguration",
|
|
75
67
|
"MesoscopeExperimentDescriptor",
|
|
76
|
-
|
|
68
|
+
"MesoscopeSystemConfiguration",
|
|
69
|
+
"MesoscopePaths",
|
|
70
|
+
"MesoscopeCameras",
|
|
71
|
+
"MesoscopeMicroControllers",
|
|
72
|
+
"MesoscopeAdditionalFirmware",
|
|
73
|
+
"get_system_configuration_data",
|
|
74
|
+
"set_system_configuration_file",
|
|
75
|
+
# Tools package
|
|
77
76
|
"transfer_directory",
|
|
78
|
-
|
|
77
|
+
"generate_project_manifest",
|
|
78
|
+
"verify_session_checksum",
|
|
79
79
|
"calculate_directory_checksum",
|
|
80
80
|
]
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
from .tools import (
|
|
2
|
+
transfer_directory as transfer_directory,
|
|
3
|
+
verify_session_checksum as verify_session_checksum,
|
|
4
|
+
generate_project_manifest as generate_project_manifest,
|
|
5
|
+
calculate_directory_checksum as calculate_directory_checksum,
|
|
6
|
+
)
|
|
7
|
+
from .server import (
|
|
8
|
+
Job as Job,
|
|
9
|
+
Server as Server,
|
|
10
|
+
ServerCredentials as ServerCredentials,
|
|
11
|
+
)
|
|
12
|
+
from .data_classes import (
|
|
13
|
+
RawData as RawData,
|
|
14
|
+
DrugData as DrugData,
|
|
15
|
+
ImplantData as ImplantData,
|
|
16
|
+
SessionData as SessionData,
|
|
17
|
+
SubjectData as SubjectData,
|
|
18
|
+
SurgeryData as SurgeryData,
|
|
19
|
+
InjectionData as InjectionData,
|
|
20
|
+
ProcedureData as ProcedureData,
|
|
21
|
+
ProcessedData as ProcessedData,
|
|
22
|
+
MesoscopePaths as MesoscopePaths,
|
|
23
|
+
ZaberPositions as ZaberPositions,
|
|
24
|
+
ExperimentState as ExperimentState,
|
|
25
|
+
MesoscopeCameras as MesoscopeCameras,
|
|
26
|
+
MesoscopePositions as MesoscopePositions,
|
|
27
|
+
ProjectConfiguration as ProjectConfiguration,
|
|
28
|
+
RunTrainingDescriptor as RunTrainingDescriptor,
|
|
29
|
+
LickTrainingDescriptor as LickTrainingDescriptor,
|
|
30
|
+
MesoscopeHardwareState as MesoscopeHardwareState,
|
|
31
|
+
MesoscopeMicroControllers as MesoscopeMicroControllers,
|
|
32
|
+
MesoscopeAdditionalFirmware as MesoscopeAdditionalFirmware,
|
|
33
|
+
MesoscopeSystemConfiguration as MesoscopeSystemConfiguration,
|
|
34
|
+
MesoscopeExperimentDescriptor as MesoscopeExperimentDescriptor,
|
|
35
|
+
MesoscopeExperimentConfiguration as MesoscopeExperimentConfiguration,
|
|
36
|
+
get_system_configuration_data as get_system_configuration_data,
|
|
37
|
+
set_system_configuration_file as set_system_configuration_file,
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
__all__ = [
|
|
41
|
+
"Server",
|
|
42
|
+
"ServerCredentials",
|
|
43
|
+
"Job",
|
|
44
|
+
"DrugData",
|
|
45
|
+
"ImplantData",
|
|
46
|
+
"SessionData",
|
|
47
|
+
"RawData",
|
|
48
|
+
"ProcessedData",
|
|
49
|
+
"SubjectData",
|
|
50
|
+
"SurgeryData",
|
|
51
|
+
"InjectionData",
|
|
52
|
+
"ProcedureData",
|
|
53
|
+
"ZaberPositions",
|
|
54
|
+
"ExperimentState",
|
|
55
|
+
"MesoscopePositions",
|
|
56
|
+
"ProjectConfiguration",
|
|
57
|
+
"MesoscopeHardwareState",
|
|
58
|
+
"RunTrainingDescriptor",
|
|
59
|
+
"LickTrainingDescriptor",
|
|
60
|
+
"MesoscopeExperimentConfiguration",
|
|
61
|
+
"MesoscopeExperimentDescriptor",
|
|
62
|
+
"MesoscopeSystemConfiguration",
|
|
63
|
+
"MesoscopePaths",
|
|
64
|
+
"MesoscopeCameras",
|
|
65
|
+
"MesoscopeMicroControllers",
|
|
66
|
+
"MesoscopeAdditionalFirmware",
|
|
67
|
+
"get_system_configuration_data",
|
|
68
|
+
"set_system_configuration_file",
|
|
69
|
+
"transfer_directory",
|
|
70
|
+
"generate_project_manifest",
|
|
71
|
+
"verify_session_checksum",
|
|
72
|
+
"calculate_directory_checksum",
|
|
73
|
+
]
|
sl_shared_assets/cli.py
CHANGED
|
@@ -3,46 +3,168 @@
|
|
|
3
3
|
from pathlib import Path
|
|
4
4
|
|
|
5
5
|
import click
|
|
6
|
+
from ataraxis_base_utilities import LogLevel, console, ensure_directory_exists
|
|
6
7
|
|
|
7
|
-
from .tools import ascend_tyche_data
|
|
8
|
+
from .tools import ascend_tyche_data, verify_session_checksum, generate_project_manifest
|
|
8
9
|
from .server import generate_server_credentials
|
|
9
|
-
from .data_classes import
|
|
10
|
+
from .data_classes import (
|
|
11
|
+
ExperimentState,
|
|
12
|
+
ProjectConfiguration,
|
|
13
|
+
MesoscopeSystemConfiguration,
|
|
14
|
+
MesoscopeExperimentConfiguration,
|
|
15
|
+
get_system_configuration_data,
|
|
16
|
+
set_system_configuration_file,
|
|
17
|
+
)
|
|
10
18
|
|
|
11
19
|
|
|
12
20
|
@click.command()
|
|
13
21
|
@click.option(
|
|
14
|
-
"-
|
|
15
|
-
"--
|
|
22
|
+
"-sp",
|
|
23
|
+
"--session_path",
|
|
24
|
+
type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
|
|
25
|
+
required=True,
|
|
26
|
+
help="The absolute path to the session whose raw data needs to be verified for potential corruption.",
|
|
27
|
+
)
|
|
28
|
+
def verify_session_integrity(session_path: str) -> None:
|
|
29
|
+
"""Checks the integrity of the target session's raw data (contents of the raw_data directory).
|
|
30
|
+
|
|
31
|
+
This command assumes that the data has been checksummed during acquisition and contains an ax_checksum.txt file
|
|
32
|
+
that stores the data checksum generated before transferring the data to long-term storage destination. This function
|
|
33
|
+
always verified the integrity of the 'raw_data' directory. It does not work with 'processed_data' or any other
|
|
34
|
+
directories. If the session data was corrupted, the command removes the 'telomere.bin' file, marking the session as
|
|
35
|
+
'incomplete' and automatically excluding it from all further automated processing runtimes.
|
|
36
|
+
"""
|
|
37
|
+
session = Path(session_path)
|
|
38
|
+
if verify_session_checksum(session):
|
|
39
|
+
console.echo(message=f"Session {session.stem} raw data integrity: verified.", level=LogLevel.SUCCESS)
|
|
40
|
+
else:
|
|
41
|
+
console.echo(message=f"Session {session.stem} raw data integrity: compromised!", level=LogLevel.ERROR)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@click.command()
|
|
45
|
+
@click.option(
|
|
46
|
+
"-pp",
|
|
47
|
+
"--project_path",
|
|
48
|
+
type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
|
|
49
|
+
required=True,
|
|
50
|
+
help="The absolute path to the project directory where raw session data is stored.",
|
|
51
|
+
)
|
|
52
|
+
@click.option(
|
|
53
|
+
"-od",
|
|
54
|
+
"--output_directory",
|
|
55
|
+
type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
|
|
56
|
+
required=True,
|
|
57
|
+
help="The absolute path to the directory where to store the generated project manifest file.",
|
|
58
|
+
)
|
|
59
|
+
@click.option(
|
|
60
|
+
"-ppp",
|
|
61
|
+
"--project_processed_path",
|
|
62
|
+
type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
|
|
63
|
+
required=False,
|
|
64
|
+
help=(
|
|
65
|
+
"The absolute path to the project directory where processed session data is stored, if different from the "
|
|
66
|
+
"directory used to store raw session data. Typically, this extra argument is only used when processing data "
|
|
67
|
+
"stored on remote compute server(s)."
|
|
68
|
+
),
|
|
69
|
+
)
|
|
70
|
+
def generate_project_manifest_file(
|
|
71
|
+
project_path: str, output_directory: str, project_processed_path: str | None
|
|
72
|
+
) -> None:
|
|
73
|
+
"""Generates the manifest .feather file that provides information about the data-processing state of all available
|
|
74
|
+
project sessions.
|
|
75
|
+
|
|
76
|
+
The manifest file is typically used when batch-processing session data on the remote compute server. It contains the
|
|
77
|
+
comprehensive snapshot of the available project's data in a table-compatible format that can also be transferred
|
|
78
|
+
between machines (as it is cached in a file).
|
|
79
|
+
"""
|
|
80
|
+
generate_project_manifest(
|
|
81
|
+
raw_project_directory=Path(project_path),
|
|
82
|
+
output_directory=Path(output_directory),
|
|
83
|
+
processed_project_directory=Path(project_processed_path) if project_processed_path else None,
|
|
84
|
+
)
|
|
85
|
+
console.echo(message=f"Project {Path(project_path).stem} data manifest file: generated.", level=LogLevel.SUCCESS)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@click.command()
|
|
89
|
+
@click.option(
|
|
90
|
+
"-od",
|
|
91
|
+
"--output_directory",
|
|
16
92
|
type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
|
|
17
93
|
required=True,
|
|
18
|
-
|
|
19
|
-
help="The path to the new local directory where to store all project subdirectories.",
|
|
94
|
+
help="The absolute path to the directory where to store the generated system configuration file.",
|
|
20
95
|
)
|
|
21
|
-
|
|
22
|
-
""
|
|
96
|
+
@click.option(
|
|
97
|
+
"-as",
|
|
98
|
+
"--acquisition_system",
|
|
99
|
+
type=str,
|
|
100
|
+
show_default=True,
|
|
101
|
+
required=True,
|
|
102
|
+
default="mesoscope-vr",
|
|
103
|
+
help=(
|
|
104
|
+
"The type (name) of the data acquisition system for which to generate the configuration file. Note, currently, "
|
|
105
|
+
"only the following types are supported: mesoscope-vr."
|
|
106
|
+
),
|
|
107
|
+
)
|
|
108
|
+
def generate_system_configuration_file(output_directory: str, acquisition_system: str) -> None:
|
|
109
|
+
"""Generates a precursor system configuration file for the target acquisition system and configures all local
|
|
110
|
+
Sun lab libraries to use that file to load the acquisition system configuration data.
|
|
23
111
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
112
|
+
This command is typically used when setting up a new data acquisition system in the lab. The system configuration
|
|
113
|
+
only needs to be specified on the machine (PC) that runs the sl-experiment library and manages the acquisition
|
|
114
|
+
runtime if the system uses multiple machines (PCs). Once the system configuration .yaml file is created via this
|
|
115
|
+
command, editing the configuration parameters in the file will automatically take effect during all following
|
|
116
|
+
runtimes.
|
|
28
117
|
"""
|
|
29
|
-
|
|
118
|
+
|
|
119
|
+
# Verifies that the input path is a valid directory path and, if necessary, creates the directory specified by the
|
|
120
|
+
# path.
|
|
121
|
+
path = Path(output_directory)
|
|
122
|
+
if not path.is_dir():
|
|
123
|
+
message = (
|
|
124
|
+
f"Unable to generate the system configuration file for the system '{acquisition_system}'. The path to "
|
|
125
|
+
f"the output directory ({path}) is not a valid directory path."
|
|
126
|
+
)
|
|
127
|
+
console.error(message=message, error=ValueError)
|
|
128
|
+
else:
|
|
129
|
+
ensure_directory_exists(path)
|
|
130
|
+
|
|
131
|
+
# Mesoscope
|
|
132
|
+
if acquisition_system.lower() == "mesoscope-vr":
|
|
133
|
+
file_name = "mesoscope_system_configuration.yaml"
|
|
134
|
+
file_path = path.joinpath(file_name)
|
|
135
|
+
system_configuration = MesoscopeSystemConfiguration()
|
|
136
|
+
system_configuration.save(file_path)
|
|
137
|
+
set_system_configuration_file(file_path)
|
|
138
|
+
message = (
|
|
139
|
+
f"Mesoscope-VR system configuration file: generated. Edit the configuration parameters stored inside the "
|
|
140
|
+
f"{file_name} file to match the state of the acquisition system and use context."
|
|
141
|
+
)
|
|
142
|
+
console.echo(message=message, level=LogLevel.SUCCESS)
|
|
143
|
+
|
|
144
|
+
# For unsupported system types, raises an error message
|
|
145
|
+
else:
|
|
146
|
+
message = (
|
|
147
|
+
f"Unable to generate the system configuration file for the system '{acquisition_system}'. The input "
|
|
148
|
+
f"acquisition system is not supported (not recognized). Currently, only the following acquisition "
|
|
149
|
+
f"systems are supported: mesoscope-vr."
|
|
150
|
+
)
|
|
151
|
+
console.error(message=message, error=ValueError)
|
|
30
152
|
|
|
31
153
|
|
|
32
154
|
@click.command()
|
|
33
155
|
@click.option(
|
|
34
|
-
"-
|
|
156
|
+
"-od",
|
|
35
157
|
"--output_directory",
|
|
36
158
|
type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
|
|
37
159
|
required=True,
|
|
38
|
-
help="The absolute path to the directory where to
|
|
160
|
+
help="The absolute path to the directory where to store the generated server credentials file.",
|
|
39
161
|
)
|
|
40
162
|
@click.option(
|
|
41
163
|
"-h",
|
|
42
164
|
"--host",
|
|
43
165
|
type=str,
|
|
44
|
-
show_default=True,
|
|
45
166
|
required=True,
|
|
167
|
+
show_default=True,
|
|
46
168
|
default="cbsuwsun.biohpc.cornell.edu",
|
|
47
169
|
help="The host name or IP address of the server to connect to.",
|
|
48
170
|
)
|
|
@@ -63,49 +185,153 @@ def replace_local_root_directory(path: str) -> None:
|
|
|
63
185
|
def generate_server_credentials_file(output_directory: str, host: str, username: str, password: str) -> None:
|
|
64
186
|
"""Generates a new server_credentials.yaml file under the specified directory, using input information.
|
|
65
187
|
|
|
66
|
-
This
|
|
67
|
-
|
|
188
|
+
This command is used to set up access to compute servers and clusters on new machines (PCs). The data stored inside
|
|
189
|
+
the server_credentials.yaml file generated by this command is used by the Server and Job classes used in many Sun
|
|
190
|
+
lab data processing libraries.
|
|
68
191
|
"""
|
|
69
192
|
generate_server_credentials(
|
|
70
193
|
output_directory=Path(output_directory), username=username, password=password, host=host
|
|
71
194
|
)
|
|
195
|
+
message = (
|
|
196
|
+
f"Server access credentials file: generated. If necessary, remember to edit the data acquisition system "
|
|
197
|
+
f"configuration file to include the path to the credentials file generated via this CLI."
|
|
198
|
+
)
|
|
199
|
+
console.echo(message=message, level=LogLevel.SUCCESS)
|
|
72
200
|
|
|
73
201
|
|
|
74
202
|
@click.command()
|
|
75
203
|
@click.option(
|
|
76
204
|
"-p",
|
|
77
|
-
"--
|
|
78
|
-
type=
|
|
205
|
+
"--project",
|
|
206
|
+
type=str,
|
|
79
207
|
required=True,
|
|
80
|
-
help="The
|
|
208
|
+
help="The name of the project to be created.",
|
|
81
209
|
)
|
|
82
210
|
@click.option(
|
|
83
|
-
"-
|
|
84
|
-
"--
|
|
85
|
-
type=
|
|
211
|
+
"-sli",
|
|
212
|
+
"--surgery_log_id",
|
|
213
|
+
type=str,
|
|
86
214
|
required=True,
|
|
87
|
-
help="The
|
|
215
|
+
help="The 44-symbol alpha-numeric ID code used by the project's surgery log Google sheet.",
|
|
88
216
|
)
|
|
89
217
|
@click.option(
|
|
90
|
-
"-
|
|
91
|
-
"--
|
|
218
|
+
"-wli",
|
|
219
|
+
"--water_restriction_log_id",
|
|
220
|
+
type=str,
|
|
221
|
+
required=True,
|
|
222
|
+
help="The 44-symbol alpha-numeric ID code used by the project's water restriction log Google sheet.",
|
|
223
|
+
)
|
|
224
|
+
def generate_project_configuration_file(project: str, surgery_log_id: str, water_restriction_log_id: str) -> None:
|
|
225
|
+
"""Generates a new project directory hierarchy and writes its configuration as a project_configuration.yaml file.
|
|
226
|
+
|
|
227
|
+
This command creates new Sun lab projects. Until a project is created in this fashion, all data-acquisition and
|
|
228
|
+
data-processing commands from sl-experiment and sl-forgery libraries targeting the project will not work. This
|
|
229
|
+
command is intended to be called on the main computer of the data-acquisition system(s) used by the project. Note,
|
|
230
|
+
this command assumes that the local machine (PC) is the main PC of the data acquisition system and has a valid
|
|
231
|
+
acquisition system configuration .yaml file.
|
|
232
|
+
"""
|
|
233
|
+
|
|
234
|
+
# Queries the data acquisition configuration data. Specifically, this is used to get the path to the root
|
|
235
|
+
# directory where all projects are stored on the local machine.
|
|
236
|
+
system_configuration = get_system_configuration_data()
|
|
237
|
+
file_path = system_configuration.paths.root_directory.joinpath(
|
|
238
|
+
project, "configuration", "project_configuration.yaml"
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
# Generates the initial project directory hierarchy
|
|
242
|
+
ensure_directory_exists(file_path)
|
|
243
|
+
|
|
244
|
+
# Saves project configuration data as a .yaml file to the 'configuration' directory of the created project
|
|
245
|
+
configuration = ProjectConfiguration(
|
|
246
|
+
project_name=project, surgery_sheet_id=surgery_log_id, water_log_sheet_id=water_restriction_log_id
|
|
247
|
+
)
|
|
248
|
+
configuration.save(path=file_path.joinpath())
|
|
249
|
+
console.echo(message=f"Project {project} data structure and configuration file: generated.", level=LogLevel.SUCCESS)
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
@click.command()
|
|
253
|
+
@click.option(
|
|
254
|
+
"-p",
|
|
255
|
+
"--project",
|
|
256
|
+
type=str,
|
|
257
|
+
required=True,
|
|
258
|
+
help="The name of the project for which to generate the new experiment configuration file.",
|
|
259
|
+
)
|
|
260
|
+
@click.option(
|
|
261
|
+
"-e",
|
|
262
|
+
"--experiment",
|
|
263
|
+
type=str,
|
|
264
|
+
required=True,
|
|
265
|
+
help="The name of the experiment. Note, the generated experiment configuration file will also use this name.",
|
|
266
|
+
)
|
|
267
|
+
@click.option(
|
|
268
|
+
"-sc",
|
|
269
|
+
"--state_count",
|
|
270
|
+
type=int,
|
|
271
|
+
required=True,
|
|
272
|
+
help="The total number of experiment and acquisition system state combinations in the experiment.",
|
|
273
|
+
)
|
|
274
|
+
def generate_experiment_configuration_file(project: str, experiment: str, state_count: int) -> None:
|
|
275
|
+
"""Generates a precursor experiment configuration .yaml file for the target experiment inside the project's
|
|
276
|
+
configuration folder.
|
|
277
|
+
|
|
278
|
+
This command assists users in creating new experiment configurations, by statically resolving the structure (layout)
|
|
279
|
+
of the appropriate experiment configuration file for the acquisition system of the local machine (PC). Specifically,
|
|
280
|
+
the generated precursor will contain the correct number of experiment state entries initialized to nonsensical
|
|
281
|
+
default value. The user needs to manually edit the configuration file to properly specify their experiment runtime
|
|
282
|
+
parameters and state transitions before running the experiment. In a sense, this command acts as an 'experiment
|
|
283
|
+
template' generator.
|
|
284
|
+
"""
|
|
285
|
+
|
|
286
|
+
# Resolves the acquisition system configuration. Uses the path to the local project directory and the project name
|
|
287
|
+
# to determine where to save the experiment configuration file
|
|
288
|
+
acquisition_system = get_system_configuration_data()
|
|
289
|
+
file_path = acquisition_system.paths.root_directory.joinpath(project, "configuration", f"{experiment}.yaml")
|
|
290
|
+
|
|
291
|
+
# Loops over the number of requested states and, for each, generates a precursor experiment state field inside the
|
|
292
|
+
# 'states' dictionary.
|
|
293
|
+
states = {}
|
|
294
|
+
for state in range(state_count):
|
|
295
|
+
states[f"state_{state + 1}"] = ExperimentState(
|
|
296
|
+
experiment_state_code=state + 1, # Assumes experiment state sequences are 1-based
|
|
297
|
+
system_state_code=0,
|
|
298
|
+
state_duration_s=60,
|
|
299
|
+
)
|
|
300
|
+
|
|
301
|
+
# Depending on the acquisition system, packs state data into the appropriate experiment configuration class and
|
|
302
|
+
# saves it to the project's configuration folder as a .yaml file.
|
|
303
|
+
if acquisition_system.name == "mesoscope-vr":
|
|
304
|
+
experiment_configuration = MesoscopeExperimentConfiguration(experiment_states=states)
|
|
305
|
+
|
|
306
|
+
else:
|
|
307
|
+
message = (
|
|
308
|
+
f"Unable to generate the experiment {experiment} configuration file for the project {project}. "
|
|
309
|
+
f"The data acquisition system of the local machine (PC) is not supported (not recognized). Currently, only "
|
|
310
|
+
f"the following acquisition systems are supported: mesoscope-vr."
|
|
311
|
+
)
|
|
312
|
+
console.error(message=message, error=ValueError)
|
|
313
|
+
raise ValueError(message) # Fall-back to appease mypy, should not be reachable
|
|
314
|
+
|
|
315
|
+
experiment_configuration.to_yaml(file_path=file_path)
|
|
316
|
+
console.echo(message=f"Experiment {experiment} configuration file: generated.", level=LogLevel.SUCCESS)
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
@click.command()
|
|
320
|
+
@click.option(
|
|
321
|
+
"-id",
|
|
322
|
+
"--input_directory",
|
|
92
323
|
type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
|
|
93
324
|
required=True,
|
|
94
|
-
help=
|
|
95
|
-
"The path to the SMB-mounted BioHPC server directory where to transfer the ascended Tyche project "
|
|
96
|
-
"hierarchy after it is created."
|
|
97
|
-
),
|
|
325
|
+
help="The absolute path to the directory that stores original Tyche animal folders.",
|
|
98
326
|
)
|
|
99
|
-
def ascend_tyche_directory(
|
|
100
|
-
"""Restructures old Tyche project data to use the modern Sun lab data structure
|
|
327
|
+
def ascend_tyche_directory(input_directory: str) -> None:
|
|
328
|
+
"""Restructures old Tyche project data to use the modern Sun lab data structure and uploads them to the processing
|
|
329
|
+
server.
|
|
101
330
|
|
|
102
|
-
This
|
|
331
|
+
This command is used to convert ('ascend') the old Tyche project data to the modern Sun lab structure. After
|
|
103
332
|
ascension, the data can be processed and analyzed using all modern Sun lab (sl-) tools and libraries. Note, this
|
|
104
333
|
process expects the input data to be preprocessed using an old Sun lab mesoscope data preprocessing pipeline. It
|
|
105
|
-
will not work for any other project or data.
|
|
334
|
+
will not work for any other project or data. Also, this command will only work on a machine (PC) that belongs to a
|
|
335
|
+
valid Sun lab data acquisition system, such as VRPC of the Mesoscope-VR system.
|
|
106
336
|
"""
|
|
107
|
-
ascend_tyche_data(
|
|
108
|
-
root_directory=Path(path),
|
|
109
|
-
output_root_directory=Path(output_directory),
|
|
110
|
-
server_root_directory=Path(server_directory),
|
|
111
|
-
)
|
|
337
|
+
ascend_tyche_data(root_directory=Path(input_directory))
|
sl_shared_assets/cli.pyi
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
from .tools import (
|
|
2
|
+
ascend_tyche_data as ascend_tyche_data,
|
|
3
|
+
verify_session_checksum as verify_session_checksum,
|
|
4
|
+
generate_project_manifest as generate_project_manifest,
|
|
5
|
+
)
|
|
6
|
+
from .server import generate_server_credentials as generate_server_credentials
|
|
7
|
+
from .data_classes import (
|
|
8
|
+
ExperimentState as ExperimentState,
|
|
9
|
+
ProjectConfiguration as ProjectConfiguration,
|
|
10
|
+
MesoscopeSystemConfiguration as MesoscopeSystemConfiguration,
|
|
11
|
+
MesoscopeExperimentConfiguration as MesoscopeExperimentConfiguration,
|
|
12
|
+
get_system_configuration_data as get_system_configuration_data,
|
|
13
|
+
set_system_configuration_file as set_system_configuration_file,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
def verify_session_integrity(session_path: str) -> None:
|
|
17
|
+
"""Checks the integrity of the target session's raw data (contents of the raw_data directory).
|
|
18
|
+
|
|
19
|
+
This command assumes that the data has been checksummed during acquisition and contains an ax_checksum.txt file
|
|
20
|
+
that stores the data checksum generated before transferring the data to long-term storage destination. This function
|
|
21
|
+
always verified the integrity of the 'raw_data' directory. It does not work with 'processed_data' or any other
|
|
22
|
+
directories. If the session data was corrupted, the command removes the 'telomere.bin' file, marking the session as
|
|
23
|
+
'incomplete' and automatically excluding it from all further automated processing runtimes.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
def generate_project_manifest_file(
|
|
27
|
+
project_path: str, output_directory: str, project_processed_path: str | None
|
|
28
|
+
) -> None:
|
|
29
|
+
"""Generates the manifest .feather file that provides information about the data-processing state of all available
|
|
30
|
+
project sessions.
|
|
31
|
+
|
|
32
|
+
The manifest file is typically used when batch-processing session data on the remote compute server. It contains the
|
|
33
|
+
comprehensive snapshot of the available project's data in a table-compatible format that can also be transferred
|
|
34
|
+
between machines (as it is cached in a file).
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
def generate_system_configuration_file(output_directory: str, acquisition_system: str) -> None:
|
|
38
|
+
"""Generates a precursor system configuration file for the target acquisition system and configures all local
|
|
39
|
+
Sun lab libraries to use that file to load the acquisition system configuration data.
|
|
40
|
+
|
|
41
|
+
This command is typically used when setting up a new data acquisition system in the lab. The system configuration
|
|
42
|
+
only needs to be specified on the machine (PC) that runs the sl-experiment library and manages the acquisition
|
|
43
|
+
runtime if the system uses multiple machines (PCs). Once the system configuration .yaml file is created via this
|
|
44
|
+
command, editing the configuration parameters in the file will automatically take effect during all following
|
|
45
|
+
runtimes.
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
def generate_server_credentials_file(output_directory: str, host: str, username: str, password: str) -> None:
|
|
49
|
+
"""Generates a new server_credentials.yaml file under the specified directory, using input information.
|
|
50
|
+
|
|
51
|
+
This command is used to set up access to compute servers and clusters on new machines (PCs). The data stored inside
|
|
52
|
+
the server_credentials.yaml file generated by this command is used by the Server and Job classes used in many Sun
|
|
53
|
+
lab data processing libraries.
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
def generate_project_configuration_file(project: str, surgery_log_id: str, water_restriction_log_id: str) -> None:
|
|
57
|
+
"""Generates a new project directory hierarchy and writes its configuration as a project_configuration.yaml file.
|
|
58
|
+
|
|
59
|
+
This command creates new Sun lab projects. Until a project is created in this fashion, all data-acquisition and
|
|
60
|
+
data-processing commands from sl-experiment and sl-forgery libraries targeting the project will not work. This
|
|
61
|
+
command is intended to be called on the main computer of the data-acquisition system(s) used by the project. Note,
|
|
62
|
+
this command assumes that the local machine (PC) is the main PC of the data acquisition system and has a valid
|
|
63
|
+
acquisition system configuration .yaml file.
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
def generate_experiment_configuration_file(project: str, experiment: str, state_count: int) -> None:
|
|
67
|
+
"""Generates a precursor experiment configuration .yaml file for the target experiment inside the project's
|
|
68
|
+
configuration folder.
|
|
69
|
+
|
|
70
|
+
This command assists users in creating new experiment configurations, by statically resolving the structure (layout)
|
|
71
|
+
of the appropriate experiment configuration file for the acquisition system of the local machine (PC). Specifically,
|
|
72
|
+
the generated precursor will contain the correct number of experiment state entries initialized to nonsensical
|
|
73
|
+
default value. The user needs to manually edit the configuration file to properly specify their experiment runtime
|
|
74
|
+
parameters and state transitions before running the experiment. In a sense, this command acts as an 'experiment
|
|
75
|
+
template' generator.
|
|
76
|
+
"""
|
|
77
|
+
|
|
78
|
+
def ascend_tyche_directory(input_directory: str) -> None:
|
|
79
|
+
"""Restructures old Tyche project data to use the modern Sun lab data structure and uploads them to the processing
|
|
80
|
+
server.
|
|
81
|
+
|
|
82
|
+
This command is used to convert ('ascend') the old Tyche project data to the modern Sun lab structure. After
|
|
83
|
+
ascension, the data can be processed and analyzed using all modern Sun lab (sl-) tools and libraries. Note, this
|
|
84
|
+
process expects the input data to be preprocessed using an old Sun lab mesoscope data preprocessing pipeline. It
|
|
85
|
+
will not work for any other project or data. Also, this command will only work on a machine (PC) that belongs to a
|
|
86
|
+
valid Sun lab data acquisition system, such as VRPC of the Mesoscope-VR system.
|
|
87
|
+
"""
|