sl-shared-assets 4.0.1__py3-none-any.whl → 5.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of sl-shared-assets might be problematic. Click here for more details.

Files changed (39) hide show
  1. sl_shared_assets/__init__.py +48 -41
  2. sl_shared_assets/command_line_interfaces/__init__.py +3 -0
  3. sl_shared_assets/command_line_interfaces/configure.py +173 -0
  4. sl_shared_assets/command_line_interfaces/manage.py +226 -0
  5. sl_shared_assets/data_classes/__init__.py +33 -32
  6. sl_shared_assets/data_classes/configuration_data.py +267 -79
  7. sl_shared_assets/data_classes/session_data.py +226 -289
  8. sl_shared_assets/server/__init__.py +24 -4
  9. sl_shared_assets/server/job.py +6 -7
  10. sl_shared_assets/server/pipeline.py +585 -0
  11. sl_shared_assets/server/server.py +57 -25
  12. sl_shared_assets/tools/__init__.py +9 -8
  13. sl_shared_assets/tools/packaging_tools.py +14 -25
  14. sl_shared_assets/tools/project_management_tools.py +602 -523
  15. sl_shared_assets/tools/transfer_tools.py +88 -23
  16. {sl_shared_assets-4.0.1.dist-info → sl_shared_assets-5.0.1.dist-info}/METADATA +46 -203
  17. sl_shared_assets-5.0.1.dist-info/RECORD +23 -0
  18. sl_shared_assets-5.0.1.dist-info/entry_points.txt +3 -0
  19. sl_shared_assets/__init__.pyi +0 -91
  20. sl_shared_assets/cli.py +0 -501
  21. sl_shared_assets/cli.pyi +0 -106
  22. sl_shared_assets/data_classes/__init__.pyi +0 -75
  23. sl_shared_assets/data_classes/configuration_data.pyi +0 -235
  24. sl_shared_assets/data_classes/runtime_data.pyi +0 -157
  25. sl_shared_assets/data_classes/session_data.pyi +0 -379
  26. sl_shared_assets/data_classes/surgery_data.pyi +0 -89
  27. sl_shared_assets/server/__init__.pyi +0 -11
  28. sl_shared_assets/server/job.pyi +0 -205
  29. sl_shared_assets/server/server.pyi +0 -298
  30. sl_shared_assets/tools/__init__.pyi +0 -19
  31. sl_shared_assets/tools/ascension_tools.py +0 -265
  32. sl_shared_assets/tools/ascension_tools.pyi +0 -68
  33. sl_shared_assets/tools/packaging_tools.pyi +0 -58
  34. sl_shared_assets/tools/project_management_tools.pyi +0 -239
  35. sl_shared_assets/tools/transfer_tools.pyi +0 -53
  36. sl_shared_assets-4.0.1.dist-info/RECORD +0 -36
  37. sl_shared_assets-4.0.1.dist-info/entry_points.txt +0 -7
  38. {sl_shared_assets-4.0.1.dist-info → sl_shared_assets-5.0.1.dist-info}/WHEEL +0 -0
  39. {sl_shared_assets-4.0.1.dist-info → sl_shared_assets-5.0.1.dist-info}/licenses/LICENSE +0 -0
sl_shared_assets/cli.py DELETED
@@ -1,501 +0,0 @@
1
- """This module stores the Command-Line Interfaces (CLIs) exposes by the library as part of the installation process."""
2
-
3
- from pathlib import Path
4
-
5
- import click
6
- from ataraxis_base_utilities import LogLevel, console, ensure_directory_exists
7
-
8
- from .tools import ascend_tyche_data, resolve_p53_marker, verify_session_checksum, generate_project_manifest
9
- from .server import Server, JupyterJob, generate_server_credentials
10
- from .data_classes import SessionData, TrackerFileNames, get_processing_tracker
11
-
12
-
13
- @click.command()
14
- @click.option(
15
- "-sp",
16
- "--session_path",
17
- type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
18
- required=True,
19
- help="The absolute path to the session directory whose raw data needs to be verified for potential corruption.",
20
- )
21
- @click.option(
22
- "-id",
23
- "--manager_id",
24
- type=int,
25
- required=True,
26
- default=0,
27
- show_default=True,
28
- help=(
29
- "The xxHash-64 hash value that represents the unique identifier for the process that manages this runtime. "
30
- "This is primarily used when calling this CLI on remote compute servers to ensure that only a single process "
31
- "can execute the CLI at a time."
32
- ),
33
- )
34
- @click.option(
35
- "-c",
36
- "--create_processed_directories",
37
- is_flag=True,
38
- show_default=True,
39
- default=False,
40
- help=(
41
- "Determines whether to create the processed data hierarchy. This flag should be disabled for most runtimes. "
42
- "Primarily, it is used by acquisition systems to generate processed data directories on the remote "
43
- "compute servers as part of the data preprocessing pipeline."
44
- ),
45
- )
46
- @click.option(
47
- "-pdr",
48
- "--processed_data_root",
49
- type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
50
- required=False,
51
- help=(
52
- "The absolute path to the directory where processed data from all projects is stored on the machine that runs "
53
- "this command. This argument is used when calling the CLI on the BioHPC server, which uses different data "
54
- "volumes for raw and processed data. Note, the input path must point to the root directory, as it will be "
55
- "automatically modified to include the project name, the animal id, and the session ID. This argument is only "
56
- "used if 'create_processed_directories' flag is True."
57
- ),
58
- )
59
- @click.option(
60
- "-um",
61
- "--update_manifest",
62
- is_flag=True,
63
- help=(
64
- "Determines whether to (re)generate the manifest file for the processed session's project. This flag "
65
- "should always be enabled when this CLI is executed on the remote compute server(s) to ensure that the "
66
- "manifest file always reflects the most actual state of each project."
67
- ),
68
- )
69
- def verify_session_integrity(
70
- session_path: Path,
71
- manager_id: int,
72
- create_processed_directories: bool,
73
- processed_data_root: Path | None,
74
- update_manifest: bool,
75
- ) -> None:
76
- """Checks the integrity of the target session's raw data (contents of the raw_data directory).
77
-
78
- This command assumes that the data has been checksummed during acquisition and contains an ax_checksum.txt file
79
- that stores the data checksum generated before transferring the data to the long-term storage destination. This
80
- function always verified the integrity of the 'raw_data' directory. It does not work with 'processed_data' or any
81
- other directories. If the session data was corrupted, the command removes the 'telomere.bin' file, marking the
82
- session as 'incomplete' and automatically excluding it from all further automated processing runtimes. If the
83
- session data is intact, it generates a 'verified.bin' marker file inside the session's raw_data folder.
84
-
85
- The command is also used by Sun lab data acquisition systems to generate the processed data hierarchy for each
86
- processed session. This use case is fully automated and should not be triggered manually by the user.
87
- """
88
- session = Path(session_path)
89
- session_data = SessionData.load(session_path=session)
90
-
91
- # Runs the verification process
92
- verify_session_checksum(
93
- session_path=session,
94
- manager_id=manager_id,
95
- create_processed_data_directory=create_processed_directories,
96
- processed_data_root=processed_data_root,
97
- update_manifest=update_manifest,
98
- )
99
-
100
- # Checks the outcome of the verification process
101
- tracker = get_processing_tracker(root=session_data.raw_data.raw_data_path, file_name=TrackerFileNames.INTEGRITY)
102
- if tracker.is_complete:
103
- # noinspection PyTypeChecker
104
- console.echo(message=f"Session {session.stem} raw data integrity: Verified.", level=LogLevel.SUCCESS)
105
- else:
106
- # noinspection PyTypeChecker
107
- console.echo(message=f"Session {session.stem} raw data integrity: Compromised!", level=LogLevel.ERROR)
108
-
109
-
110
- @click.command()
111
- @click.option(
112
- "-pp",
113
- "--project_path",
114
- type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
115
- required=True,
116
- help="The absolute path to the project directory where raw session data is stored.",
117
- )
118
- @click.option(
119
- "-od",
120
- "--output_directory",
121
- type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
122
- required=True,
123
- help="The absolute path to the directory where to store the generated project manifest file.",
124
- )
125
- @click.option(
126
- "-pdr",
127
- "--processed_data_root",
128
- type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
129
- required=False,
130
- help=(
131
- "The absolute path to the directory where processed data from all projects is stored on the machine that runs "
132
- "this command. This argument is used when calling the CLI on the BioHPC server, which uses different data "
133
- "volumes for raw and processed data. Note, the input path must point to the root directory, as it will be "
134
- "automatically modified to include the project name. Note, if the system cannot properly resolve the path to "
135
- "the processed data, the generated manifest will indicate that no data processing has been performed for the "
136
- "project."
137
- ),
138
- )
139
- def generate_project_manifest_file(
140
- project_path: Path, output_directory: Path, processed_data_root: Path | None
141
- ) -> None:
142
- """Generates the manifest .feather file that provides information about the data-processing state of all available
143
- project sessions.
144
-
145
- The manifest file is typically used when batch-processing session data on the remote compute server. It contains the
146
- comprehensive snapshot of the available project's data in a table-compatible format that can also be transferred
147
- between machines (as it is cached in a file).
148
- """
149
- generate_project_manifest(
150
- raw_project_directory=Path(project_path),
151
- output_directory=Path(output_directory),
152
- processed_data_root=Path(processed_data_root) if processed_data_root else None,
153
- )
154
- # noinspection PyTypeChecker
155
- console.echo(message=f"Project {Path(project_path).stem} data manifest file: generated.", level=LogLevel.SUCCESS)
156
-
157
-
158
- @click.command()
159
- @click.option(
160
- "-od",
161
- "--output_directory",
162
- type=click.Path(exists=False, file_okay=False, dir_okay=True, path_type=Path),
163
- required=True,
164
- help="The absolute path to the directory where to store the generated server credentials file.",
165
- )
166
- @click.option(
167
- "-h",
168
- "--host",
169
- type=str,
170
- required=True,
171
- show_default=True,
172
- default="cbsuwsun.biohpc.cornell.edu",
173
- help="The host name or IP address of the server to connect to.",
174
- )
175
- @click.option(
176
- "-u",
177
- "--username",
178
- type=str,
179
- required=True,
180
- help="The username to use for server authentication.",
181
- )
182
- @click.option(
183
- "-p",
184
- "--password",
185
- type=str,
186
- required=True,
187
- help="The password to use for server authentication.",
188
- )
189
- @click.option(
190
- "-sr",
191
- "--storage_root",
192
- type=str,
193
- required=True,
194
- show_default=True,
195
- default="/local/storage",
196
- help=(
197
- "The absolute path to to the root storage (slow) server directory. Typically, this is the path to the "
198
- "top-level (root) directory of the HDD RAID volume."
199
- ),
200
- )
201
- @click.option(
202
- "-wr",
203
- "--working_root",
204
- type=str,
205
- required=True,
206
- show_default=True,
207
- default="/local/workdir",
208
- help=(
209
- "The absolute path to the root working (fast) server directory. Typically, this is the path to the top-level "
210
- "(root) directory of the NVME RAID volume. If the server uses the same volume for both storage and working "
211
- "directories, enter the same path under both 'storage_root' and 'working_root'."
212
- ),
213
- )
214
- @click.option(
215
- "-sdn",
216
- "--shared_directory_name",
217
- type=str,
218
- required=True,
219
- show_default=True,
220
- default="sun_data",
221
- help=(
222
- "The name of the shared directory used to store all Sun lab project data on the storage and working server "
223
- "volumes."
224
- ),
225
- )
226
- def generate_server_credentials_file(
227
- output_directory: Path,
228
- host: str,
229
- username: str,
230
- password: str,
231
- storage_root: str,
232
- working_root: str,
233
- shared_directory_name: str,
234
- ) -> None:
235
- """Generates a new server_credentials.yaml file under the specified directory, using input information.
236
-
237
- This command is used to set up access to compute servers and clusters on new machines (PCs). The data stored inside
238
- the server_credentials.yaml file generated by this command is used by the Server and Job classes used in many Sun
239
- lab data processing libraries.
240
- """
241
-
242
- # If necessary, generates the output directory hierarchy before creating the credentials' file.
243
- ensure_directory_exists(output_directory)
244
-
245
- # Generates the credentials' file
246
- generate_server_credentials(
247
- output_directory=Path(output_directory),
248
- username=username,
249
- password=password,
250
- host=host,
251
- storage_root=storage_root,
252
- working_root=working_root,
253
- shared_directory_name=shared_directory_name,
254
- )
255
- message = (
256
- f"Server access credentials file: generated. If necessary, remember to edit the data acquisition system "
257
- f"configuration file to include the path to the credentials file generated via this CLI."
258
- )
259
- # noinspection PyTypeChecker
260
- console.echo(message=message, level=LogLevel.SUCCESS)
261
- message = f"File location: {output_directory}"
262
- # noinspection PyTypeChecker
263
- console.echo(message=message, level=LogLevel.SUCCESS)
264
-
265
-
266
- @click.command()
267
- @click.option(
268
- "-id",
269
- "--input_directory",
270
- type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
271
- required=True,
272
- help="The absolute path to the directory that stores original Tyche animal folders.",
273
- )
274
- def ascend_tyche_directory(input_directory: Path) -> None:
275
- """Restructures old Tyche project data to use the modern Sun lab data structure and uploads them to the processing
276
- server.
277
-
278
- This command is used to convert ('ascend') the old Tyche project data to the modern Sun lab structure. After
279
- ascension, the data can be processed and analyzed using all modern Sun lab (sl-) tools and libraries. Note, this
280
- process expects the input data to be preprocessed using an old Sun lab mesoscope data preprocessing pipeline. It
281
- will not work for any other project or data. Also, this command will only work on a machine (PC) that belongs to a
282
- valid Sun lab data acquisition system, such as VRPC of the Mesoscope-VR system.
283
- """
284
- ascend_tyche_data(root_directory=Path(input_directory))
285
-
286
-
287
- @click.command()
288
- @click.option(
289
- "-cp",
290
- "--credentials_path",
291
- type=click.Path(exists=True, file_okay=True, dir_okay=False, path_type=Path),
292
- required=True,
293
- help=(
294
- "The absolute path to the server_credentials.yaml file that stores access credentials for the target Sun lab "
295
- "server. If necessary, use the 'sl-create-server-credentials' command to generate the file."
296
- ),
297
- )
298
- @click.option(
299
- "-n",
300
- "--name",
301
- type=str,
302
- required=True,
303
- show_default=True,
304
- default="jupyter_server",
305
- help=(
306
- "The descriptive name to be given to the remote Jupyter server job. Primarily, this is used to identify the "
307
- "job inside the log files."
308
- ),
309
- )
310
- @click.option(
311
- "-e",
312
- "--environment",
313
- type=str,
314
- required=True,
315
- help=(
316
- "The name of the conda environment to use for running the Jupyter server. At a minimum, the target environment "
317
- "must contain the 'jupyterlab' and the 'notebook' Python packages. Note, the user whose credentials are used "
318
- "to connect to the server must have a configured conda / mamba shell that exposes the target environment for "
319
- "the job to run as expected."
320
- ),
321
- )
322
- @click.option(
323
- "-d",
324
- "--directory",
325
- type=click.Path(exists=False, file_okay=True, dir_okay=True, path_type=Path),
326
- required=False,
327
- help=(
328
- "The absolute path to the server directory to use as the root directory for the jupyter session. If not "
329
- "provided, this is automatically resolved to user's working directory. Note, during runtime, Jupyter will only "
330
- "have access to files stored in or under that root directory."
331
- ),
332
- )
333
- @click.option(
334
- "-c",
335
- "--cores",
336
- type=int,
337
- required=True,
338
- show_default=True,
339
- default=2,
340
- help=(
341
- "The number of CPU cores to allocate to the Jupyter server. Note, during the interactive Jupyter runtime, it "
342
- "is be impossible to use more than this number of CPU cores."
343
- ),
344
- )
345
- @click.option(
346
- "-m",
347
- "--memory",
348
- type=int,
349
- required=True,
350
- show_default=True,
351
- default=32,
352
- help=(
353
- "The RAM, in Gigabytes, to allocate to the Jupyter server. Note, during the interactive Jupyter runtime, it "
354
- "is be impossible to use more than this amount of RAM."
355
- ),
356
- )
357
- @click.option(
358
- "-t",
359
- "--time",
360
- type=int,
361
- required=True,
362
- show_default=True,
363
- default=240,
364
- help=(
365
- "The maximum runtime duration for this Jupyter server instance, in minutes. If the server job is still running "
366
- "at the end of this time limit, the job will be forcibly terminated by SLURM. Note, to prevent hogging the "
367
- "server, make sure this parameter is always set to the smallest feasible period of time you intend to interact "
368
- "with the server."
369
- ),
370
- )
371
- @click.option(
372
- "-p",
373
- "--port",
374
- type=int,
375
- required=True,
376
- show_default=True,
377
- default=0,
378
- help=(
379
- "The port to use for the Jupyter server communication on the remote server. Valid port values are from 8888 "
380
- "to 9999. Most runtimes should leave this set to the default value (0), which will randomly select one of the "
381
- "valid ports. Using random selection minimizes the chances of colliding with other interactive jupyter "
382
- "sessions."
383
- ),
384
- )
385
- def start_jupyter_server(
386
- credentials_path: Path, name: str, environment: str, directory: Path, cores: int, memory: int, time: int, port: int
387
- ) -> None:
388
- """Starts an interactive Jupyter session on the remote Sun lab server.
389
-
390
- This command should be used to run Jupyter lab and notebooks sessions on the remote Sun lab server. Since all lab
391
- data is stored on the server, this allows running light interactive analysis sessions on the same node as the data,
392
- while leveraging considerable compute resources of the server.
393
-
394
- Calling this command initializes a SLURM session that runs the interactive Jupyter server. Since this server
395
- directly competes for resources with all other headless jobs running on the server, it is imperative that each
396
- jupyter runtime uses only the minimum amount of resources and run-time as necessary. Do not use this command to run
397
- heavy data processing pipelines! Instead, consult with library documentation and use the headless Job class.
398
- """
399
- # Initializes server connection
400
- server = Server(credentials_path)
401
- job: JupyterJob | None = None
402
- try:
403
- # If the caller did not provide an explicit notebook directory, defaults to the user's working directory
404
- if directory is None:
405
- directory = (server.user_working_root,)
406
-
407
- # Launches the specified Jupyter server
408
- job = server.launch_jupyter_server(
409
- job_name=name,
410
- conda_environment=environment,
411
- notebook_directory=directory,
412
- cpus_to_use=cores,
413
- ram_gb=memory,
414
- port=port,
415
- time_limit=time,
416
- )
417
-
418
- # Displays the server connection details to the user via terminal
419
- job.print_connection_info()
420
-
421
- # Blocks in-place until the user shuts down the server. This allows terminating the jupyter job early if the
422
- # user is done working with the server
423
- input("Enter anything to shut down the server: ")
424
-
425
- # Ensures that the server created as part of this CLI is always terminated when the CLI terminates
426
- finally:
427
- # Terminates the server job
428
- if job is not None:
429
- server.abort_job(job)
430
-
431
- # Closes the server connection if it is still open
432
- server.close()
433
-
434
-
435
- @click.command()
436
- @click.option(
437
- "-sp",
438
- "--session_path",
439
- type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
440
- required=True,
441
- help="The absolute path to the session directory for which to resolve the dataset integration readiness marker.",
442
- )
443
- @click.option(
444
- "-c",
445
- "--create_processed_directories",
446
- is_flag=True,
447
- show_default=True,
448
- default=False,
449
- help="Determines whether to create the processed data hierarchy. This flag should be disabled for most runtimes.",
450
- )
451
- @click.option(
452
- "-pdr",
453
- "--processed_data_root",
454
- type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
455
- required=False,
456
- help=(
457
- "The absolute path to the directory where processed data from all projects is stored on the machine that runs "
458
- "this command. This argument is used when calling the CLI on the BioHPC server, which uses different data "
459
- "volumes for raw and processed data. Note, the input path must point to the root directory, as it will be "
460
- "automatically modified to include the project name."
461
- ),
462
- )
463
- @click.option(
464
- "-r",
465
- "--remove",
466
- is_flag=True,
467
- show_default=True,
468
- default=False,
469
- help="Determines whether the command should create or remove the dataset integration marker.",
470
- )
471
- @click.option(
472
- "-um",
473
- "--update_manifest",
474
- is_flag=True,
475
- help=(
476
- "Determines whether to (re)generate the manifest file for the processed session's project. This flag "
477
- "should always be enabled when this CLI is executed on the remote compute server(s) to ensure that the "
478
- "manifest file always reflects the most actual state of each project."
479
- ),
480
- )
481
- def resolve_dataset_marker(
482
- session_path: Path,
483
- create_processed_directories: bool,
484
- processed_data_root: Path | None,
485
- remove: bool,
486
- update_manifest: bool,
487
- ) -> None:
488
- """Depending on configuration, either creates or removes the p53.bin marker from the target session.
489
-
490
- The p53.bin marker determines whether the session is ready for dataset integration. When the marker exists,
491
- processing pipelines are not allowed to work with the session data, ensuring that all processed data remains
492
- unchanged. If the marker does not exist, dataset integration pipelines are not allowed to work with the session
493
- data, enabling processing pipelines to safely modify the data at any time.
494
- """
495
- resolve_p53_marker(
496
- session_path=session_path,
497
- create_processed_data_directory=create_processed_directories,
498
- processed_data_root=processed_data_root,
499
- remove=remove,
500
- update_manifest=update_manifest,
501
- )
sl_shared_assets/cli.pyi DELETED
@@ -1,106 +0,0 @@
1
- from pathlib import Path
2
-
3
- from .tools import (
4
- ascend_tyche_data as ascend_tyche_data,
5
- resolve_p53_marker as resolve_p53_marker,
6
- verify_session_checksum as verify_session_checksum,
7
- generate_project_manifest as generate_project_manifest,
8
- )
9
- from .server import (
10
- Server as Server,
11
- JupyterJob as JupyterJob,
12
- generate_server_credentials as generate_server_credentials,
13
- )
14
- from .data_classes import (
15
- SessionData as SessionData,
16
- TrackerFileNames as TrackerFileNames,
17
- get_processing_tracker as get_processing_tracker,
18
- )
19
-
20
- def verify_session_integrity(
21
- session_path: Path,
22
- manager_id: int,
23
- create_processed_directories: bool,
24
- processed_data_root: Path | None,
25
- update_manifest: bool,
26
- ) -> None:
27
- """Checks the integrity of the target session's raw data (contents of the raw_data directory).
28
-
29
- This command assumes that the data has been checksummed during acquisition and contains an ax_checksum.txt file
30
- that stores the data checksum generated before transferring the data to the long-term storage destination. This
31
- function always verified the integrity of the 'raw_data' directory. It does not work with 'processed_data' or any
32
- other directories. If the session data was corrupted, the command removes the 'telomere.bin' file, marking the
33
- session as 'incomplete' and automatically excluding it from all further automated processing runtimes. If the
34
- session data is intact, it generates a 'verified.bin' marker file inside the session's raw_data folder.
35
-
36
- The command is also used by Sun lab data acquisition systems to generate the processed data hierarchy for each
37
- processed session. This use case is fully automated and should not be triggered manually by the user.
38
- """
39
-
40
- def generate_project_manifest_file(
41
- project_path: Path, output_directory: Path, processed_data_root: Path | None
42
- ) -> None:
43
- """Generates the manifest .feather file that provides information about the data-processing state of all available
44
- project sessions.
45
-
46
- The manifest file is typically used when batch-processing session data on the remote compute server. It contains the
47
- comprehensive snapshot of the available project's data in a table-compatible format that can also be transferred
48
- between machines (as it is cached in a file).
49
- """
50
-
51
- def generate_server_credentials_file(
52
- output_directory: Path,
53
- host: str,
54
- username: str,
55
- password: str,
56
- storage_root: str,
57
- working_root: str,
58
- shared_directory_name: str,
59
- ) -> None:
60
- """Generates a new server_credentials.yaml file under the specified directory, using input information.
61
-
62
- This command is used to set up access to compute servers and clusters on new machines (PCs). The data stored inside
63
- the server_credentials.yaml file generated by this command is used by the Server and Job classes used in many Sun
64
- lab data processing libraries.
65
- """
66
-
67
- def ascend_tyche_directory(input_directory: Path) -> None:
68
- """Restructures old Tyche project data to use the modern Sun lab data structure and uploads them to the processing
69
- server.
70
-
71
- This command is used to convert ('ascend') the old Tyche project data to the modern Sun lab structure. After
72
- ascension, the data can be processed and analyzed using all modern Sun lab (sl-) tools and libraries. Note, this
73
- process expects the input data to be preprocessed using an old Sun lab mesoscope data preprocessing pipeline. It
74
- will not work for any other project or data. Also, this command will only work on a machine (PC) that belongs to a
75
- valid Sun lab data acquisition system, such as VRPC of the Mesoscope-VR system.
76
- """
77
-
78
- def start_jupyter_server(
79
- credentials_path: Path, name: str, environment: str, directory: Path, cores: int, memory: int, time: int, port: int
80
- ) -> None:
81
- """Starts an interactive Jupyter session on the remote Sun lab server.
82
-
83
- This command should be used to run Jupyter lab and notebooks sessions on the remote Sun lab server. Since all lab
84
- data is stored on the server, this allows running light interactive analysis sessions on the same node as the data,
85
- while leveraging considerable compute resources of the server.
86
-
87
- Calling this command initializes a SLURM session that runs the interactive Jupyter server. Since this server
88
- directly competes for resources with all other headless jobs running on the server, it is imperative that each
89
- jupyter runtime uses only the minimum amount of resources and run-time as necessary. Do not use this command to run
90
- heavy data processing pipelines! Instead, consult with library documentation and use the headless Job class.
91
- """
92
-
93
- def resolve_dataset_marker(
94
- session_path: Path,
95
- create_processed_directories: bool,
96
- processed_data_root: Path | None,
97
- remove: bool,
98
- update_manifest: bool,
99
- ) -> None:
100
- """Depending on configuration, either creates or removes the p53.bin marker from the target session.
101
-
102
- The p53.bin marker determines whether the session is ready for dataset integration. When the marker exists,
103
- processing pipelines are not allowed to work with the session data, ensuring that all processed data remains
104
- unchanged. If the marker does not exist, dataset integration pipelines are not allowed to work with the session
105
- data, enabling processing pipelines to safely modify the data at any time.
106
- """
@@ -1,75 +0,0 @@
1
- from .runtime_data import (
2
- ZaberPositions as ZaberPositions,
3
- MesoscopePositions as MesoscopePositions,
4
- RunTrainingDescriptor as RunTrainingDescriptor,
5
- LickTrainingDescriptor as LickTrainingDescriptor,
6
- MesoscopeHardwareState as MesoscopeHardwareState,
7
- WindowCheckingDescriptor as WindowCheckingDescriptor,
8
- MesoscopeExperimentDescriptor as MesoscopeExperimentDescriptor,
9
- )
10
- from .session_data import (
11
- RawData as RawData,
12
- SessionData as SessionData,
13
- SessionTypes as SessionTypes,
14
- ProcessedData as ProcessedData,
15
- TrackerFileNames as TrackerFileNames,
16
- ProcessingTracker as ProcessingTracker,
17
- generate_manager_id as generate_manager_id,
18
- get_processing_tracker as get_processing_tracker,
19
- )
20
- from .surgery_data import (
21
- DrugData as DrugData,
22
- ImplantData as ImplantData,
23
- SubjectData as SubjectData,
24
- SurgeryData as SurgeryData,
25
- InjectionData as InjectionData,
26
- ProcedureData as ProcedureData,
27
- )
28
- from .configuration_data import (
29
- MesoscopePaths as MesoscopePaths,
30
- ExperimentState as ExperimentState,
31
- ExperimentTrial as ExperimentTrial,
32
- MesoscopeCameras as MesoscopeCameras,
33
- AcquisitionSystems as AcquisitionSystems,
34
- MesoscopeMicroControllers as MesoscopeMicroControllers,
35
- MesoscopeAdditionalFirmware as MesoscopeAdditionalFirmware,
36
- MesoscopeSystemConfiguration as MesoscopeSystemConfiguration,
37
- MesoscopeExperimentConfiguration as MesoscopeExperimentConfiguration,
38
- get_system_configuration_data as get_system_configuration_data,
39
- set_system_configuration_file as set_system_configuration_file,
40
- )
41
-
42
- __all__ = [
43
- "DrugData",
44
- "ImplantData",
45
- "SessionData",
46
- "RawData",
47
- "ProcessedData",
48
- "SubjectData",
49
- "SurgeryData",
50
- "InjectionData",
51
- "ProcedureData",
52
- "ZaberPositions",
53
- "ExperimentState",
54
- "MesoscopePositions",
55
- "MesoscopeHardwareState",
56
- "RunTrainingDescriptor",
57
- "LickTrainingDescriptor",
58
- "MesoscopeExperimentConfiguration",
59
- "MesoscopeExperimentDescriptor",
60
- "MesoscopeSystemConfiguration",
61
- "set_system_configuration_file",
62
- "get_system_configuration_data",
63
- "MesoscopePaths",
64
- "MesoscopeCameras",
65
- "MesoscopeMicroControllers",
66
- "MesoscopeAdditionalFirmware",
67
- "ProcessingTracker",
68
- "ExperimentTrial",
69
- "AcquisitionSystems",
70
- "SessionTypes",
71
- "WindowCheckingDescriptor",
72
- "get_processing_tracker",
73
- "generate_manager_id",
74
- "TrackerFileNames",
75
- ]