janas 2.2.1__tar.gz → 2.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {janas-2.2.1 → janas-2.2.2}/CHANGELOG.md +23 -0
- {janas-2.2.1 → janas-2.2.2}/PKG-INFO +1 -1
- {janas-2.2.1 → janas-2.2.2}/setup.py +1 -1
- {janas-2.2.1 → janas-2.2.2}/src/janas/janas_cmd_utils.py +21 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas/utils_csparc.py +132 -26
- {janas-2.2.1 → janas-2.2.2}/src/janas.egg-info/PKG-INFO +1 -1
- {janas-2.2.1 → janas-2.2.2}/src/janas.egg-info/SOURCES.txt +1 -0
- janas-2.2.2/tests/test_csparc_finalise.py +198 -0
- {janas-2.2.1 → janas-2.2.2}/tests/test_csparc_login.py +30 -0
- {janas-2.2.1 → janas-2.2.2}/CMakeLists.txt +0 -0
- {janas-2.2.1 → janas-2.2.2}/LICENSE +0 -0
- {janas-2.2.1 → janas-2.2.2}/MANIFEST.in +0 -0
- {janas-2.2.1 → janas-2.2.2}/README.md +0 -0
- {janas-2.2.1 → janas-2.2.2}/pyproject.toml +0 -0
- {janas-2.2.1 → janas-2.2.2}/requirements.txt +0 -0
- {janas-2.2.1 → janas-2.2.2}/setup.cfg +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas/IO_utils.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas/__init__.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas/assessParticles.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas/benchmarking_janas_cmd_session_manager.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas/bin/janas_app_meanMinMax +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas/bin/janas_app_starProcess +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas/bin_wrapper.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas/images/selection_finished.png +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas/images/selection_step1.png +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas/images/selection_step2.png +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas/images/selection_step3.png +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas/images/selection_step4.png +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas/images/selection_unstarted.png +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas/janas_alternative_selector.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas/janas_cmd_caller.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas/janas_cmd_optimizer.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas/janas_cmd_session_manager.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas/janas_developer_test.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas/janas_mapProcess.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas/janas_progress.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas/janas_reconstructor.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas/locres_repro.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas/locres_utils.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas/projector_utils.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas/reconstructor_utils.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas/setup_environment_shell.sh +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas/starDisplay.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas/starHandler.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas/utils.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas/version.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas.egg-info/dependency_links.txt +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas.egg-info/entry_points.txt +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas.egg-info/not-zip-safe +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas.egg-info/requires.txt +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/janas.egg-info/top_level.txt +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/src_cpp/CsvStarReadWriteAnalyseLibs.h +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/src_cpp/ctfLibs.h +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/src_cpp/derivative_libs.h +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/src_cpp/euler_libs.h +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/src_cpp/imageProcessingLib.h +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/src_cpp/janas_app_meanMinMax.cxx +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/src_cpp/janas_app_starProcess.cxx +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/src_cpp/janas_core.cpp +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/src_cpp/matrix_computations.h +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/src_cpp/mrcIO.h +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/src_cpp/randomLibs.h +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/src_cpp/recline.c +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/src_cpp/recline.h +0 -0
- {janas-2.2.1 → janas-2.2.2}/src/src_cpp/scores.h +0 -0
- {janas-2.2.1 → janas-2.2.2}/tests/test_backmap_stars.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/tests/test_behaviour_preserving_fixes.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/tests/test_clean_csparc_blob_path.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/tests/test_create_stack_path_mode.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/tests/test_csparc2star_missing_pose.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/tests/test_custom_selected_stacks.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/tests/test_euler_anchor_assignment.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/tests/test_eulerhist_show_flag.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/tests/test_fixstack.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/tests/test_generated_session_scripts.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/tests/test_janas_progress.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/tests/test_locres_release_fixes.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/tests/test_locres_reproducibility.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/tests/test_locres_v3_closure.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/tests/test_mrc_reader.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/tests/test_reconstructor_nan_guard.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/tests/test_sanitize_particle.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/tests/test_score_particles_arrays_equivalence.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/tests/test_scoreparticles_additional_options.py +0 -0
- {janas-2.2.1 → janas-2.2.2}/tests/test_support_floor.py +0 -0
|
@@ -1,5 +1,28 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 2.2.2
|
|
4
|
+
|
|
5
|
+
- **Refined particle metadata retrieved through the cryosparc-tools API.**
|
|
6
|
+
After a CryoSPARC refinement (`janas_utils csparc_localnurefinement`,
|
|
7
|
+
`csparc_nurefinement`), the job's `particles` output is now loaded with
|
|
8
|
+
`Job.load_output("particles")` and saved next to the output STAR file,
|
|
9
|
+
instead of searching the job directory for `<job>_<iter>_particles.cs`,
|
|
10
|
+
which failed on CryoSPARC installations whose job directories use a
|
|
11
|
+
different layout (`FileNotFoundError: No refined particle .cs files
|
|
12
|
+
found`). The filesystem search remains as a fallback, now also below
|
|
13
|
+
sub-directories and through the job's file listing, and the error message
|
|
14
|
+
lists the job directory contents. Exported half-maps are downloaded through
|
|
15
|
+
the API when they are not found on disk.
|
|
16
|
+
Verified against cryoSPARC v5.0.6 with cryosparc-tools 5.0.3 (local
|
|
17
|
+
refinement and non-uniform refinement jobs); cryoSPARC 4.x with
|
|
18
|
+
cryosparc-tools 4.x remains supported through the legacy login and the
|
|
19
|
+
filesystem fallback.
|
|
20
|
+
- **`--no-ssd-cache` for the cryoSPARC refinement commands.** `janas_utils
|
|
21
|
+
csparc_nurefinement` and `csparc_localnurefinement` can disable particle
|
|
22
|
+
caching on the worker SSD (`compute_use_ssd=False`), which is required on
|
|
23
|
+
lanes without an SSD cache (cluster lanes otherwise fail with "SSD path not
|
|
24
|
+
specified in worker config nor environment variable").
|
|
25
|
+
|
|
3
26
|
## 2.2.1
|
|
4
27
|
|
|
5
28
|
- **Fixed the signal-subtraction test in the generated selection scripts
|
|
@@ -99,7 +99,7 @@ janas_core = Extension(
|
|
|
99
99
|
|
|
100
100
|
setup(
|
|
101
101
|
name='janas',
|
|
102
|
-
version='2.2.
|
|
102
|
+
version='2.2.2',
|
|
103
103
|
description='Pipeline for per-particle selection and 3D class reassignment in single-particle cryo-EM.',
|
|
104
104
|
long_description=read_long_description(),
|
|
105
105
|
long_description_content_type='text/markdown',
|
|
@@ -2747,6 +2747,16 @@ janas_csparc_nurefinement.add_argument(
|
|
|
2747
2747
|
help="force re-splitting of half-sets in NU refinement",
|
|
2748
2748
|
)
|
|
2749
2749
|
|
|
2750
|
+
janas_csparc_nurefinement.add_argument(
|
|
2751
|
+
"--no-ssd-cache",
|
|
2752
|
+
action="store_true",
|
|
2753
|
+
help=(
|
|
2754
|
+
"Do not cache particle images on the worker SSD (compute_use_ssd=False). "
|
|
2755
|
+
"Required on lanes without an SSD cache, e.g. cluster lanes that fail with "
|
|
2756
|
+
"'SSD path not specified in worker config'."
|
|
2757
|
+
),
|
|
2758
|
+
)
|
|
2759
|
+
|
|
2750
2760
|
janas_csparc_nurefinement.add_argument(
|
|
2751
2761
|
"--local",
|
|
2752
2762
|
action="store_true",
|
|
@@ -2851,6 +2861,7 @@ def csparc_nurefinement_cli(args):
|
|
|
2851
2861
|
ref_map=args.ref,
|
|
2852
2862
|
ini_high=args.ini_high,
|
|
2853
2863
|
resplit=args.resplit,
|
|
2864
|
+
use_ssd_cache=not args.no_ssd_cache,
|
|
2854
2865
|
do_local=args.local,
|
|
2855
2866
|
min_angular_step=args.min_angular_step,
|
|
2856
2867
|
loglevel=args.loglevel,
|
|
@@ -2929,6 +2940,15 @@ janas_csparc_localnurefinement.add_argument(
|
|
|
2929
2940
|
"(refine_gs_resplit=True)."
|
|
2930
2941
|
),
|
|
2931
2942
|
)
|
|
2943
|
+
janas_csparc_localnurefinement.add_argument(
|
|
2944
|
+
"--no-ssd-cache",
|
|
2945
|
+
action="store_true",
|
|
2946
|
+
help=(
|
|
2947
|
+
"Do not cache particle images on the worker SSD (compute_use_ssd=False). "
|
|
2948
|
+
"Required on lanes without an SSD cache, e.g. cluster lanes that fail with "
|
|
2949
|
+
"'SSD path not specified in worker config'."
|
|
2950
|
+
),
|
|
2951
|
+
)
|
|
2932
2952
|
janas_csparc_localnurefinement.add_argument(
|
|
2933
2953
|
"--min-angular-step",
|
|
2934
2954
|
type=float,
|
|
@@ -2994,6 +3014,7 @@ def csparc_localnurefinement_cli(args):
|
|
|
2994
3014
|
ref_map=args.ref,
|
|
2995
3015
|
mask_map=args.mask,
|
|
2996
3016
|
resplit=args.resplit,
|
|
3017
|
+
use_ssd_cache=not args.no_ssd_cache,
|
|
2997
3018
|
min_angular_step=args.min_angular_step,
|
|
2998
3019
|
loglevel=args.loglevel,
|
|
2999
3020
|
precomputed_job_uid=args.precomputed,
|
|
@@ -348,6 +348,12 @@ class CryoSPARCToolsSession:
|
|
|
348
348
|
return lane_names
|
|
349
349
|
|
|
350
350
|
# ----- job helpers -----
|
|
351
|
+
def find_job(self, job_uid: str):
|
|
352
|
+
"""
|
|
353
|
+
Return the cryosparc-tools job object for ``job_uid`` in this project.
|
|
354
|
+
"""
|
|
355
|
+
return self._cs.find_job(self.project_uid, job_uid)
|
|
356
|
+
|
|
351
357
|
def get_project_dir(self) -> Path:
|
|
352
358
|
"""
|
|
353
359
|
Return the absolute CryoSPARC project directory for this session.
|
|
@@ -487,6 +493,7 @@ class CryoSPARCToolsSession:
|
|
|
487
493
|
symmetry: str = "C1",
|
|
488
494
|
min_angular_step: float = 0.2,
|
|
489
495
|
resplit: bool = False,
|
|
496
|
+
use_ssd_cache: bool = True,
|
|
490
497
|
):
|
|
491
498
|
"""
|
|
492
499
|
Run a 'new_local_refine' job directly from:
|
|
@@ -516,6 +523,9 @@ class CryoSPARCToolsSession:
|
|
|
516
523
|
# Useful in practice – same meaning as in the GUI
|
|
517
524
|
"use_alignment_prior": False,
|
|
518
525
|
"refine_gs_resplit": bool(resplit),
|
|
526
|
+
# Lanes without an SSD cache (e.g. cluster lanes) fail with
|
|
527
|
+
# "SSD path not specified" unless particle caching is disabled.
|
|
528
|
+
"compute_use_ssd": bool(use_ssd_cache),
|
|
519
529
|
}
|
|
520
530
|
|
|
521
531
|
job = self._cs.create_job(
|
|
@@ -538,6 +548,7 @@ class CryoSPARCToolsSession:
|
|
|
538
548
|
symmetry: str = "C1",
|
|
539
549
|
ini_high: Optional[float] = None,
|
|
540
550
|
resplit: bool = False,
|
|
551
|
+
use_ssd_cache: bool = True,
|
|
541
552
|
):
|
|
542
553
|
"""
|
|
543
554
|
Run non-uniform refinement connecting a particles job and a volume job.
|
|
@@ -547,6 +558,7 @@ class CryoSPARCToolsSession:
|
|
|
547
558
|
params = {
|
|
548
559
|
"refine_symmetry": symmetry,
|
|
549
560
|
"refine_gs_resplit": bool(resplit),
|
|
561
|
+
"compute_use_ssd": bool(use_ssd_cache),
|
|
550
562
|
}
|
|
551
563
|
if ini_high is not None:
|
|
552
564
|
params["refine_res_init"] = float(ini_high)
|
|
@@ -574,6 +586,7 @@ class CryoSPARCToolsSession:
|
|
|
574
586
|
refine_job,
|
|
575
587
|
symmetry: str = "C1",
|
|
576
588
|
min_angular_step: float = 0.01,
|
|
589
|
+
use_ssd_cache: bool = True,
|
|
577
590
|
):
|
|
578
591
|
"""
|
|
579
592
|
Run Local Refinement on top of an existing refinement job.
|
|
@@ -592,6 +605,7 @@ class CryoSPARCToolsSession:
|
|
|
592
605
|
"refine_symmetry": symmetry,
|
|
593
606
|
"use_alignment_prior": True,
|
|
594
607
|
"min_angular_step": float(min_angular_step),
|
|
608
|
+
"compute_use_ssd": bool(use_ssd_cache),
|
|
595
609
|
},
|
|
596
610
|
)
|
|
597
611
|
return self._queue_and_wait(job, label="new_local_refine")
|
|
@@ -603,6 +617,7 @@ def export_refine_maps_from_jobdir(
|
|
|
603
617
|
job_dir: Path,
|
|
604
618
|
refine_job_uid: str,
|
|
605
619
|
output_basename: str,
|
|
620
|
+
job=None,
|
|
606
621
|
):
|
|
607
622
|
"""
|
|
608
623
|
Dato job_dir (es. .../P31/J17) ed un basename (es. 'class_1_updatedNU'),
|
|
@@ -616,18 +631,20 @@ def export_refine_maps_from_jobdir(
|
|
|
616
631
|
out_dir = base.parent if base.parent != Path("") else Path(".")
|
|
617
632
|
stem = base.name
|
|
618
633
|
|
|
619
|
-
# full map
|
|
620
|
-
full_candidates = sorted(job_dir.glob(f"{refine_job_uid}_*_volume_map.mrc"))
|
|
621
|
-
# half-maps
|
|
622
|
-
halfA_candidates = sorted(job_dir.glob(f"{refine_job_uid}_*_volume_map_half_A.mrc"))
|
|
623
|
-
halfB_candidates = sorted(job_dir.glob(f"{refine_job_uid}_*_volume_map_half_B.mrc"))
|
|
624
|
-
|
|
625
634
|
def last_or_none(cands):
|
|
626
635
|
return cands[-1] if cands else None
|
|
627
636
|
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
637
|
+
def find_map(suffix: str) -> Optional[Path]:
|
|
638
|
+
# Files are searched first at the top of the job directory (the
|
|
639
|
+
# historical layout), then anywhere below it.
|
|
640
|
+
cands = sorted(job_dir.glob(f"{refine_job_uid}_*{suffix}"))
|
|
641
|
+
if not cands:
|
|
642
|
+
cands = sorted(p for p in job_dir.rglob(f"*{suffix}") if p.is_file())
|
|
643
|
+
return last_or_none(cands)
|
|
644
|
+
|
|
645
|
+
full_map = find_map("_volume_map.mrc")
|
|
646
|
+
halfA = find_map("_volume_map_half_A.mrc")
|
|
647
|
+
halfB = find_map("_volume_map_half_B.mrc")
|
|
631
648
|
|
|
632
649
|
def safe_copy(src: Optional[Path], dest: Path):
|
|
633
650
|
if src is None:
|
|
@@ -641,6 +658,104 @@ def export_refine_maps_from_jobdir(
|
|
|
641
658
|
safe_copy(halfA, out_dir / f"{stem}_recH1.mrc")
|
|
642
659
|
safe_copy(halfB, out_dir / f"{stem}_recH2.mrc")
|
|
643
660
|
|
|
661
|
+
# Fallback: fetch through the cryosparc-tools job API when the maps are
|
|
662
|
+
# not visible on the local filesystem.
|
|
663
|
+
if job is not None and (full_map is None or halfA is None or halfB is None):
|
|
664
|
+
try:
|
|
665
|
+
names = list(job.list_files(recursive=True))
|
|
666
|
+
except Exception as e:
|
|
667
|
+
print(f"[csparc_nurefinement] WARNING: could not list job files via API: {e}")
|
|
668
|
+
names = []
|
|
669
|
+
for suffix, dest, found in (
|
|
670
|
+
("_volume_map.mrc", out_dir / f"{stem}_rec.mrc", full_map),
|
|
671
|
+
("_volume_map_half_A.mrc", out_dir / f"{stem}_recH1.mrc", halfA),
|
|
672
|
+
("_volume_map_half_B.mrc", out_dir / f"{stem}_recH2.mrc", halfB),
|
|
673
|
+
):
|
|
674
|
+
if found is not None:
|
|
675
|
+
continue
|
|
676
|
+
remote = sorted(n for n in names if str(n).endswith(suffix))
|
|
677
|
+
if not remote:
|
|
678
|
+
continue
|
|
679
|
+
try:
|
|
680
|
+
job.download_file(remote[-1], dest)
|
|
681
|
+
except Exception as e:
|
|
682
|
+
print(f"[csparc_nurefinement] WARNING: failed to download {remote[-1]} -> {dest}: {e}")
|
|
683
|
+
|
|
684
|
+
|
|
685
|
+
def _locate_refined_particles(
|
|
686
|
+
session: CryoSPARCToolsSession,
|
|
687
|
+
refine_job_uid: str,
|
|
688
|
+
job_dir: Path,
|
|
689
|
+
star_out: str,
|
|
690
|
+
):
|
|
691
|
+
"""
|
|
692
|
+
Return (path to a .cs file with the refined particle metadata, job object).
|
|
693
|
+
|
|
694
|
+
The job's ``particles`` output is loaded through the cryosparc-tools API
|
|
695
|
+
(independent of how the CryoSPARC version lays out the job directory)
|
|
696
|
+
and saved next to ``star_out``; the job directory is searched on the
|
|
697
|
+
filesystem only when the API route is not available.
|
|
698
|
+
"""
|
|
699
|
+
out_dir = Path(star_out).resolve().parent
|
|
700
|
+
local_cs = out_dir / f"{Path(star_out).stem}_{refine_job_uid}_particles.cs"
|
|
701
|
+
|
|
702
|
+
job = None
|
|
703
|
+
try:
|
|
704
|
+
job = session.find_job(refine_job_uid)
|
|
705
|
+
except Exception as e:
|
|
706
|
+
print(f"[csparc] WARNING: could not look up job {refine_job_uid} via API: {e}")
|
|
707
|
+
|
|
708
|
+
# 1) API: final version of the 'particles' output, all slots
|
|
709
|
+
if job is not None:
|
|
710
|
+
try:
|
|
711
|
+
dataset = job.load_output("particles")
|
|
712
|
+
dataset.save(local_cs)
|
|
713
|
+
print(f"[csparc] Refined particle metadata for {refine_job_uid} saved to {local_cs}")
|
|
714
|
+
return local_cs, job
|
|
715
|
+
except Exception as e:
|
|
716
|
+
print(f"[csparc] WARNING: could not load the 'particles' output of {refine_job_uid} via API: {e}")
|
|
717
|
+
|
|
718
|
+
# 2) filesystem. Layouts seen so far: <job>_<iter>_particles.cs (non-uniform
|
|
719
|
+
# refinement), particles_<iter>.cs (local refinement, no job prefix) and
|
|
720
|
+
# <job>_particles.cs (reconstruction-only jobs); passthrough and rejected
|
|
721
|
+
# files are never the refined set.
|
|
722
|
+
def _usable(p: Path) -> bool:
|
|
723
|
+
return p.is_file() and "passthrough" not in p.name and "rejected" not in p.name
|
|
724
|
+
|
|
725
|
+
candidates = sorted(p for p in job_dir.glob(f"{refine_job_uid}_*_particles.cs") if _usable(p))
|
|
726
|
+
if not candidates:
|
|
727
|
+
candidates = sorted(p for p in job_dir.glob("particles_*.cs") if _usable(p))
|
|
728
|
+
if not candidates:
|
|
729
|
+
candidates = sorted(
|
|
730
|
+
p for p in job_dir.rglob("*particles*.cs") if _usable(p)
|
|
731
|
+
)
|
|
732
|
+
if candidates:
|
|
733
|
+
return candidates[-1], job
|
|
734
|
+
|
|
735
|
+
# 3) API file listing + download
|
|
736
|
+
if job is not None:
|
|
737
|
+
try:
|
|
738
|
+
names = sorted(
|
|
739
|
+
str(n) for n in job.list_files(recursive=True)
|
|
740
|
+
if str(n).endswith("_particles.cs") and "passthrough" not in str(n)
|
|
741
|
+
)
|
|
742
|
+
if names:
|
|
743
|
+
dataset = job.download_dataset(names[-1])
|
|
744
|
+
dataset.save(local_cs)
|
|
745
|
+
print(f"[csparc] Downloaded {names[-1]} of {refine_job_uid} to {local_cs}")
|
|
746
|
+
return local_cs, job
|
|
747
|
+
except Exception as e:
|
|
748
|
+
print(f"[csparc] WARNING: could not download the particle dataset of {refine_job_uid}: {e}")
|
|
749
|
+
|
|
750
|
+
try:
|
|
751
|
+
listing = sorted(p.name for p in job_dir.iterdir())[:40]
|
|
752
|
+
except Exception:
|
|
753
|
+
listing = []
|
|
754
|
+
raise FileNotFoundError(
|
|
755
|
+
f"No refined particle .cs files found in {job_dir} for job {refine_job_uid}; "
|
|
756
|
+
f"job directory contains: {listing}"
|
|
757
|
+
)
|
|
758
|
+
|
|
644
759
|
|
|
645
760
|
|
|
646
761
|
def _finalise_refinement_job(
|
|
@@ -667,23 +782,8 @@ def _finalise_refinement_job(
|
|
|
667
782
|
f"CryoSPARC job directory does not exist: {job_dir}"
|
|
668
783
|
)
|
|
669
784
|
|
|
670
|
-
#
|
|
671
|
-
|
|
672
|
-
p for p in job_dir.glob(f"{refine_job_uid}_*_particles.cs")
|
|
673
|
-
if "passthrough" not in p.name
|
|
674
|
-
)
|
|
675
|
-
if not particle_cs_candidates:
|
|
676
|
-
particle_cs_candidates = sorted(
|
|
677
|
-
p for p in job_dir.glob("*_particles.cs")
|
|
678
|
-
if "passthrough" not in p.name
|
|
679
|
-
)
|
|
680
|
-
|
|
681
|
-
if not particle_cs_candidates:
|
|
682
|
-
raise FileNotFoundError(
|
|
683
|
-
f"No refined particle .cs files found in {job_dir} for job {refine_job_uid}"
|
|
684
|
-
)
|
|
685
|
-
|
|
686
|
-
csfile = particle_cs_candidates[-1]
|
|
785
|
+
# Refined particle metadata: API first, filesystem as fallback
|
|
786
|
+
csfile, job = _locate_refined_particles(session, refine_job_uid, job_dir, star_out)
|
|
687
787
|
|
|
688
788
|
# Update STAR from .cs
|
|
689
789
|
utils.update_star_from_csparc(
|
|
@@ -699,6 +799,7 @@ def _finalise_refinement_job(
|
|
|
699
799
|
job_dir=job_dir,
|
|
700
800
|
refine_job_uid=refine_job_uid,
|
|
701
801
|
output_basename=output_basename,
|
|
802
|
+
job=job,
|
|
702
803
|
)
|
|
703
804
|
|
|
704
805
|
return str(csfile)
|
|
@@ -727,6 +828,7 @@ def run_csparc_nu_refinement_and_update_star(
|
|
|
727
828
|
output_basename: Optional[str] = None,
|
|
728
829
|
precomputed_job_uid: Optional[str] = None,
|
|
729
830
|
base_url: Optional[str] = None,
|
|
831
|
+
use_ssd_cache: bool = True,
|
|
730
832
|
) -> str:
|
|
731
833
|
"""
|
|
732
834
|
High-level pipeline:
|
|
@@ -816,6 +918,7 @@ def run_csparc_nu_refinement_and_update_star(
|
|
|
816
918
|
symmetry=symmetry,
|
|
817
919
|
ini_high=ini_high,
|
|
818
920
|
resplit=resplit,
|
|
921
|
+
use_ssd_cache=use_ssd_cache,
|
|
819
922
|
)
|
|
820
923
|
|
|
821
924
|
# 5b) optional local refinement *on top* of NU
|
|
@@ -824,6 +927,7 @@ def run_csparc_nu_refinement_and_update_star(
|
|
|
824
927
|
refine_job=refine_job,
|
|
825
928
|
symmetry=symmetry,
|
|
826
929
|
min_angular_step=min_angular_step,
|
|
930
|
+
use_ssd_cache=use_ssd_cache,
|
|
827
931
|
)
|
|
828
932
|
|
|
829
933
|
# 6–8) common post-processing
|
|
@@ -860,6 +964,7 @@ def run_csparc_local_refinement_and_update_star(
|
|
|
860
964
|
output_basename: Optional[str] = None,
|
|
861
965
|
precomputed_job_uid: Optional[str] = None,
|
|
862
966
|
base_url: Optional[str] = None,
|
|
967
|
+
use_ssd_cache: bool = True,
|
|
863
968
|
) -> str:
|
|
864
969
|
"""
|
|
865
970
|
Local refinement pipeline:
|
|
@@ -955,6 +1060,7 @@ def run_csparc_local_refinement_and_update_star(
|
|
|
955
1060
|
symmetry=symmetry,
|
|
956
1061
|
min_angular_step=min_angular_step,
|
|
957
1062
|
resplit=resplit,
|
|
1063
|
+
use_ssd_cache=use_ssd_cache,
|
|
958
1064
|
)
|
|
959
1065
|
|
|
960
1066
|
# 7–9) common post-processing
|
|
@@ -65,6 +65,7 @@ tests/test_behaviour_preserving_fixes.py
|
|
|
65
65
|
tests/test_clean_csparc_blob_path.py
|
|
66
66
|
tests/test_create_stack_path_mode.py
|
|
67
67
|
tests/test_csparc2star_missing_pose.py
|
|
68
|
+
tests/test_csparc_finalise.py
|
|
68
69
|
tests/test_csparc_login.py
|
|
69
70
|
tests/test_custom_selected_stacks.py
|
|
70
71
|
tests/test_euler_anchor_assignment.py
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Retrieval of the refined particle metadata after a CryoSPARC refinement job.
|
|
3
|
+
|
|
4
|
+
``_finalise_refinement_job`` must not depend on the on-disk layout of the
|
|
5
|
+
job directory (which differs between CryoSPARC versions): the ``particles``
|
|
6
|
+
output is loaded through the cryosparc-tools job API first, and the job
|
|
7
|
+
directory is searched on the filesystem only as a fallback. No CryoSPARC
|
|
8
|
+
instance is needed: the session and job objects are stubs.
|
|
9
|
+
|
|
10
|
+
Run with::
|
|
11
|
+
|
|
12
|
+
python -m pytest tests/test_csparc_finalise.py -v
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import importlib
|
|
18
|
+
import os
|
|
19
|
+
import sys
|
|
20
|
+
import types
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
|
|
23
|
+
import pytest
|
|
24
|
+
|
|
25
|
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
26
|
+
SRC = os.path.normpath(os.path.join(HERE, "..", "src"))
|
|
27
|
+
if os.path.isdir(SRC) and SRC not in sys.path:
|
|
28
|
+
sys.path.insert(0, SRC)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@pytest.fixture
|
|
32
|
+
def utils_csparc():
|
|
33
|
+
"""janas.utils_csparc importable even when cryosparc-tools is absent."""
|
|
34
|
+
stubbed = importlib.util.find_spec("cryosparc") is None
|
|
35
|
+
if stubbed:
|
|
36
|
+
fake_tools = types.ModuleType("cryosparc.tools")
|
|
37
|
+
fake_tools.CryoSPARC = object
|
|
38
|
+
fake_pkg = types.ModuleType("cryosparc")
|
|
39
|
+
fake_pkg.tools = fake_tools
|
|
40
|
+
sys.modules["cryosparc"] = fake_pkg
|
|
41
|
+
sys.modules["cryosparc.tools"] = fake_tools
|
|
42
|
+
import janas.utils_csparc as module
|
|
43
|
+
try:
|
|
44
|
+
yield module
|
|
45
|
+
finally:
|
|
46
|
+
if stubbed:
|
|
47
|
+
# Only remove what this fixture inserted; a real cryosparc-tools
|
|
48
|
+
# package must stay imported (re-importing it re-registers its
|
|
49
|
+
# models and fails).
|
|
50
|
+
sys.modules.pop("cryosparc.tools", None)
|
|
51
|
+
sys.modules.pop("cryosparc", None)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class _FakeDataset:
|
|
55
|
+
def __init__(self, tag: str):
|
|
56
|
+
self.tag = tag
|
|
57
|
+
|
|
58
|
+
def save(self, path, *, format=1):
|
|
59
|
+
Path(path).write_text(self.tag)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class _FakeJob:
|
|
63
|
+
def __init__(self, load_error=None, files=(), dataset_tag="api"):
|
|
64
|
+
self.uid = "J30"
|
|
65
|
+
self.load_error = load_error
|
|
66
|
+
self.files = list(files)
|
|
67
|
+
self.dataset_tag = dataset_tag
|
|
68
|
+
self.downloaded = []
|
|
69
|
+
|
|
70
|
+
def load_output(self, name, slots="all", version="F"):
|
|
71
|
+
assert name == "particles"
|
|
72
|
+
if self.load_error is not None:
|
|
73
|
+
raise self.load_error
|
|
74
|
+
return _FakeDataset(self.dataset_tag)
|
|
75
|
+
|
|
76
|
+
def list_files(self, prefix="", recursive=False):
|
|
77
|
+
return list(self.files)
|
|
78
|
+
|
|
79
|
+
def download_dataset(self, path):
|
|
80
|
+
self.downloaded.append(str(path))
|
|
81
|
+
return _FakeDataset("downloaded:" + str(path))
|
|
82
|
+
|
|
83
|
+
def download_file(self, path, target=""):
|
|
84
|
+
self.downloaded.append(str(path))
|
|
85
|
+
Path(target).write_bytes(b"mrc")
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
class _FakeSession:
|
|
89
|
+
def __init__(self, project_dir: Path, job):
|
|
90
|
+
self._project_dir = project_dir
|
|
91
|
+
self._job = job
|
|
92
|
+
|
|
93
|
+
def get_project_dir(self):
|
|
94
|
+
return self._project_dir
|
|
95
|
+
|
|
96
|
+
def find_job(self, job_uid):
|
|
97
|
+
assert job_uid == "J30"
|
|
98
|
+
return self._job
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _run(module, tmp_path, job, monkeypatch, output_basename=None):
|
|
102
|
+
project_dir = tmp_path / "CS-project"
|
|
103
|
+
(project_dir / "J30").mkdir(parents=True, exist_ok=True)
|
|
104
|
+
star_in = tmp_path / "in.star"
|
|
105
|
+
star_in.write_text("data_particles\n")
|
|
106
|
+
star_out = tmp_path / "out" / "refined.star"
|
|
107
|
+
star_out.parent.mkdir(exist_ok=True)
|
|
108
|
+
calls = []
|
|
109
|
+
monkeypatch.setattr(
|
|
110
|
+
module.utils, "update_star_from_csparc",
|
|
111
|
+
lambda **kw: calls.append(kw),
|
|
112
|
+
)
|
|
113
|
+
result = module._finalise_refinement_job(
|
|
114
|
+
session=_FakeSession(project_dir, job),
|
|
115
|
+
refine_job_uid="J30",
|
|
116
|
+
star_in=str(star_in),
|
|
117
|
+
star_out=str(star_out),
|
|
118
|
+
loglevel="WARNING",
|
|
119
|
+
output_basename=output_basename,
|
|
120
|
+
)
|
|
121
|
+
return Path(result), calls, project_dir / "J30"
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def test_particles_output_is_loaded_through_the_job_api(utils_csparc, tmp_path, monkeypatch):
|
|
125
|
+
cs, calls, _ = _run(utils_csparc, tmp_path, _FakeJob(), monkeypatch)
|
|
126
|
+
assert cs.name == "refined_J30_particles.cs"
|
|
127
|
+
assert cs.parent == tmp_path / "out"
|
|
128
|
+
assert cs.read_text() == "api"
|
|
129
|
+
assert calls[0]["csfile"] == str(cs)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def test_filesystem_fallback_when_the_api_route_fails(utils_csparc, tmp_path, monkeypatch):
|
|
133
|
+
project_dir = tmp_path / "CS-project"
|
|
134
|
+
job_dir = project_dir / "J30"
|
|
135
|
+
job_dir.mkdir(parents=True)
|
|
136
|
+
(job_dir / "J30_002_particles.cs").write_text("old")
|
|
137
|
+
(job_dir / "J30_005_particles.cs").write_text("last")
|
|
138
|
+
(job_dir / "J30_005_particles_passthrough.cs").write_text("skip")
|
|
139
|
+
job = _FakeJob(load_error=TypeError("no results"))
|
|
140
|
+
cs, calls, _ = _run(utils_csparc, tmp_path, job, monkeypatch)
|
|
141
|
+
assert cs == job_dir / "J30_005_particles.cs"
|
|
142
|
+
assert calls[0]["csfile"] == str(cs)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def test_nested_layout_is_found_on_the_filesystem(utils_csparc, tmp_path, monkeypatch):
|
|
146
|
+
project_dir = tmp_path / "CS-project"
|
|
147
|
+
nested = project_dir / "J30" / "outputs" / "particles"
|
|
148
|
+
nested.mkdir(parents=True)
|
|
149
|
+
(nested / "J30_009_particles.cs").write_text("nested")
|
|
150
|
+
job = _FakeJob(load_error=RuntimeError("api down"))
|
|
151
|
+
cs, _, _ = _run(utils_csparc, tmp_path, job, monkeypatch)
|
|
152
|
+
assert cs == nested / "J30_009_particles.cs"
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def test_api_file_listing_is_used_before_giving_up(utils_csparc, tmp_path, monkeypatch):
|
|
156
|
+
job = _FakeJob(load_error=TypeError("no results"), files=["J30_007_particles.cs", "J30_007_particles_passthrough.cs"])
|
|
157
|
+
cs, calls, _ = _run(utils_csparc, tmp_path, job, monkeypatch)
|
|
158
|
+
assert job.downloaded == ["J30_007_particles.cs"]
|
|
159
|
+
assert cs.read_text() == "downloaded:J30_007_particles.cs"
|
|
160
|
+
assert calls[0]["csfile"] == str(cs)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def test_error_lists_the_job_directory(utils_csparc, tmp_path, monkeypatch):
|
|
164
|
+
project_dir = tmp_path / "CS-project"
|
|
165
|
+
job_dir = project_dir / "J30"
|
|
166
|
+
job_dir.mkdir(parents=True)
|
|
167
|
+
(job_dir / "job.json").write_text("{}")
|
|
168
|
+
job = _FakeJob(load_error=TypeError("no results"))
|
|
169
|
+
with pytest.raises(FileNotFoundError) as excinfo:
|
|
170
|
+
_run(utils_csparc, tmp_path, job, monkeypatch)
|
|
171
|
+
assert "job.json" in str(excinfo.value)
|
|
172
|
+
assert "J30" in str(excinfo.value)
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def test_maps_are_downloaded_when_absent_from_the_filesystem(utils_csparc, tmp_path, monkeypatch):
|
|
176
|
+
job = _FakeJob(files=["J30_009_volume_map.mrc", "J30_009_volume_map_half_A.mrc", "J30_009_volume_map_half_B.mrc"])
|
|
177
|
+
basename = tmp_path / "out" / "refined_maps"
|
|
178
|
+
(tmp_path / "out").mkdir(exist_ok=True)
|
|
179
|
+
_run(utils_csparc, tmp_path, job, monkeypatch, output_basename=str(basename))
|
|
180
|
+
assert (tmp_path / "out" / "refined_maps_rec.mrc").read_bytes() == b"mrc"
|
|
181
|
+
assert (tmp_path / "out" / "refined_maps_recH1.mrc").exists()
|
|
182
|
+
assert (tmp_path / "out" / "refined_maps_recH2.mrc").exists()
|
|
183
|
+
assert sorted(job.downloaded) == sorted(job.files)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def test_local_refinement_layout_without_job_prefix_is_found(utils_csparc, tmp_path, monkeypatch):
|
|
187
|
+
# Layout written by cryoSPARC local refinement jobs: particles_<iter>.cs
|
|
188
|
+
# without the job prefix, next to passthrough and rejected files.
|
|
189
|
+
project_dir = tmp_path / "CS-project"
|
|
190
|
+
job_dir = project_dir / "J30"
|
|
191
|
+
job_dir.mkdir(parents=True)
|
|
192
|
+
for name in ("particles_0000.cs", "particles_0004.cs", "particles_rejected_0004.cs",
|
|
193
|
+
"J30_passthrough_particles.cs", "J30_particles.csg"):
|
|
194
|
+
(job_dir / name).write_text(name)
|
|
195
|
+
job = _FakeJob(load_error=TypeError("no results"))
|
|
196
|
+
cs, calls, _ = _run(utils_csparc, tmp_path, job, monkeypatch)
|
|
197
|
+
assert cs == job_dir / "particles_0004.cs"
|
|
198
|
+
assert calls[0]["csfile"] == str(cs)
|
|
@@ -256,3 +256,33 @@ def test_cli_user_is_optional_on_the_token_path(monkeypatch, tmp_path):
|
|
|
256
256
|
monkeypatch.setenv("CRYOSPARC_BASE_URL", "https://cryo.example.org:39000")
|
|
257
257
|
assert cmd_utils._resolve_cryosparc_user(None) is None
|
|
258
258
|
assert cmd_utils._resolve_cryosparc_user("me@example.org") == "me@example.org"
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
# ---------------------------------------------------------------------------
|
|
262
|
+
# SSD cache flag reaches the cryoSPARC job parameters
|
|
263
|
+
# ---------------------------------------------------------------------------
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def test_ssd_cache_flag_reaches_the_job_parameters(monkeypatch, utils_csparc):
|
|
267
|
+
monkeypatch.setenv("CRYOSPARC_BASE_URL", "https://cryo.example.org:39000")
|
|
268
|
+
session = _session(utils_csparc)
|
|
269
|
+
created = []
|
|
270
|
+
|
|
271
|
+
def create_job(project_uid, workspace_uid, job_type, params=None, connections=None):
|
|
272
|
+
created.append((job_type, dict(params or {})))
|
|
273
|
+
return types.SimpleNamespace(uid="J9", type=job_type)
|
|
274
|
+
|
|
275
|
+
session._cs.create_job = create_job
|
|
276
|
+
monkeypatch.setattr(session, "_queue_and_wait", lambda job, label=None: job)
|
|
277
|
+
particles = types.SimpleNamespace(uid="J1")
|
|
278
|
+
volume = types.SimpleNamespace(uid="J2")
|
|
279
|
+
|
|
280
|
+
session.local_refine_from_imports(particles, volume, "imported_volume_1", use_ssd_cache=False)
|
|
281
|
+
session.nonuniform_refine(particles, volume, "imported_volume_1", use_ssd_cache=False)
|
|
282
|
+
session.local_refine(types.SimpleNamespace(uid="J3"), use_ssd_cache=False)
|
|
283
|
+
assert [t for t, _ in created] == ["new_local_refine", "nonuniform_refine_new", "new_local_refine"]
|
|
284
|
+
assert all(p["compute_use_ssd"] is False for _, p in created)
|
|
285
|
+
|
|
286
|
+
created.clear()
|
|
287
|
+
session.local_refine_from_imports(particles, volume, "imported_volume_1")
|
|
288
|
+
assert created[0][1]["compute_use_ssd"] is True
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|