cryoemservices 0.1.3__tar.gz

cryoemservices-0.1.3/LICENSE

@@ -0,0 +1,28 @@
+BSD 3-Clause License
+
+Copyright (c) 2023, Diamond Light Source
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

cryoemservices-0.1.3/PKG-INFO

@@ -0,0 +1,156 @@
+Metadata-Version: 2.1
+Name: cryoemservices
+Version: 0.1.3
+Summary: Services for CryoEM processing
+Author: Diamond Light Source - Data Analysis et al.
+Author-email: dataanalysis@diamond.ac.uk
+License: BSD 3-Clause
+Project-URL: GitHub, https://github.com/DiamondLightSource/cryoem-services
+Project-URL: Bug-Tracker, https://github.com/DiamondLightSource/cryoem-services/issues
+Keywords: cryoem-services
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: GNU General Public License v2 (GPLv2)
+Classifier: Natural Language :: English
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: gemmi
+Requires-Dist: htcondor
+Requires-Dist: icebreaker-em
+Requires-Dist: importlib_metadata
+Requires-Dist: ispyb
+Requires-Dist: marshmallow-sqlalchemy
+Requires-Dist: mrcfile
+Requires-Dist: numpy
+Requires-Dist: pillow
+Requires-Dist: plotly
+Requires-Dist: pydantic==1.10.7
+Requires-Dist: starfile
+Requires-Dist: workflows
+Requires-Dist: zocalo
+
+# cryoem-services
+Services and configuration for cryo-EM pipelines.
+
+This package consists of a number of services to process cryo-EM micrographs,
+both for single particle analysis and tomography,
+using a range of commonly used cryo-EM processing software.
+These services can be run independently to process data,
+or as part of a wider structure for performing live analysis during microscope collection.
+For live analysis, this package integrates with a package
+for transferring and monitoring collected data,
+[Murfey](https://github.com/DiamondLightSource/python-murfey),
+and a database for storing processing outcomes,
+[ISPyB](https://github.com/DiamondLightSource/ispyb-database).
+
+To run these services the software executables being called must be installed.
+These do not come with this package.
+
+
+# Tomography processing
+
+The tomography processing pipeline consists of:
+- Motion correction
+- CTF estimation
+- Tomogram alignment
+- Tomogram denoising using [Topaz](http://topaz-em.readthedocs.io)
+
+
+# Single particle analysis
+
+The single particle analysis pipeline produces a project
+that can be opened and continued using
+[CCP-EM doppio](https://www.ccpem.ac.uk/docs/doppio/user_guide.html)
+or [Relion](https://relion.readthedocs.io).
+
+The processing pipeline consists of:
+- Motion correction
+- CTF estimation
+- Particle picking
+- (Optionally) Ice thickness estimation 
+- Particle extraction and rebatching
+- 2D classification using Relion
+- Automated 2D class selection using Relion
+- 3D classification using Relion
+
+
+# Services currently available
+
+The following services are provided for running the pipelines:
+- Utility services:
+    - **ClusterSubmission**: Submits zocalo wrappers to an HPC cluster
+    - **Dispatcher**: Converts recipes into messages suitable for processing services
+    - **Images**: Creates thumbnail images for viewing processing outcomes
+    - **ISPyB**: Inserts results into an ISPyB database
+    - **NodeCreator**: Creates Relion project files for the services run
+- Processing services:
+    - **CrYOLO**: Particle picking on micrographs using 
+[crYOLO](https://cryolo.readthedocs.io)
+    - **CTFFind**: CTF estimation on micrographs using 
+[CTFFIND4](https://grigoriefflab.umassmed.edu/ctffind4)
+    - **Extract**: Extracts picked particles from micrographs
+    - **IceBreaker**: Ice thickness estimation with 
+[IceBreaker](https://github.com/DiamondLightSource/python-icebreaker)
+    - **MotionCorr**: Motion correction of micrographs using 
+[MotionCor2](http://emcore.ucsf.edu/ucsf-software) 
+or [Relion](https://relion.readthedocs.io)
+    - **MotionCorrSlurm**: MotionCor2 processing submitted to a slurm HPC cluster
+    - **SelectClasses**: Runs automated 2D class selection using 
+[Relion](https://relion.readthedocs.io) and re-batches the particles from these classes
+    - **SelectParticles**: Creates files listing batches of extracted particles
+    - **TomoAlign**: Tomogram reconstruction from a list of micrographs using 
+[imod](https://bio3d.colorado.edu/imod) and [AreTomo](http://msg.ucsf.edu/software)
+
+There are also two zocalo wrapper scripts that can be run on an HPC cluster.
+These perform 2D and 3D classification using [Relion](https://relion.readthedocs.io).
+
+
+# Running services using zocalo
+The services in this package are run using
+[zocalo](https://github.com/DiamondLightSource/python-zocalo)
+and [python-workflows](https://github.com/DiamondLightSource/python-workflows).
+To start a service run the `zocalo.service` command and specify the service name.
+For example, to start a motion correction service:
+
+```bash
+$ zocalo.service -s MotionCorr
+```
+
+Once started, these services will initialise and then wait for messages to be sent to them.
+Messages are sent through a message broker,
+currently [RabbitMQ](http://www.rabbitmq.com) is supported using pika transport in `python-workflows`.
+Individual processing stages can be run by sending a dictionary of the parameters,
+but the processing pipelines are designed to run through recipes.
+
+A recipe is a specication of a series of steps to carry out,
+and how these steps interact with each other.
+Recipes for the current processing pipelines are provided in the `recipes` folder.
+
+To run a recipe in python a dictionary needs to be provided consisting of
+the recipe name and the parameters expected by the recipe.
+The following snippet shows an example of the setup needed.
+This will send a message to a running **Dispatcher** service which
+prepares the recipe for the processing services.
+
+```python
+import workflows.transport.pika_transport as pt
+
+example_message = {
+    "recipes": ["em-tomo-align"],
+    "parameters": {
+        "path_pattern": "micrograph_*.mrc",
+        "pix_size": "1",
+        ...
+    },
+}
+
+transport = pt.PikaTransport()
+transport.connect()
+transport.send("processing_recipe", example_message)
+```
+

cryoemservices-0.1.3/README.md

@@ -0,0 +1,120 @@
+# cryoem-services
+Services and configuration for cryo-EM pipelines.
+
+This package consists of a number of services to process cryo-EM micrographs,
+both for single particle analysis and tomography,
+using a range of commonly used cryo-EM processing software.
+These services can be run independently to process data,
+or as part of a wider structure for performing live analysis during microscope collection.
+For live analysis, this package integrates with a package
+for transferring and monitoring collected data,
+[Murfey](https://github.com/DiamondLightSource/python-murfey),
+and a database for storing processing outcomes,
+[ISPyB](https://github.com/DiamondLightSource/ispyb-database).
+
+To run these services the software executables being called must be installed.
+These do not come with this package.
+
+
+# Tomography processing
+
+The tomography processing pipeline consists of:
+- Motion correction
+- CTF estimation
+- Tomogram alignment
+- Tomogram denoising using [Topaz](http://topaz-em.readthedocs.io)
+
+
+# Single particle analysis
+
+The single particle analysis pipeline produces a project
+that can be opened and continued using
+[CCP-EM doppio](https://www.ccpem.ac.uk/docs/doppio/user_guide.html)
+or [Relion](https://relion.readthedocs.io).
+
+The processing pipeline consists of:
+- Motion correction
+- CTF estimation
+- Particle picking
+- (Optionally) Ice thickness estimation 
+- Particle extraction and rebatching
+- 2D classification using Relion
+- Automated 2D class selection using Relion
+- 3D classification using Relion
+
+
+# Services currently available
+
+The following services are provided for running the pipelines:
+- Utility services:
+    - **ClusterSubmission**: Submits zocalo wrappers to an HPC cluster
+    - **Dispatcher**: Converts recipes into messages suitable for processing services
+    - **Images**: Creates thumbnail images for viewing processing outcomes
+    - **ISPyB**: Inserts results into an ISPyB database
+    - **NodeCreator**: Creates Relion project files for the services run
+- Processing services:
+    - **CrYOLO**: Particle picking on micrographs using 
+[crYOLO](https://cryolo.readthedocs.io)
+    - **CTFFind**: CTF estimation on micrographs using 
+[CTFFIND4](https://grigoriefflab.umassmed.edu/ctffind4)
+    - **Extract**: Extracts picked particles from micrographs
+    - **IceBreaker**: Ice thickness estimation with 
+[IceBreaker](https://github.com/DiamondLightSource/python-icebreaker)
+    - **MotionCorr**: Motion correction of micrographs using 
+[MotionCor2](http://emcore.ucsf.edu/ucsf-software) 
+or [Relion](https://relion.readthedocs.io)
+    - **MotionCorrSlurm**: MotionCor2 processing submitted to a slurm HPC cluster
+    - **SelectClasses**: Runs automated 2D class selection using 
+[Relion](https://relion.readthedocs.io) and re-batches the particles from these classes
+    - **SelectParticles**: Creates files listing batches of extracted particles
+    - **TomoAlign**: Tomogram reconstruction from a list of micrographs using 
+[imod](https://bio3d.colorado.edu/imod) and [AreTomo](http://msg.ucsf.edu/software)
+
+There are also two zocalo wrapper scripts that can be run on an HPC cluster.
+These perform 2D and 3D classification using [Relion](https://relion.readthedocs.io).
+
+
+# Running services using zocalo
+The services in this package are run using
+[zocalo](https://github.com/DiamondLightSource/python-zocalo)
+and [python-workflows](https://github.com/DiamondLightSource/python-workflows).
+To start a service run the `zocalo.service` command and specify the service name.
+For example, to start a motion correction service:
+
+```bash
+$ zocalo.service -s MotionCorr
+```
+
+Once started, these services will initialise and then wait for messages to be sent to them.
+Messages are sent through a message broker,
+currently [RabbitMQ](http://www.rabbitmq.com) is supported using pika transport in `python-workflows`.
+Individual processing stages can be run by sending a dictionary of the parameters,
+but the processing pipelines are designed to run through recipes.
+
+A recipe is a specication of a series of steps to carry out,
+and how these steps interact with each other.
+Recipes for the current processing pipelines are provided in the `recipes` folder.
+
+To run a recipe in python a dictionary needs to be provided consisting of
+the recipe name and the parameters expected by the recipe.
+The following snippet shows an example of the setup needed.
+This will send a message to a running **Dispatcher** service which
+prepares the recipe for the processing services.
+
+```python
+import workflows.transport.pika_transport as pt
+
+example_message = {
+    "recipes": ["em-tomo-align"],
+    "parameters": {
+        "path_pattern": "micrograph_*.mrc",
+        "pix_size": "1",
+        ...
+    },
+}
+
+transport = pt.PikaTransport()
+transport.connect()
+transport.send("processing_recipe", example_message)
+```
+

cryoemservices-0.1.3/pyproject.toml

@@ -0,0 +1,6 @@
+[build-system]
+requires = ["setuptools >= 40.6.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[tool.isort]
+profile="black"

cryoemservices-0.1.3/setup.cfg

@@ -0,0 +1,100 @@
+[metadata]
+name = cryoemservices
+version = 0.1.3
+description = Services for CryoEM processing
+long_description = file: README.md
+long_description_content_type = text/markdown
+author = Diamond Light Source - Data Analysis et al.
+author_email = dataanalysis@diamond.ac.uk
+license = BSD 3-Clause
+license_files = LICENSE
+classifiers = 
+	Development Status :: 3 - Alpha
+	Intended Audience :: Developers
+	License :: OSI Approved :: GNU General Public License v2 (GPLv2)
+	Natural Language :: English
+	Programming Language :: Python :: 3
+	Programming Language :: Python :: 3.8
+	Programming Language :: Python :: 3.9
+	Programming Language :: Python :: 3.10
+keywords = cryoem-services
+project_urls = 
+	GitHub = https://github.com/DiamondLightSource/cryoem-services
+	Bug-Tracker = https://github.com/DiamondLightSource/cryoem-services/issues
+
+[options]
+include_package_data = True
+install_requires = 
+	gemmi
+	htcondor
+	icebreaker-em
+	importlib_metadata
+	ispyb
+	marshmallow-sqlalchemy
+	mrcfile
+	numpy
+	pillow
+	plotly
+	pydantic ==1.10.7
+	starfile
+	workflows
+	zocalo
+packages = find:
+package_dir = 
+	=src
+python_requires = >=3.8
+zip_safe = False
+
+[options.entry_points]
+ccpem_pipeliner.jobs = 
+	combine_star_files_job = cryoemservices.pipeliner_plugins.combine_star_job:ProcessStarFiles
+console_scripts = 
+	combine_star_files = cryoemservices.pipeliner_plugins.combine_star_files:main
+	cryoemservices.resubmit_wrapper = cryoemservices.cli.resubmit_wrapper:run
+cryoemservices.services.cluster.schedulers = 
+	slurm = cryoemservices.services.cluster_submission:submit_to_slurm
+cryoemservices.services.images.plugins = 
+	mrc_to_jpeg = cryoemservices.services.images_plugins:mrc_to_jpeg
+	picked_particles = cryoemservices.services.images_plugins:picked_particles
+	mrc_central_slice = cryoemservices.services.images_plugins:mrc_central_slice
+	mrc_to_apng = cryoemservices.services.images_plugins:mrc_to_apng
+workflows.services = 
+	ClusterSubmission = cryoemservices.services.cluster_submission:ClusterSubmission
+	CrYOLO = cryoemservices.services.cryolo:CrYOLO
+	CTFFind = cryoemservices.services.ctffind:CTFFind
+	DenoiseIris = cryoemservices.services.denoise_iris:DenoiseIris
+	Extract = cryoemservices.services.extract:Extract
+	IceBreaker = cryoemservices.services.icebreaker:IceBreaker
+	Images = cryoemservices.services.images:Images
+	ISPyB = cryoemservices.services.ispyb:EMISPyB
+	MotionCorr = cryoemservices.services.motioncorr:MotionCorr
+	MotionCorrSlurm = cryoemservices.services.motioncorr_slurm:MotionCorrSlurm
+	NodeCreator = cryoemservices.services.node_creator:NodeCreator
+	SelectClasses = cryoemservices.services.select_classes:SelectClasses
+	SelectParticles = cryoemservices.services.select_particles:SelectParticles
+	TomoAlign = cryoemservices.services.tomo_align:TomoAlign
+	TomoAlignIris = cryoemservices.services.tomo_align_iris:TomoAlignIris
+zocalo.services.dispatcher.filters = 
+	ispyb = cryoemservices.util.dispatcher_tools:ispyb_filter
+zocalo.services.dispatcher.ready_for_processing = 
+	ispyb = cryoemservices.util.dispatcher_tools:ready_for_processing
+zocalo.wrappers = 
+	Class2D = cryoemservices.wrappers.class2d_wrapper:Class2DWrapper
+	Class3D = cryoemservices.wrappers.class3d_wrapper:Class3DWrapper
+
+[options.packages.find]
+where = src
+
+[flake8]
+ignore = E203, E266, E501, W503
+max-line-length = 88
+select = 
+	E401,E711,E712,E713,E714,E721,E722,E901,
+	F401,F402,F403,F405,F541,F631,F632,F633,F811,F812,F821,F822,F841,F901,
+	W191,W291,W292,W293,W602,W603,W604,W605,W606,
+	C4,
+
+[egg_info]
+tag_build = 
+tag_date = 0
+

cryoemservices-0.1.3/setup.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+
+from __future__ import annotations
+
+import setuptools
+
+if __name__ == "__main__":
+    # Do not add any parameters here. Edit setup.cfg instead.
+    setuptools.setup()

cryoemservices-0.1.3/src/cryoemservices/__init__.py

@@ -0,0 +1,3 @@
+from __future__ import annotations
+
+__version__ = "0.1.3"

cryoemservices-0.1.3/src/cryoemservices/cli/resubmit_wrapper.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+import workflows.transport.pika_transport as pt
+from workflows.recipe import RecipeWrapper
+
+
+def run():
+    parser = argparse.ArgumentParser(
+        description="Resubmit a failed zocalo wrapper script using the .recipewrap file"
+    )
+    parser.add_argument(
+        "-w",
+        "--wrapper",
+        help="Location of the .recipewrap wrapper file to resubmit",
+        dest="wrapper",
+        required=True,
+    )
+    parser.add_argument(
+        "-c",
+        "--config",
+        help="Transport configuration file for connecting to the message broker",
+        dest="config",
+        required=True,
+    )
+    args = parser.parse_args()
+
+    if not Path(args.wrapper).is_file():
+        print(f"{args.wrapper} cannot be found")
+        return
+    if not Path(args.config).is_file():
+        print(f"{args.config} cannot be found")
+        return
+
+    # Connect to the message transport
+    transport = pt.PikaTransport()
+    transport.load_configuration_file(args.config)
+    transport.connect()
+
+    # Load and submit the wrapper part of the recipe
+    with open(args.wrapper, "r") as wrap:
+        recipe = json.load(wrap)
+    rw = RecipeWrapper(message=recipe, transport=transport)
+    rw._send_to_destination(rw.recipe_pointer, None, rw.payload, {})
+
+
+if __name__ == "__main__":
+    run()