pytme 0.2.0b0__tar.gz → 0.2.2__tar.gz

{pytme-0.2.0b0 → pytme-0.2.2}/MANIFEST.in

@@ -60,7 +60,6 @@ global-exclude .DS_Store
 global-exclude .gitconfig
 global-exclude *.pickle
 prune tme/temp
-prune tme/preprocessing
 global-exclude doc/compileApiReference.sh
 prune public
 global-exclude tme/matching_backend.py
@@ -70,4 +69,4 @@ global-exclude tme/scoring.py
 global-exclude tme/package.py
 global-exclude tme/transforms.py
 global-exclude tme/tests/test_packaging.py
-global-exclude scripts/match_template_devel.py
+global-exclude scripts/match_template_devel.py

pytme-0.2.2/PKG-INFO

@@ -0,0 +1,91 @@
+Metadata-Version: 2.1
+Name: pytme
+Version: 0.2.2
+Summary: Python Template Matching Engine
+Author: Valentin Maurer
+Author-email: Valentin Maurer <valentin.maurer@embl-hamburg.de>
+License: Proprietary
+Project-URL: Homepage, https://github.com/KosinskiLab/pyTME
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: mrcfile>=1.4.3
+Requires-Dist: numpy>=1.22.2
+Requires-Dist: pyfftw>=0.13.1
+Requires-Dist: pytest>=6.2.5
+Requires-Dist: PyYAML>=6.0
+Requires-Dist: scikit-image>=0.19.0
+Requires-Dist: scikit_learn>=1.2.1
+Requires-Dist: scipy>=1.9.1
+Requires-Dist: pybind11
+Requires-Dist: psutil
+Requires-Dist: tifffile
+Requires-Dist: h5py
+Provides-Extra: cupy
+Requires-Dist: cupy; extra == "cupy"
+Requires-Dist: voltools==0.6.0; extra == "cupy"
+Provides-Extra: pytorch
+Requires-Dist: torch; extra == "pytorch"
+Provides-Extra: jax
+Requires-Dist: jax; extra == "jax"
+Requires-Dist: jaxlib; extra == "jax"
+Provides-Extra: all
+Requires-Dist: cupy; extra == "all"
+Requires-Dist: voltools==0.6.0; extra == "all"
+Requires-Dist: torch; extra == "all"
+Requires-Dist: jax; extra == "all"
+Requires-Dist: jaxlib; extra == "all"
+
+# Python Template Matching Engine (PyTME)
+
+[![Build Status](https://img.shields.io/github/actions/workflow/status/KosinskiLab/pyTME/main.yml?label=CI)](https://github.com/KosinskiLab/pyTME/actions)
+[![PyPI](https://img.shields.io/pypi/v/pytme.svg)](https://pypi.org/project/pytme/)
+
+**[Documentation](https://kosinskilab.github.io/pyTME/)** | **[Installation](https://kosinskilab.github.io/pyTME/quickstart/installation.html)** | **[API](https://kosinskilab.github.io/pyTME/reference/index.html)**
+
+PyTME is a Python library for data-intensive n-dimensional template matching using CPUs and GPUs.
+
+With its [backend-agnostic design](https://kosinskilab.github.io/pyTME/reference/backends.html), the same code can be run on diverse hardware platforms using a best-of-breed approach. The underyling abstract backend specification allows for adding new backends to benefit from gains in performance and capabilities without modifying the library's core routines. The implementation of template matching scores is modular and provides developers with a flexible framework for rapid prototyping. Furthermore, pyTME supports a unique callback capability through [analyzers](https://kosinskilab.github.io/pyTME/reference/analyzer.html), which allows for injection of custom code, enabling real-time processing and manipulation of results
+
+PyTME includes a [graphical user interface](https://kosinskilab.github.io/pyTME/quickstart/preprocessing.html#practical-example) that provides simplified mask creation, interactive filter exploration, result visualization, and manual refinement capabilities. This GUI serves as an accessible entry point to the library's core functionalities, allowing users to efficiently interact with and analyze their data.
+
+Finally, pyTME offers specialized tools for cryogenic electron microscopy data, such as wedge masks, CTF correction, as well as [means for handling structural data](https://kosinskilab.github.io/pyTME/reference/data_structure.html). Through dedicated [integrations](https://kosinskilab.github.io/pyTME/quickstart/integrations.html), the output of pyTME seamlessly integrates with commonly used cryogenic electron microscopy software such as RELION, Dynamo and IMOD.
+
+Running into bugs or missing a feature? Help us improve the project by opening an [issue](https://github.com/KosinskiLab/pyTME/issues).
+
+## Installation
+
+We recommend installation using one of the following methods
+
+| Method   | Command                                                 |
+|----------|---------------------------------------------------------|
+| PyPi     | `pip install pytme`                                     |
+| Source   | `pip install git+https://github.com/KosinskiLab/pyTME`  |
+| Docker   | `docker build -t pytme -f docker/Dockerfile_GPU .`      |
+
+You can find alternative installation methods in the [documentation](https://kosinskilab.github.io/pyTME/quickstart/installation.html).
+
+
+## Quickstart
+
+Learn how to get started with
+
+- [Preprocessing:](https://kosinskilab.github.io/pyTME/quickstart/preprocessing.html) Picking the right mask and filter for template matching.
+- [Template matching:](https://kosinskilab.github.io/pyTME/quickstart/match_template.html) Find your template of interest.
+- [Postprocessing](https://kosinskilab.github.io/pyTME/quickstart/postprocessing.html) Analyze template matching results and downstream integrations.
+
+## How to Cite
+
+If PyTME contributed significantly to your research, please cite the corresponding publication on [SoftwareX](https://www.sciencedirect.com/science/article/pii/S2352711024000074).
+
+```bibtex
+@article{Maurer:2024aa,
+    author = {Maurer, Valentin J. and Siggel, Marc and Kosinski, Jan},
+    journal = {SoftwareX},
+    pages = {101636},
+    title = {PyTME (Python Template Matching Engine): A fast, flexible, and multi-purpose template matching library for cryogenic electron microscopy data},
+    volume = {25},
+    year = {2024}}
+```

pytme-0.2.2/README.md

@@ -0,0 +1,51 @@
+# Python Template Matching Engine (PyTME)
+
+[![Build Status](https://img.shields.io/github/actions/workflow/status/KosinskiLab/pyTME/main.yml?label=CI)](https://github.com/KosinskiLab/pyTME/actions)
+[![PyPI](https://img.shields.io/pypi/v/pytme.svg)](https://pypi.org/project/pytme/)
+
+**[Documentation](https://kosinskilab.github.io/pyTME/)** | **[Installation](https://kosinskilab.github.io/pyTME/quickstart/installation.html)** | **[API](https://kosinskilab.github.io/pyTME/reference/index.html)**
+
+PyTME is a Python library for data-intensive n-dimensional template matching using CPUs and GPUs.
+
+With its [backend-agnostic design](https://kosinskilab.github.io/pyTME/reference/backends.html), the same code can be run on diverse hardware platforms using a best-of-breed approach. The underyling abstract backend specification allows for adding new backends to benefit from gains in performance and capabilities without modifying the library's core routines. The implementation of template matching scores is modular and provides developers with a flexible framework for rapid prototyping. Furthermore, pyTME supports a unique callback capability through [analyzers](https://kosinskilab.github.io/pyTME/reference/analyzer.html), which allows for injection of custom code, enabling real-time processing and manipulation of results
+
+PyTME includes a [graphical user interface](https://kosinskilab.github.io/pyTME/quickstart/preprocessing.html#practical-example) that provides simplified mask creation, interactive filter exploration, result visualization, and manual refinement capabilities. This GUI serves as an accessible entry point to the library's core functionalities, allowing users to efficiently interact with and analyze their data.
+
+Finally, pyTME offers specialized tools for cryogenic electron microscopy data, such as wedge masks, CTF correction, as well as [means for handling structural data](https://kosinskilab.github.io/pyTME/reference/data_structure.html). Through dedicated [integrations](https://kosinskilab.github.io/pyTME/quickstart/integrations.html), the output of pyTME seamlessly integrates with commonly used cryogenic electron microscopy software such as RELION, Dynamo and IMOD.
+
+Running into bugs or missing a feature? Help us improve the project by opening an [issue](https://github.com/KosinskiLab/pyTME/issues).
+
+## Installation
+
+We recommend installation using one of the following methods
+
+| Method   | Command                                                 |
+|----------|---------------------------------------------------------|
+| PyPi     | `pip install pytme`                                     |
+| Source   | `pip install git+https://github.com/KosinskiLab/pyTME`  |
+| Docker   | `docker build -t pytme -f docker/Dockerfile_GPU .`      |
+
+You can find alternative installation methods in the [documentation](https://kosinskilab.github.io/pyTME/quickstart/installation.html).
+
+
+## Quickstart
+
+Learn how to get started with
+
+- [Preprocessing:](https://kosinskilab.github.io/pyTME/quickstart/preprocessing.html) Picking the right mask and filter for template matching.
+- [Template matching:](https://kosinskilab.github.io/pyTME/quickstart/match_template.html) Find your template of interest.
+- [Postprocessing](https://kosinskilab.github.io/pyTME/quickstart/postprocessing.html) Analyze template matching results and downstream integrations.
+
+## How to Cite
+
+If PyTME contributed significantly to your research, please cite the corresponding publication on [SoftwareX](https://www.sciencedirect.com/science/article/pii/S2352711024000074).
+
+```bibtex
+@article{Maurer:2024aa,
+    author = {Maurer, Valentin J. and Siggel, Marc and Kosinski, Jan},
+    journal = {SoftwareX},
+    pages = {101636},
+    title = {PyTME (Python Template Matching Engine): A fast, flexible, and multi-purpose template matching library for cryogenic electron microscopy data},
+    volume = {25},
+    year = {2024}}
+```

{pytme-0.2.0b0 → pytme-0.2.2}/pyproject.toml

@@ -7,7 +7,7 @@ name="pytme"
 authors = [
   { name = "Valentin Maurer", email = "valentin.maurer@embl-hamburg.de" },
 ]
-version="0.2.0b"
+version="0.2.2"
 description="Python Template Matching Engine"
 readme="README.md"
 requires-python = ">=3.11"
@@ -16,15 +16,14 @@ dependencies=[
     "numpy>=1.22.2",
     "pyfftw>=0.13.1",
     "pytest>=6.2.5",
-    "PyWavelets>=1.2.0",
     "PyYAML>=6.0",
     "scikit-image>=0.19.0",
     "scikit_learn>=1.2.1",
     "scipy>=1.9.1",
     "pybind11",
     "psutil",
-    "tifffile[all]",
-    "h5py"
+    "tifffile",
+    "h5py",
 ]
 license = {text = "Proprietary"}
 classifiers = [
@@ -32,6 +31,18 @@ classifiers = [
     "Operating System :: OS Independent",
 ]
 
+[project.optional-dependencies]
+cupy = ["cupy", "voltools==0.6.0"]
+pytorch = ["torch"]
+jax = ["jax", "jaxlib"]
+all = [
+    "cupy",
+    "voltools==0.6.0",
+    "torch",
+    "jax",
+    "jaxlib"
+]
+
 [project.scripts]
 match_template = "scripts:match_template.main"
 estimate_ram_usage = "scripts:estimate_ram_usage.main"

{pytme-0.2.0b0 → pytme-0.2.2}/pytme.egg-info/SOURCES.txt

@@ -17,20 +17,22 @@ tme/__init__.py
 tme/__version__.py
 tme/analyzer.py
 tme/density.py
-tme/helpers.py
-tme/matching_constrained.py
 tme/matching_data.py
 tme/matching_exhaustive.py
 tme/matching_memory.py
 tme/matching_optimization.py
+tme/matching_scores.py
 tme/matching_utils.py
+tme/memory.py
 tme/orientations.py
 tme/parser.py
 tme/preprocessor.py
 tme/structure.py
 tme/types.py
 tme/backends/__init__.py
+tme/backends/_jax_utils.py
 tme/backends/cupy_backend.py
+tme/backends/jax_backend.py
 tme/backends/matching_backend.py
 tme/backends/mlx_backend.py
 tme/backends/npfftw_backend.py
@@ -57,4 +59,10 @@ tme/data/c48u8649.npy
 tme/data/c600v.npy
 tme/data/c600vc.npy
 tme/data/metadata.yaml
-tme/data/quat_to_numpy.py
+tme/data/quat_to_numpy.py
+tme/preprocessing/__init__.py
+tme/preprocessing/_utils.py
+tme/preprocessing/composable_filter.py
+tme/preprocessing/compose.py
+tme/preprocessing/frequency_filters.py
+tme/preprocessing/tilt_series.py

pytme-0.2.2/scripts/extract_candidates.py

@@ -0,0 +1,257 @@
+#!python3
+""" Prepare orientations stack for refinement.
+
+    Copyright (c) 2023 European Molecular Biology Laboratory
+
+    Author: Valentin Maurer <valentin.maurer@embl-hamburg.de>
+"""
+import argparse
+from os.path import splitext
+
+import numpy as np
+
+from tme import Density, Orientations
+from tme.matching_utils import (
+    load_pickle,
+    generate_tempfile_name,
+    rotation_aligning_vectors,
+    euler_from_rotationmatrix,
+    euler_to_rotationmatrix,
+)
+
+
+class ProgressBar:
+    """
+    ASCII progress bar.
+    """
+
+    def __init__(self, message : str, nchars : int, total : int):
+        self._size = nchars - len(message)
+        self._message = message
+        self._total = total
+
+    def update(self, cur):
+        x = int(cur * self._size / self._total)
+        print(
+            "%s[%s%s] %i/%i\r"
+            % (self._message, "#" * x, "." * (self._size - x), cur, self._total),
+            end=''
+        )
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description="Extract matching candidates for further refinement."
+    )
+
+    io_group = parser.add_argument_group("Input / Output")
+    io_group.add_argument(
+        "--input_file",
+        required=False,
+        type=str,
+        help="Path to the output of match_template.py.",
+    )
+    io_group.add_argument(
+        "--orientations",
+        required=True,
+        type=str,
+        help="Path to file generated by postprocess.py using output_format orientations.",
+    )
+    io_group.add_argument(
+        "--target",
+        required=False,
+        type=str,
+        help="Extract candidates from this target, can be at different sampling rate.",
+    )
+    io_group.add_argument(
+        "--template",
+        required=False,
+        type=str,
+        help="Extract candidates from this target, can be at different sampling rate.",
+    )
+    io_group.add_argument(
+        "-o",
+        "--output_file",
+        required=True,
+        type=str,
+        help="Path to output HDF5 file.",
+    )
+
+    alignment_group = parser.add_argument_group("Alignment")
+    alignment_group.add_argument(
+        "--align_orientations",
+        action="store_true",
+        required=False,
+        help="Whether to align extracted orientations based on their angles. Allows "
+        "for efficient subsequent sampling of cone angles.",
+    )
+    alignment_group.add_argument(
+        "--angles_are_vector",
+        action="store_true",
+        required=False,
+        help="Considers euler_z euler_y, euler_x as vector that will be rotated to align "
+        "with the z-axis (1,0,0). Only considered when --align_orientations is set."
+    )
+    alignment_group.add_argument(
+        "--interpolation_order",
+        dest="interpolation_order",
+        required=False,
+        type=int,
+        default=1,
+        help="Interpolation order for alignment, less than zero is no interpolation."
+    )
+
+    extraction_group = parser.add_argument_group("Extraction")
+    extraction_group.add_argument(
+        "--box_size",
+        required=False,
+        type=int,
+        help="Box size for extraction, defaults to two times the template.",
+    )
+    extraction_group.add_argument(
+        "--translation_uncertainty",
+        required=False,
+        type=int,
+        help="Sets box size for extraction to template box plus this value.",
+    )
+    extraction_group.add_argument(
+        "--keep_out_of_box",
+        action="store_true",
+        required=False,
+        help="Whether to keep orientations that fall outside the box. If the "
+        "orientations are sensible, it is safe to pass this flag.",
+    )
+
+    args = parser.parse_args()
+
+    data_present = args.target is not None and args.template is not None
+    if args.input_file is None and not data_present:
+        raise ValueError(
+            "Either --input_file or --target and --template need to be specified."
+        )
+    elif args.input_file is not None and data_present:
+        raise ValueError(
+            "Please specific either --input_file or --target and --template."
+        )
+
+    return args
+
+
+def main():
+    args = parse_args()
+    orientations = Orientations.from_file(args.orientations)
+
+    if args.input_file is not None:
+        data = load_pickle(args.input_file)
+        target_origin, _, sampling_rate, cli_args = data[-1]
+        args.target, args.template  = cli_args.target, cli_args.template
+
+    target = Density.from_file(args.target, use_memmap=True)
+
+    try:
+        template = Density.from_file(args.template)
+    except Exception:
+        template = Density.from_structure(args.template, sampling_rate = target.sampling_rate)
+
+    box_size = args.box_size
+    if box_size is None:
+        box_size = np.multiply(template.shape, 2)
+    if args.translation_uncertainty is not None:
+        box_size = np.add(template.shape, args.translation_uncertainty)
+
+    box_size = np.array(box_size)
+    box_size = np.repeat(box_size, template.data.ndim // box_size.size).astype(int)
+
+    extraction_shape = np.copy(box_size)
+    if args.align_orientations:
+        extraction_shape[:] = int(np.linalg.norm(box_size) + 1)
+
+    orientations, cand_slices, obs_slices = orientations.get_extraction_slices(
+        target_shape=target.shape,
+        extraction_shape=extraction_shape,
+        drop_out_of_box=not args.keep_out_of_box,
+        return_orientations=True,
+    )
+
+    if args.align_orientations:
+        orientations.rotations = orientations.rotations.astype(np.float32)
+        for index in range(orientations.rotations.shape[0]):
+            rotation_matrix = euler_to_rotationmatrix(orientations.rotations[index])
+            rotation_matrix = np.linalg.inv(rotation_matrix)
+            if args.angles_are_vector:
+                rotation_matrix = rotation_aligning_vectors(
+                    orientations.rotations[index], target_vector=(1,0,0)
+                )
+            orientations.rotations[index] = euler_from_rotationmatrix(rotation_matrix)
+
+
+    filename = generate_tempfile_name()
+    output_dtype = target.data.dtype
+    if args.align_orientations is not None:
+        output_dtype = np.float32
+
+    target.data = target.data.astype(output_dtype)
+
+    dens = Density(
+        np.memmap(
+            filename,
+            mode="w+",
+            shape=(len(obs_slices), *box_size),
+            dtype=output_dtype,
+        ),
+        sampling_rate=(1, *target.sampling_rate),
+        origin=(0, *target.origin),
+    )
+    dens.data[:] = target.metadata["mean"]
+
+    data_subset = np.zeros(extraction_shape, dtype = target.data.dtype)
+    pbar = ProgressBar(message = "Orientation ", nchars = 80, total = len(obs_slices))
+    for index, (obs_slice, cand_slice) in enumerate(zip(obs_slices, cand_slices)):
+        pbar.update(index + 1)
+
+        data_subset.fill(0)
+        data_subset[cand_slice] = target.data[obs_slice]
+        target_subset = Density(
+            data_subset,
+            sampling_rate=target.sampling_rate,
+            origin=target.origin,
+        )
+
+        if args.align_orientations:
+            rotation_matrix = euler_to_rotationmatrix(orientations.rotations[index])
+            target_subset = target_subset.rigid_transform(
+                rotation_matrix=rotation_matrix,
+                use_geometric_center=True,
+                order=args.interpolation_order,
+            )
+        target_subset.pad(box_size, center=True)
+
+        # target_value = target.data[tuple(orientations.translations[index].astype(int))]
+        # center = np.divide(target_subset.data.shape, 2).astype(int ) + np.mod(target_subset.shape, 2)
+        # print(np.where(target_subset.data == target_value), center)
+        # print(target_subset.data[tuple(center.astype(int))],
+            # target_value,
+            # target_subset.data[tuple(center.astype(int))] == target_value
+        # )
+
+        dens.data[index] = target_subset.data
+    print("")
+
+    target_meta = {
+        k: v for k, v in target.metadata.items() if k in ("mean", "max", "min", "std")
+    }
+    dens.metadata.update(target_meta)
+    dens.metadata["batch_dimension"] = (0, )
+
+    dens.to_file(args.output_file)
+    orientations.to_file(
+        f"{splitext(args.output_file)[0]}_aligned.tsv",
+        file_format="text"
+    )
+    # orientations.to_file(
+    #     f"{splitext(args.output_file)[0]}_aligned.star",
+    #     file_format="relion"
+    # )
+
+if __name__ == "__main__":
+    main()