virtool-workflow 0.0.0__tar.gz

virtool_workflow-0.0.0/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Canadian Food Inspection Agency
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

virtool_workflow-0.0.0/PKG-INFO

@@ -0,0 +1,71 @@
+Metadata-Version: 2.3
+Name: virtool-workflow
+Version: 0.0.0
+Summary: A framework for developing bioinformatics workflows for Virtool.
+License: MIT
+Author: Ian Boyes
+Maintainer: Ian Boyes
+Requires-Python: >=3.12.3,<3.13.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Requires-Dist: aiofiles (>=0.7.0,<0.8.0)
+Requires-Dist: aiohttp (>=3.8.1,<4.0.0)
+Requires-Dist: biopython (>=1.81,<2.0)
+Requires-Dist: click (>=8.1.7,<9.0.0)
+Requires-Dist: orjson (>=3.9.9,<4.0.0)
+Requires-Dist: pydantic-factories (>=1.17.3,<2.0.0)
+Requires-Dist: pyfixtures (>=1.0.0,<2.0.0)
+Requires-Dist: sentry-sdk (>=2.3.1,<3.0.0)
+Requires-Dist: structlog-sentry (>=2.2.1,<3.0.0)
+Requires-Dist: virtool (>=31.1.3,<32.0.0)
+Description-Content-Type: text/markdown
+
+# Virtool Workflow
+
+![Tests](https://github.com/virtool/virtool-workflow/workflows/ci/badge.svg?branch=main)
+[![PyPI version](https://badge.fury.io/py/virtool-workflow.svg)](https://badge.fury.io/py/virtool-workflow)
+
+A framework for developing bioinformatic workflows in Python.
+
+```python
+from virtool_workflow import step
+
+
+@step
+def step_function():
+    ...
+
+
+@step
+def step_function_2():
+    ...
+```
+
+## Contributing
+
+### Commits
+
+We require specific commit formatting. Any commit that does not follow the guidelines
+will be squashed at our discretion.
+
+Read our [commit and release](https://dev.virtool.ca/en/latest/commits_releases.html)
+documentation for more information.
+
+### Tests
+
+Run tests with:
+
+```shell
+# Bring up Redis and the test container.
+docker compose up -d
+
+# Run tests in the test container.
+docker compose exec test poetry run pytest
+```
+
+Run specific tests like:
+
+```shell
+docker compose exec test poetry run pytest tests/test_status.py
+```
+

virtool_workflow-0.0.0/README.md

@@ -0,0 +1,48 @@
+# Virtool Workflow
+
+![Tests](https://github.com/virtool/virtool-workflow/workflows/ci/badge.svg?branch=main)
+[![PyPI version](https://badge.fury.io/py/virtool-workflow.svg)](https://badge.fury.io/py/virtool-workflow)
+
+A framework for developing bioinformatic workflows in Python.
+
+```python
+from virtool_workflow import step
+
+
+@step
+def step_function():
+    ...
+
+
+@step
+def step_function_2():
+    ...
+```
+
+## Contributing
+
+### Commits
+
+We require specific commit formatting. Any commit that does not follow the guidelines
+will be squashed at our discretion.
+
+Read our [commit and release](https://dev.virtool.ca/en/latest/commits_releases.html)
+documentation for more information.
+
+### Tests
+
+Run tests with:
+
+```shell
+# Bring up Redis and the test container.
+docker compose up -d
+
+# Run tests in the test container.
+docker compose exec test poetry run pytest
+```
+
+Run specific tests like:
+
+```shell
+docker compose exec test poetry run pytest tests/test_status.py
+```

virtool_workflow-0.0.0/pyproject.toml

@@ -0,0 +1,87 @@
+[project]
+name = "virtool-workflow"
+description = "A framework for developing bioinformatics workflows for Virtool."
+version = "0.0.0"
+authors = [
+    {name = "Ian Boyes"},
+    {name = "Reece Hoffman"},
+    {name = "Blake Smith"},
+    {name = "Ryan Fang"},
+    {name = "Matt Curtis"},
+    {name = "Aman Monga"},
+    {name = "Bryce Davidson"},
+    {name = "Christine Wong Chong"},
+    {name = "Lilly Roberts"},
+    {name = "Markus Swoveland"},
+    {name = "Ryan Fang"},
+]
+dynamic = ["dependencies"]
+license = "MIT"
+maintainers = [
+    {name = "Ian Boyes"},
+    {name = "Reece Hoffman"},
+]
+readme = "README.md"
+repository = "https://github.com/virtool/virtool-workflow"
+requires-python = ">=3.12.3,<3.13.0"
+
+[project.scripts]
+run-workflow = "virtool_workflow.cli:cli_main"
+
+[tool.poetry]
+packages = [
+    { include = "virtool_workflow" },
+]
+
+[tool.poetry.dependencies]
+aiofiles = "^0.7.0"
+aiohttp = "^3.8.1"
+biopython = "^1.81"
+click = "^8.1.7"
+orjson = "^3.9.9"
+pydantic-factories = "^1.17.3"
+pyfixtures = "^1.0.0"
+sentry-sdk = "^2.3.1"
+structlog-sentry = "^2.2.1"
+virtool = "^31.1.3"
+
+[tool.poetry.group.dev.dependencies]
+piccolo-theme = "^0.24.0"
+pytest = "^7.4.2"
+pytest-asyncio = "^0.21.0"
+pytest-mock = "^3.10.0"
+pytest-structlog = "^1.0"
+ruff = "^0.4.6"
+sphinx = "^4.0.2"
+sphinx-autobuild = "^2021.3.14"
+sphinx-nameko-theme = "^0.0.3"
+sphinx-toolbox = "^3.5.0"
+syrupy = "^3.0.5"
+pytest-aiohttp = "^1.1.0"
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+
+[tool.ruff]
+exclude = [
+    "docs",
+    ".eggs",
+    ".git",
+    ".github",
+    ".mypy_cache",
+    ".pytest-cache",
+    ".ruff_cache",
+    "__pypackages__",
+]
+
+[tool.ruff.lint]
+ignore = [
+    "ANN101",
+    "D203",
+    "D213"
+]
+select = ["ALL"]
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"

virtool_workflow-0.0.0/virtool_workflow/__init__.py

@@ -0,0 +1,13 @@
+"""
+A framework for defining Virtool workflows.
+"""
+from virtool_workflow.decorators import step
+from virtool_workflow.runtime.run_subprocess import RunSubprocess
+from virtool_workflow.workflow import Workflow, WorkflowStep
+
+__all__ = [
+    "step",
+    "RunSubprocess",
+    "Workflow",
+    "WorkflowStep",
+]

virtool_workflow-0.0.0/virtool_workflow/analysis/__init__.py

@@ -0,0 +1 @@
+"""Tools for workflows relating to sequence analysis."""

virtool_workflow-0.0.0/virtool_workflow/analysis/fastqc.py

@@ -0,0 +1,467 @@
+"""Utilities and fixtures for running FastQC."""
+from __future__ import annotations
+
+import asyncio
+import shutil
+import statistics
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+from typing import IO, Protocol, TextIO
+
+from pyfixtures import fixture
+
+from virtool_workflow import RunSubprocess
+from virtool_workflow.analysis.utils import ReadPaths
+
+
+@dataclass
+class NucleotidePoint:
+    g: float
+    a: float
+    t: float
+    c: float
+
+
+@dataclass
+class QualityPoint:
+    mean: float
+    median: float
+    lower_quartile: float
+    upper_quartile: float
+    tenth_percentile: float
+    ninetieth_percentile: float
+
+
+class BaseQualityParser:
+    """Parse the section of FastQC output containing per-base quality data."""
+
+    pattern = ">>Per base sequence quality"
+
+    def __init__(self):
+        self.data: list[QualityPoint] = []
+
+    def composite(self, parser: BaseQualityParser):
+        p = BaseQualityParser()
+
+        p.data = [
+            QualityPoint(
+                mean=statistics.mean([this.mean, other.mean]),
+                median=statistics.mean([this.median, other.median]),
+                lower_quartile=statistics.mean(
+                    [this.lower_quartile, other.lower_quartile],
+                ),
+                upper_quartile=statistics.mean(
+                    [this.upper_quartile, other.upper_quartile],
+                ),
+                tenth_percentile=statistics.mean(
+                    [this.tenth_percentile, other.tenth_percentile],
+                ),
+                ninetieth_percentile=statistics.mean(
+                    [this.ninetieth_percentile, other.ninetieth_percentile],
+                ),
+            )
+            for this, other in zip(self.data, parser.data)
+        ]
+
+        return p
+
+    def handle(self, f: TextIO):
+        max_index = -1
+
+        while True:
+            line = f.readline().rstrip()
+
+            if line == ">>END_MODULE":
+                break
+
+            if not line or line[0] == "#":
+                continue
+
+            split = line.split()
+
+            # Convert all fields except first to 2-decimal floats.
+            try:
+                values = [float(value) for value in split[1:]]
+            except ValueError as err:
+                if "NaN" not in str(err):
+                    raise
+
+                values = _handle_base_quality_nan(split)
+
+            (
+                mean,
+                median,
+                lower_quartile,
+                upper_quartile,
+                tenth_percentile,
+                ninetieth_percentile,
+            ) = values
+
+            indexes = _calculate_index_range(split[0])
+
+            for i in indexes:
+                self.data.append(
+                    QualityPoint(
+                        mean=mean,
+                        median=median,
+                        lower_quartile=lower_quartile,
+                        upper_quartile=upper_quartile,
+                        tenth_percentile=tenth_percentile,
+                        ninetieth_percentile=ninetieth_percentile,
+                    ),
+                )
+
+                if i - max_index != 1:
+                    raise ValueError("Non-contiguous index")
+
+                max_index = i
+
+
+class BasicStatisticsParser:
+    """Parse the section of FastQC output containing basic statistics."""
+
+    pattern = ">>Basic Statistics"
+
+    def __init__(self):
+        self.count = 0
+        self.encoding = None
+        self.gc = None
+        self.length = None
+
+        self._populated = False
+
+    def composite(self, parser: BasicStatisticsParser):
+        p = BasicStatisticsParser()
+
+        p.count = self.count + parser.count
+        p.encoding = self.encoding
+        p.gc = (self.gc + parser.gc) / 2
+        p.length = [
+            min(self.length + parser.length),
+            max(self.length + parser.length),
+        ]
+
+        return p
+
+    def handle(self, f: IO):
+        while True:
+            line = f.readline().rstrip()
+
+            if line.startswith("#"):
+                continue
+
+            if line == ">>END_MODULE":
+                break
+
+            if "Total Sequences" in line:
+                self.count = int(line.split("\t")[1])
+
+            elif "Encoding" in line:
+                self.encoding = line.split("\t")[1]
+
+            elif "Sequence length" in line:
+                length_range = [int(s) for s in line.split("\t")[1].split("-")]
+                self.length = [min(length_range), max(length_range)]
+
+            # GC-content
+            elif "%GC" in line and "#" not in line:
+                self.gc = float(line.split("\t")[1])
+
+
+class NucleotideCompositionParser:
+    """Parse the section of FastQC output containing per-base nucleotide composition."""
+
+    pattern = ">>Per base sequence content"
+
+    def __init__(self):
+        self.data: list[NucleotidePoint] = []
+
+    def composite(self, parser: NucleotideCompositionParser):
+        """Make a composite dataset given another :class:`.NucleotideCompositionParser`."""
+        p = NucleotideCompositionParser()
+
+        p.data = [
+            NucleotidePoint(
+                g=(this.g + other.g) / 2,
+                a=(this.a + other.a) / 2,
+                t=(this.t + other.t) / 2,
+                c=(this.c + other.c) / 2,
+            )
+            for this, other in zip(self.data, parser.data)
+        ]
+
+        return p
+
+    def handle(self, f: TextIO):
+        max_index = -1
+
+        while True:
+            line = f.readline().rstrip()
+
+            if line == ">>END_MODULE":
+                break
+
+            if not line or line[0] == "#":
+                continue
+
+            split = line.split()
+
+            try:
+                g, a, t, c = (float(value) for value in split[1:])
+            except ValueError as err:
+                if "NaN" not in str(err):
+                    raise
+
+                g, a, t, c = _handle_base_quality_nan(split)
+
+            indexes = _calculate_index_range(split[0])
+
+            for i in indexes:
+                self.data.append(NucleotidePoint(g, a, t, c))
+
+                if i - max_index != 1:
+                    raise ValueError("Non-contiguous index")
+
+                max_index = i
+
+
+class SequenceQualityParser:
+    """Parse the section of FastQC output containing per-sequence quality data."""
+
+    pattern = ">>Per sequence quality scores"
+
+    def __init__(self):
+        self.data = [0] * 50
+
+    def composite(self, parser: SequenceQualityParser):
+        p = SequenceQualityParser()
+        p.data = [sum(both) for both in zip(self.data, parser.data)]
+
+        return p
+
+    def handle(self, f: TextIO):
+        while True:
+            line = f.readline().rstrip()
+
+            if not line or line.startswith("#"):
+                continue
+
+            if line == ">>END_MODULE":
+                break
+
+            line = line.split()
+
+            quality = int(line[0])
+            count = int(float(line[1]))
+
+            self.data[quality] = count
+
+
+@dataclass
+class FastQCSide:
+    base_quality: BaseQualityParser
+    basic_statistics: BasicStatisticsParser
+    nucleotide_composition: NucleotideCompositionParser
+    sequence_quality: SequenceQualityParser
+
+
+def _calculate_index_range(base: str) -> range:
+    pos = [int(x) for x in base.split("-")]
+
+    if len(pos) > 1:
+        return range(pos[0] - 1, pos[1])
+
+    return range(pos[0] - 1, pos[0])
+
+
+def _handle_base_quality_nan(split_line: list) -> list:
+    """Parse a per-base quality line from FastQC containing NaN values.
+
+    :param split_line: the quality line split into a :class:`.List`
+    :return: replacement values
+
+    """
+    values = split_line[1:]
+
+    for value in values:
+        try:
+            return [value for _ in values]
+        except ValueError:
+            pass
+
+    # Return all zeroes if none of the quality values are numbers.
+    if set(values) == {"NaN"}:
+        return [0] * 4
+
+    joined = ",".join(split_line)
+
+    raise ValueError(f"Could not parse base quality values '{joined}'")
+
+
+def _parse_fastqc(fastqc_path: Path, output_path: Path) -> dict:
+    """Parse the FastQC results at `fastqc_path`.
+
+    All FastQC data except the textual data file are removed.
+
+    :param fastqc_path: the FastQC output data path
+    :param sample_path: the FastQC text output file will be moved here
+    :return: a dict containing a representation of the parsed FastQC data
+
+    """
+    output_path.mkdir(exist_ok=True, parents=True)
+
+    sides = []
+
+    # Get the text data files from the FastQC output
+    for path in fastqc_path.iterdir():
+        if not path.is_dir():
+            continue
+
+        for file_path in path.iterdir():
+            if file_path.name != "fastqc_data.txt":
+                continue
+
+            new_path = output_path / f"{path.name}.txt"
+
+            shutil.move(file_path, new_path)
+
+            base_quality = BaseQualityParser()
+            basic_statistics = BasicStatisticsParser()
+            nucleotide_composition = NucleotideCompositionParser()
+            sequence_quality = SequenceQualityParser()
+
+            with open(new_path) as f:
+                while True:
+                    line = f.readline()
+
+                    if not line:
+                        break
+
+                    if basic_statistics.pattern in line:
+                        basic_statistics.handle(f)
+
+                    if base_quality.pattern in line:
+                        base_quality.handle(f)
+
+                    if nucleotide_composition.pattern in line:
+                        nucleotide_composition.handle(f)
+
+                    if SequenceQualityParser.pattern in line:
+                        sequence_quality.handle(f)
+
+            sides.append(
+                FastQCSide(
+                    base_quality=base_quality,
+                    basic_statistics=basic_statistics,
+                    nucleotide_composition=nucleotide_composition,
+                    sequence_quality=sequence_quality,
+                ),
+            )
+
+    if len(sides) == 1:
+        left = sides[0]
+
+        return {
+            "bases": [
+                [
+                    round(n, 3)
+                    for n in [
+                        point.mean,
+                        point.median,
+                        point.lower_quartile,
+                        point.upper_quartile,
+                        point.tenth_percentile,
+                        point.ninetieth_percentile,
+                    ]
+                ]
+                for point in left.base_quality.data
+            ],
+            "composition": [
+                [round(n, 1) for n in [point.g, point.a, point.t, point.c]]
+                for point in left.nucleotide_composition.data
+            ],
+            "count": left.basic_statistics.count,
+            "encoding": left.basic_statistics.encoding,
+            "gc": left.basic_statistics.gc,
+            "length": left.basic_statistics.length,
+            "sequences": left.sequence_quality.data,
+        }
+
+    left, right = sides
+
+    basic = left.basic_statistics.composite(right.basic_statistics)
+
+    return {
+        "bases": [
+            [
+                round(n, 3)
+                for n in [
+                    point.mean,
+                    point.median,
+                    point.lower_quartile,
+                    point.upper_quartile,
+                    point.tenth_percentile,
+                    point.ninetieth_percentile,
+                ]
+            ]
+            for point in left.base_quality.composite(right.base_quality).data
+        ],
+        "composition": [
+            [round(n, 1) for n in [point.g, point.a, point.t, point.c]]
+            for point in left.nucleotide_composition.composite(
+                right.nucleotide_composition,
+            ).data
+        ],
+        "count": basic.count,
+        "length": basic.length,
+        "encoding": left.basic_statistics.encoding,
+        "gc": basic.gc,
+        "sequences": left.sequence_quality.composite(right.sequence_quality).data,
+    }
+
+
+class FastQCRunner(Protocol):
+    """A protocol describing callables that can be used to run FastQC."""
+
+    async def __call__(self, paths: ReadPaths, output_path: Path) -> dict:
+        ...
+
+
+@fixture
+async def fastqc(run_subprocess: RunSubprocess):
+    """Provides an asynchronous function that can run FastQC as a subprocess.
+
+    The function takes a one or two paths to FASTQ read files (:class:`.ReadPaths`) in
+    a tuple.
+
+    Example:
+    -------
+    .. code-block:: python
+
+        @step
+        async def step_one(fastqc: FastQCRunner, work_path: Path):
+            fastqc_result = await fastqc((
+                work_path / "reads_1.fq",
+                work_path / "reads_2.fq"
+            ))
+
+    """
+    temp_path = Path(await asyncio.to_thread(tempfile.mkdtemp))
+
+    async def func(paths: ReadPaths, output_path: Path) -> dict:
+        command = [
+            "fastqc",
+            "-f",
+            "fastq",
+            "-o",
+            str(temp_path),
+            "--extract",
+            *[str(path) for path in paths],
+        ]
+
+        await run_subprocess(command)
+
+        return _parse_fastqc(temp_path, output_path)
+
+    return func