virtool-workflow 6.0.0a11__tar.gz → 7.1.0__tar.gz

virtool_workflow-7.1.0/PKG-INFO

@@ -0,0 +1,74 @@
+Metadata-Version: 2.1
+Name: virtool-workflow
+Version: 7.1.0
+Summary: A framework for developing bioinformatics workflows for Virtool.
+Home-page: https://github.com/virtool/virtool-workflow
+License: MIT
+Author: Ian Boyes
+Maintainer: Ian Boyes
+Requires-Python: >=3.12,<3.13
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: aiofiles (>=0.7.0,<0.8.0)
+Requires-Dist: aiohttp (>=3.8.1,<4.0.0)
+Requires-Dist: biopython (>=1.81,<2.0)
+Requires-Dist: click (>=8.1.7,<9.0.0)
+Requires-Dist: orjson (>=3.9.9,<4.0.0)
+Requires-Dist: pydantic-factories (>=1.17.3,<2.0.0)
+Requires-Dist: pyfixtures (>=1.0.0,<2.0.0)
+Requires-Dist: sentry-sdk (>=2.3.1,<3.0.0)
+Requires-Dist: virtool-core (>=14.0.0,<15.0.0)
+Project-URL: Repository, https://github.com/virtool/virtool-workflow
+Description-Content-Type: text/markdown
+
+# Virtool Workflow
+
+![Tests](https://github.com/virtool/virtool-workflow/workflows/ci/badge.svg?branch=main)
+[![PyPI version](https://badge.fury.io/py/virtool-workflow.svg)](https://badge.fury.io/py/virtool-workflow)
+
+A framework for developing bioinformatic workflows in Python.
+
+```python
+from virtool_workflow import step
+
+
+@step
+def step_function():
+    ...
+
+
+@step
+def step_function_2():
+    ...
+```
+
+## Contributing
+
+### Commits
+
+We require specific commit formatting. Any commit that does not follow the guidelines
+will be squashed at our discretion.
+
+Read our [commit and release](https://dev.virtool.ca/en/latest/commits_releases.html)
+documentation for more information.
+
+### Tests
+
+Run tests with:
+
+```shell
+# Bring up Redis and the test container.
+docker compose up -d
+
+# Run tests in the test container.
+docker compose exec test poetry run pytest
+
+```
+
+Run specific tests like:
+
+```shell
+docker compose exec test poetry run pytest tests/test_status.py
+```
+

virtool_workflow-7.1.0/README.md

@@ -0,0 +1,49 @@
+# Virtool Workflow
+
+![Tests](https://github.com/virtool/virtool-workflow/workflows/ci/badge.svg?branch=main)
+[![PyPI version](https://badge.fury.io/py/virtool-workflow.svg)](https://badge.fury.io/py/virtool-workflow)
+
+A framework for developing bioinformatic workflows in Python.
+
+```python
+from virtool_workflow import step
+
+
+@step
+def step_function():
+    ...
+
+
+@step
+def step_function_2():
+    ...
+```
+
+## Contributing
+
+### Commits
+
+We require specific commit formatting. Any commit that does not follow the guidelines
+will be squashed at our discretion.
+
+Read our [commit and release](https://dev.virtool.ca/en/latest/commits_releases.html)
+documentation for more information.
+
+### Tests
+
+Run tests with:
+
+```shell
+# Bring up Redis and the test container.
+docker compose up -d
+
+# Run tests in the test container.
+docker compose exec test poetry run pytest
+
+```
+
+Run specific tests like:
+
+```shell
+docker compose exec test poetry run pytest tests/test_status.py
+```

{virtool_workflow-6.0.0a11 → virtool_workflow-7.1.0}/pyproject.toml

@@ -1,32 +1,44 @@
 [tool.poetry]
 name = "virtool-workflow"
-version = "6.0.0-alpha.11"
+version = "7.1.0"
 description = "A framework for developing bioinformatics workflows for Virtool."
-authors = ["Ian Boyes", "Blake Smith", "Ryan Fang"]
+authors = [
+    "Ian Boyes",
+    "Blake Smith",
+    "Ryan Fang",
+    "Matt Curtis",
+    "Aman Monga",
+    "Bryce Davidson",
+    "Christine Wong Chong",
+    "Lilly Roberts",
+    "Markus Swoveland",
+    "Ryan Fang",
+]
 license = "MIT"
+maintainers = [
+    "Ian Boyes",
+    "Reece Hoffman"
+]
 readme = "README.md"
 repository = "https://github.com/virtool/virtool-workflow"
 classifiers = [
-    "Topic :: Software Development :: Libraries",
-    "Programming Language :: Python :: 3.10"
+    "Programming Language :: Python :: 3.12",
 ]
 packages = [
     { include = "virtool_workflow" },
 ]
 
 [tool.poetry.dependencies]
-python = "~3.10"
-click = "^8.1.7"
-aiohttp = "^3.8.1"
+python = "~3.12"
 aiofiles = "^0.7.0"
-virtool-core = "^11.0.0"
-aioredis = "1.3.1"
-sentry-sdk = "^1.5.7"
-pyfixtures = "^1.0.0"
-orjson = "^3.9.9"
+aiohttp = "^3.8.1"
 biopython = "^1.81"
+click = "^8.1.7"
+orjson = "^3.9.9"
 pydantic-factories = "^1.17.3"
-
+pyfixtures = "^1.0.0"
+sentry-sdk = "^2.3.1"
+virtool-core = "^14.0.0"
 
 [tool.poetry.scripts]
 run-workflow = "virtool_workflow.cli:cli_main"
@@ -36,13 +48,14 @@ pytest = "^7.4.2"
 pytest-aiohttp = "^1.0.0"
 pytest-asyncio = "^0.21.0"
 pytest-mock = "^3.10.0"
-pre-commit = "^2.18.1"
 sphinx = "^4.0.2"
 syrupy = "^3.0.5"
 sphinx-autobuild = "^2021.3.14"
 sphinx-nameko-theme = "^0.0.3"
 pytest-structlog = "^0.6"
 sphinx-toolbox = "^3.5.0"
+ruff = "^0.4.6"
+piccolo-theme = "^0.22.0"
 
 [tool.pytest.ini_options]
 asyncio_mode = "auto"
@@ -58,8 +71,14 @@ exclude = [
     ".ruff_cache",
     "__pypackages__",
 ]
+target-version = "py312"
 
 [tool.ruff.lint]
+ignore = [
+    "ANN101",
+    "D203",
+    "D213"
+]
 select = ["ALL"]
 
 [build-system]

{virtool_workflow-6.0.0a11 → virtool_workflow-7.1.0}/virtool_workflow/api/acquire.py

@@ -4,7 +4,11 @@ from aiohttp import ClientConnectionError, ClientSession, TCPConnector
 from structlog import get_logger
 from virtool_core.models.job import JobAcquired
 
-from virtool_workflow.errors import JobAlreadyAcquired, JobsAPIError, JobsAPIServerError
+from virtool_workflow.errors import (
+    JobAlreadyAcquiredError,
+    JobsAPIError,
+    JobsAPIServerError,
+)
 
 logger = get_logger("api")
 
@@ -39,7 +43,7 @@ async def acquire_job_by_id(
 
                     if resp.status == 400:
                         if "already acquired" in await resp.text():
-                            raise JobAlreadyAcquired(await resp.json())
+                            raise JobAlreadyAcquiredError(await resp.json())
 
                     logger.critical(
                         "unexpected api error during job acquisition",

{virtool_workflow-6.0.0a11 → virtool_workflow-7.1.0}/virtool_workflow/api/utils.py

@@ -10,10 +10,10 @@ from aiohttp import (
 from structlog import get_logger
 
 from virtool_workflow.errors import (
-    JobsAPIBadRequest,
-    JobsAPIConflict,
-    JobsAPIForbidden,
-    JobsAPINotFound,
+    JobsAPIBadRequestError,
+    JobsAPIConflictError,
+    JobsAPIForbiddenError,
+    JobsAPINotFoundError,
     JobsAPIServerError,
 )
 
@@ -80,10 +80,10 @@ async def raise_exception_by_status_code(resp: ClientResponse):
     :raise JobsAPIServerError: the response status code is 500
     """
     status_exception_map = {
-        400: JobsAPIBadRequest,
-        403: JobsAPIForbidden,
-        404: JobsAPINotFound,
-        409: JobsAPIConflict,
+        400: JobsAPIBadRequestError,
+        403: JobsAPIForbiddenError,
+        404: JobsAPINotFoundError,
+        409: JobsAPIConflictError,
         500: JobsAPIServerError,
     }
 

{virtool_workflow-6.0.0a11 → virtool_workflow-7.1.0}/virtool_workflow/data/indexes.py

@@ -13,7 +13,7 @@ from virtool_core.models.reference import ReferenceNested
 from virtool_core.utils import decompress_file
 
 from virtool_workflow.api.client import APIClient
-from virtool_workflow.errors import MissingJobArgument
+from virtool_workflow.errors import MissingJobArgumentError
 from virtool_workflow.files import VirtoolFileFormat
 
 logger = get_logger("api")
@@ -43,9 +43,7 @@ class WFIndex:
 
     @property
     def bowtie_path(self) -> Path:
-        """The path to the Bowtie2 index prefix for the Virtool index.
-
-        """
+        """The path to the Bowtie2 index prefix for the Virtool index."""
         return self.path / "reference"
 
     @property
@@ -151,7 +149,10 @@ class WFNewIndex:
         await self._api.patch_json(f"/indexes/{self.id}", {})
 
     async def upload(
-        self, path: Path, fmt: VirtoolFileFormat = "fasta", name: str | None = None,
+        self,
+        path: Path,
+        fmt: VirtoolFileFormat = "fasta",
+        name: str | None = None,
     ):
         """Upload a file to associate with the index being built.
 
@@ -272,14 +273,16 @@ async def index(
 
 @fixture
 async def new_index(
-    _api: APIClient, job: Job, proc: int, work_path: Path,
+    _api: APIClient,
+    job: Job,
+    proc: int,
+    work_path: Path,
 ) -> WFNewIndex:
-    """The :class:`.WFNewIndex` for an index being created by the current job.
-    """
+    """The :class:`.WFNewIndex` for an index being created by the current job."""
     try:
         id_ = job.args["index_id"]
     except KeyError:
-        raise MissingJobArgument("Missing jobs args key 'index_id'")
+        raise MissingJobArgumentError("Missing jobs args key 'index_id'")
 
     log = logger.bind(resource="new_index", id=id_, job_id=job.id)
     log.info("loading index")

{virtool_workflow-6.0.0a11 → virtool_workflow-7.1.0}/virtool_workflow/data/samples.py

@@ -12,7 +12,7 @@ from virtool_core.models.samples import Quality, Sample
 from virtool_workflow.analysis.utils import ReadPaths
 from virtool_workflow.api.client import APIClient
 from virtool_workflow.data.uploads import WFUploads
-from virtool_workflow.errors import JobsAPINotFound
+from virtool_workflow.errors import JobsAPINotFoundError
 from virtool_workflow.files import VirtoolFileFormat
 
 logger = get_logger("api")
@@ -83,7 +83,10 @@ class WFNewSample:
 
 @fixture
 async def sample(
-    _api: APIClient, job: Job, uploads: WFUploads, work_path: Path,
+    _api: APIClient,
+    job: Job,
+    uploads: WFUploads,
+    work_path: Path,
 ) -> WFSample:
     """The sample associated with the current job."""
     id_ = job.args["sample_id"]
@@ -92,8 +95,8 @@ async def sample(
 
     try:
         sample_json = await _api.get_json(base_url_path)
-    except JobsAPINotFound:
-        raise JobsAPINotFound("Sample not found")
+    except JobsAPINotFoundError:
+        raise JobsAPINotFoundError("Sample not found")
 
     sample = Sample(**sample_json)
 
@@ -101,7 +104,8 @@ async def sample(
     await asyncio.to_thread(reads_path.mkdir, exist_ok=True, parents=True)
 
     await _api.get_file(
-        f"{base_url_path}/reads/reads_1.fq.gz", reads_path / "reads_1.fq.gz",
+        f"{base_url_path}/reads/reads_1.fq.gz",
+        reads_path / "reads_1.fq.gz",
     )
 
     if sample.paired:
@@ -110,7 +114,8 @@ async def sample(
             reads_path / "reads_2.fq.gz",
         )
         await _api.get_file(
-            f"{base_url_path}/reads/reads_2.fq.gz", reads_path / "reads_2.fq.gz",
+            f"{base_url_path}/reads/reads_2.fq.gz",
+            reads_path / "reads_2.fq.gz",
         )
     else:
         read_paths = (reads_path / "reads_1.fq.gz",)
@@ -127,7 +132,10 @@ async def sample(
 
 @fixture
 async def new_sample(
-    _api: APIClient, job: Job, uploads: WFUploads, work_path: Path,
+    _api: APIClient,
+    job: Job,
+    uploads: WFUploads,
+    work_path: Path,
 ) -> WFNewSample:
     """The sample associated with the current job."""
     id_ = job.args["sample_id"]

{virtool_workflow-6.0.0a11 → virtool_workflow-7.1.0}/virtool_workflow/data/subtractions.py

@@ -15,7 +15,7 @@ from virtool_core.models.subtraction import (
 from virtool_workflow.api.client import APIClient
 from virtool_workflow.data.analyses import WFAnalysis
 from virtool_workflow.data.uploads import WFUploads
-from virtool_workflow.errors import MissingJobArgument
+from virtool_workflow.errors import MissingJobArgumentError
 
 logger = get_logger("api")
 
@@ -118,7 +118,9 @@ class WFNewSubtraction:
 
 @fixture
 async def subtractions(
-    _api: APIClient, analysis: WFAnalysis, work_path: Path,
+    _api: APIClient,
+    analysis: WFAnalysis,
+    work_path: Path,
 ) -> list[WFSubtraction]:
     """The subtractions to be used for the current analysis job."""
     subtraction_work_path = work_path / "subtractions"
@@ -158,7 +160,10 @@ async def subtractions(
 
 @fixture
 async def new_subtraction(
-    _api: APIClient, job: Job, uploads: WFUploads, work_path: Path,
+    _api: APIClient,
+    job: Job,
+    uploads: WFUploads,
+    work_path: Path,
 ) -> WFNewSubtraction:
     """A new subtraction that will be created during the current job.
 
@@ -167,12 +172,12 @@ async def new_subtraction(
     try:
         id_ = job.args["subtraction_id"]
     except KeyError:
-        raise MissingJobArgument("subtraction_id")
+        raise MissingJobArgumentError("subtraction_id")
 
     try:
         upload_id = job.args["files"][0]["id"]
     except KeyError:
-        raise MissingJobArgument("files")
+        raise MissingJobArgumentError("files")
 
     subtraction_json = await _api.get_json(f"/subtractions/{id_}")
     subtraction_ = Subtraction(**subtraction_json)
@@ -221,7 +226,9 @@ async def new_subtraction(
         log.info("Uploading subtraction file")
 
         await _api.put_file(
-            f"/subtractions/{subtraction_.id}/files/{filename}", path, "unknown",
+            f"/subtractions/{subtraction_.id}/files/{filename}",
+            path,
+            "unknown",
         )
 
         log.info("Finished uploading subtraction file")

{virtool_workflow-6.0.0a11 → virtool_workflow-7.1.0}/virtool_workflow/decorators.py

@@ -1,11 +1,13 @@
 """Create Workflows by decorating module scope functions."""
+
+from collections.abc import Callable
 from types import ModuleType
-from typing import Callable
 
+from virtool_workflow.errors import WorkflowStepsError
 from virtool_workflow.workflow import Workflow
 
 
-def step(func: Callable = None, *, name: str | None = None) -> Callable:
+def step(func: Callable | None = None, *, name: str | None = None) -> Callable:
     """Mark a function as a workflow step function.
 
     :param func: the workflow step function
@@ -16,7 +18,7 @@ def step(func: Callable = None, *, name: str | None = None) -> Callable:
         return lambda _f: step(_f, name=name)
 
     func.__workflow_marker__ = "step"
-    func.__workflow_step_props__ = dict(name=name)
+    func.__workflow_step_props__ = {"name": name}
 
     return func
 
@@ -39,7 +41,7 @@ def collect(module: ModuleType) -> Workflow:
         if marked.__workflow_marker__ == "step":
             workflow.step(marked, **marked.__workflow_step_props__)
 
-    if len(workflow.steps) == 0:
-        raise ValueError(f"No workflow steps could be found in {module}")
+    if not workflow.steps:
+        raise WorkflowStepsError(str(module))
 
     return workflow

virtool_workflow-7.1.0/virtool_workflow/errors.py

@@ -0,0 +1,62 @@
+"""Custom exceptions for ``virtool_workflow``."""
+
+from subprocess import SubprocessError
+
+
+class JobAlreadyAcquiredError(Exception):
+    """Raised when an attempt is made to reacquire a job."""
+
+    def __init__(self, job_id: str) -> None:
+        """Initialize the exception with a message containing the job ID."""
+        super().__init__(
+            f"Job {job_id} is has already been acquired.",
+        )
+
+
+class JobsAPIError(Exception):
+    """A base exception for errors due to HTTP errors from the jobs API."""
+
+
+class JobsAPIBadRequestError(JobsAPIError):
+    """A ``400 Bad Request`` response was received from the jobs API."""
+
+    status = 400
+
+
+class JobsAPIForbiddenError(JobsAPIError):
+    """A ``403 Forbidden`` response was received from the jobs API."""
+
+    status = 403
+
+
+class JobsAPINotFoundError(JobsAPIError):
+    """A ``404 Not Found`` response was received from the jobs API."""
+
+    status = 404
+
+
+class JobsAPIConflictError(JobsAPIError):
+    """A ``409 Conflict`` response was received from the jobs API."""
+
+    status = 409
+
+
+class JobsAPIServerError(JobsAPIError):
+    """A ``500 Internal Server Error`` response was received from the jobs API."""
+
+    status = 500
+
+
+class MissingJobArgumentError(ValueError):
+    """The `job.args` dict is missing a required key for some funcionality."""
+
+
+class WorkflowStepsError(Exception):
+    """Raised when no workflow steps are found in a module."""
+
+    def __init__(self, module: str) -> None:
+        super().__init__(f"No workflow steps could be found in {module}")
+
+
+class SubprocessFailedError(SubprocessError):
+    """Subprocess exited with non-zero status during a workflow."""

virtool_workflow-7.1.0/virtool_workflow/files.py

@@ -0,0 +1,40 @@
+"""Dataclasses for describing files uploaded to the Virtool server."""
+
+import datetime
+from dataclasses import dataclass
+from typing import Literal
+
+VirtoolFileFormat = Literal[
+    "sam",
+    "bam",
+    "fasta",
+    "fastq",
+    "csv",
+    "tsv",
+    "json",
+    "unknown",
+]
+"""A literal type hint for the format of a :class:`.VirtoolFile`."""
+
+
+@dataclass
+class VirtoolFile:
+    """A description of a file  uploaded to the Virtool server."""
+
+    id: int
+    """The unique ID for the file."""
+
+    name: str
+    """The name of the file."""
+
+    size: int
+    """The size of the file in bytes."""
+
+    format: VirtoolFileFormat
+    """The format of the file."""
+
+    name_on_disk: str | None = None
+    """The actual name of the file on disk."""
+
+    uploaded_at: datetime.datetime | None = None
+    """When the file was uploaded."""

{virtool_workflow-6.0.0a11 → virtool_workflow-7.1.0}/virtool_workflow/hooks.py

@@ -1,5 +1,4 @@
-"""Hooks provide a way to do things when events happen during the workflow lifecycle.
-"""
+"""Hooks do things when events happen during the workflow lifecycle."""
 
 from virtool_workflow.runtime.hook import Hook
 
@@ -124,7 +123,7 @@ __all__ = [
 ]
 
 
-def cleanup_builtin_status_hooks():
+def cleanup_builtin_status_hooks() -> None:
     """Clear callbacks for built-in status hooks.
 
     This prevents carryover of hooks between tests. Carryover won't be encountered in

{virtool_workflow-6.0.0a11 → virtool_workflow-7.1.0}/virtool_workflow/pytest_plugin/data.py

@@ -116,7 +116,6 @@ def data(
         "test": True,
     }
     job.ping = JobPing(pinged_at=static_datetime)
-    job.rights = {}
 
     """A finalized sample to be used for testing analyses."""
     sample = SampleFactory.build()

{virtool_workflow-6.0.0a11 → virtool_workflow-7.1.0}/virtool_workflow/runtime/redis.py

@@ -2,8 +2,8 @@ import asyncio
 from asyncio import CancelledError
 from typing import Callable
 
-from aioredis import Redis
 from structlog import get_logger
+from virtool_core.redis import Redis
 
 logger = get_logger("redis")
 
@@ -11,10 +11,11 @@ CANCELLATION_CHANNEL = "channel:cancel"
 
 
 async def get_next_job_with_timeout(
-    list_name: str, redis: Redis, timeout: int = None
+    list_name: str,
+    redis: Redis,
+    timeout: int | None = None,
 ) -> str:
-    """
-    Get the next job ID from a Redis list and raise a  :class:``Timeout`` error if one
+    """Get the next job ID from a Redis list and raise a  :class:``Timeout`` error if one
     is not found in ``timeout`` seconds.
 
     :param list_name: the name of the list to pop from
@@ -24,35 +25,39 @@ async def get_next_job_with_timeout(
 
     """
     logger.info(
-        "Waiting for a job", timeout=f"{timeout if timeout else 'infinity'} seconds"
+        "Waiting for a job",
+        timeout=f"{timeout if timeout else 'infinity'} seconds",
     )
 
     return await asyncio.wait_for(get_next_job(list_name, redis), timeout)
 
 
 async def get_next_job(list_name: str, redis: Redis) -> str:
-    """
-    Get the next job ID from a Redis list.
+    """Get the next job ID from a Redis list.
 
     :param list_name: the name of the list to pop from
     :param redis: the Redis client
     :return: the next job ID
 
     """
-    result = await redis.blpop(list_name)
-
-    if result is not None:
-        job_id = str(result[1], encoding="utf-8")
+    if (job_id := await redis.blpop(list_name)) is not None:
         logger.info("pulled job id from redis", id=job_id)
         return job_id
 
+    raise ValueError("Unexpected None from job id list")
+
 
 async def wait_for_cancellation(redis: Redis, job_id: str, func: Callable):
-    (channel,) = await redis.subscribe(CANCELLATION_CHANNEL)
+    """Call a function ``func`` when a job matching ``job_id`` is cancelled.
 
+    :param redis: the Redis client
+    :param job_id: the job ID to watch for
+    :param func: the function to call when the job is cancelled
+
+    """
     try:
-        async for cancelled_job_id in channel.iter():
-            if cancelled_job_id.decode() == job_id:
+        async for cancelled_job_id in redis.subscribe(CANCELLATION_CHANNEL):
+            if cancelled_job_id == job_id:
                 return func()
 
     except CancelledError:

{virtool_workflow-6.0.0a11 → virtool_workflow-7.1.0}/virtool_workflow/runtime/run.py

@@ -6,12 +6,11 @@ from asyncio import CancelledError
 from pathlib import Path
 from typing import Callable
 
-import pkg_resources
 import structlog
 from pyfixtures import FixtureScope, runs_in_new_fixture_context
 from structlog import get_logger
 from virtool_core.models.job import JobState
-from virtool_core.redis import configure_redis
+from virtool_core.redis import Redis
 
 from virtool_workflow.api.acquire import acquire_job_by_id
 from virtool_workflow.api.client import api_client
@@ -42,6 +41,7 @@ from virtool_workflow.runtime.redis import (
     wait_for_cancellation,
 )
 from virtool_workflow.runtime.sentry import configure_sentry
+from virtool_workflow.utils import get_virtool_workflow_version
 from virtool_workflow.workflow import Workflow
 
 logger = get_logger("runtime")
@@ -229,7 +229,7 @@ async def start_runtime(
 
     logger.info(
         "found virtool-workflow",
-        version=pkg_resources.get_distribution("virtool-workflow").version,
+        version=get_virtool_workflow_version(),
     )
 
     workflow = workflow_loader()
@@ -239,7 +239,7 @@ async def start_runtime(
 
     configure_sentry(sentry_dsn)
 
-    async with configure_redis(redis_connection_string, timeout=15) as redis:
+    async with Redis(redis_connection_string) as redis:
         try:
             job_id = await get_next_job_with_timeout(redis_list_name, redis, timeout)
         except asyncio.TimeoutError:
@@ -277,7 +277,7 @@ async def start_runtime(
         events.cancelled.set()
         run_workflow_task.cancel()
 
-    async with configure_redis(redis_connection_string) as redis:
+    async with Redis(redis_connection_string) as redis:
         cancellation_task = asyncio.create_task(
             wait_for_cancellation(redis, job_id, cancel_workflow),
         )

{virtool_workflow-6.0.0a11 → virtool_workflow-7.1.0}/virtool_workflow/runtime/run_subprocess.py

@@ -1,20 +1,21 @@
 """Code for running and managing subprocesses."""
+
 import asyncio
 from asyncio.subprocess import Process
 from pathlib import Path
-from typing import Awaitable, Callable, Coroutine, Protocol
+from typing import Callable, Coroutine, Protocol
 
 from pyfixtures import fixture
 from structlog import get_logger
 from virtool_core.utils import timestamp
 
-from virtool_workflow.errors import SubprocessFailed
+from virtool_workflow.errors import SubprocessFailedError
 
 logger = get_logger("subprocess")
 
 
 class LineOutputHandler(Protocol):
-    async def __call__(self, line: str):
+    async def __call__(self, line: bytes):
         """Handle input from stdin, or stderr, line by line.
 
         :param line: A line of output from the stream.
@@ -45,7 +46,8 @@ class RunSubprocess(Protocol):
 
 
 async def watch_pipe(
-    stream: asyncio.StreamReader, handler: Callable[[bytes], Awaitable[None]],
+    stream: asyncio.StreamReader,
+    handler: LineOutputHandler,
 ):
     """Watch the stdout or stderr stream and pass lines to the `handler` callback function.
 
@@ -62,6 +64,21 @@ async def watch_pipe(
         await handler(line)
 
 
+def stderr_logger(line: bytes):
+    """Log a line of stderr output and try to decode it as UTF-8.
+
+    If the line is not decodable, log it as a string.
+
+    :param line: a line of stderr output
+    """
+    line = line.rstrip()
+
+    try:
+        logger.info("stderr", line=line.decode())
+    except UnicodeDecodeError:
+        logger.info("stderr", line=line)
+
+
 async def _run_subprocess(
     command: list[str],
     stdout_handler: LineOutputHandler | None = None,
@@ -75,12 +92,17 @@ async def _run_subprocess(
 
     stdout = asyncio.subprocess.PIPE if stdout_handler else asyncio.subprocess.DEVNULL
 
-    async def _stderr_handler(line):
-        logger.info("stderr", line=line.rstrip())
+    if stderr_handler:
 
-        if stderr_handler:
+        async def _stderr_handler(line):
+            stderr_logger(line)
             await stderr_handler(line)
 
+    else:
+
+        async def _stderr_handler(line):
+            stderr_logger(line)
+
     process = await asyncio.create_subprocess_exec(
         *(str(arg) for arg in command),
         cwd=cwd,
@@ -106,9 +128,12 @@ async def _run_subprocess(
 
         process.terminate()
 
-        # Have to do this to avoid Event loop closed error.
+        # Have to do this in Python 3.10 to avoid Event loop closed error.
         # https://github.com/python/cpython/issues/88050
-        process._transport.close()
+        try:
+            process._transport.close()
+        except AttributeError:
+            pass
 
         await process.wait()
         logger.info("subprocess exited", code=process.returncode)
@@ -122,7 +147,7 @@ async def _run_subprocess(
     # Exit code 15 indicates that the process was terminated. This is expected
     # when the workflow fails for some other reason, hence not an exception
     if process.returncode not in [0, 15, -15]:
-        raise SubprocessFailed(
+        raise SubprocessFailedError(
             f"{command[0]} failed with exit code {process.returncode}\n"
             f"arguments: {command}\n",
         )

{virtool_workflow-6.0.0a11 → virtool_workflow-7.1.0}/virtool_workflow/runtime/sentry.py

@@ -1,16 +1,16 @@
 import logging
 
-import pkg_resources
 import sentry_sdk
 from sentry_sdk.integrations.logging import LoggingIntegration
 from structlog import get_logger
 
+from virtool_workflow.utils import get_virtool_workflow_version
+
 logger = get_logger("runtime")
 
 
 def configure_sentry(dsn: str):
-    """Initialize Sentry for log aggregation.
-    """
+    """Initialize Sentry for log aggregation."""
     if dsn:
         logger.info("initializing sentry", dsn=f"{dsn[:15]}...")
 
@@ -21,7 +21,7 @@ def configure_sentry(dsn: str):
                     event_level=logging.WARNING,
                 ),
             ],
-            release=pkg_resources.get_distribution("virtool-workflow").version,
+            release=get_virtool_workflow_version(),
             traces_sample_rate=0.2,
         )
     else:

{virtool_workflow-6.0.0a11 → virtool_workflow-7.1.0}/virtool_workflow/utils.py

@@ -2,6 +2,7 @@ import asyncio
 import tarfile
 from collections.abc import Callable
 from functools import wraps
+from importlib import metadata
 from inspect import iscoroutinefunction
 from pathlib import Path
 
@@ -18,6 +19,14 @@ def coerce_to_coroutine_function(func: Callable):
     return _func
 
 
+def get_virtool_workflow_version() -> str:
+    """Get the version of the installed virtool-workflow package."""
+    try:
+        return metadata.version("virtool-workflow")
+    except metadata.PackageNotFoundError:
+        return "0.0.0"
+
+
 async def make_directory(path: Path):
     await asyncio.to_thread(path.mkdir, exist_ok=True, parents=True)
 

virtool_workflow-6.0.0a11/PKG-INFO

@@ -1,119 +0,0 @@
-Metadata-Version: 2.1
-Name: virtool-workflow
-Version: 6.0.0a11
-Summary: A framework for developing bioinformatics workflows for Virtool.
-Home-page: https://github.com/virtool/virtool-workflow
-License: MIT
-Author: Ian Boyes
-Requires-Python: >=3.10,<3.11
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Topic :: Software Development :: Libraries
-Requires-Dist: aiofiles (>=0.7.0,<0.8.0)
-Requires-Dist: aiohttp (>=3.8.1,<4.0.0)
-Requires-Dist: aioredis (==1.3.1)
-Requires-Dist: biopython (>=1.81,<2.0)
-Requires-Dist: click (>=8.1.7,<9.0.0)
-Requires-Dist: orjson (>=3.9.9,<4.0.0)
-Requires-Dist: pydantic-factories (>=1.17.3,<2.0.0)
-Requires-Dist: pyfixtures (>=1.0.0,<2.0.0)
-Requires-Dist: sentry-sdk (>=1.5.7,<2.0.0)
-Requires-Dist: virtool-core (>=11.0.0,<12.0.0)
-Project-URL: Repository, https://github.com/virtool/virtool-workflow
-Description-Content-Type: text/markdown
-
-# Virtool Workflow
-
-![Tests](https://github.com/virtool/virtool-workflow/workflows/ci/badge.svg?branch=main)
-[![PyPI version](https://badge.fury.io/py/virtool-workflow.svg)](https://badge.fury.io/py/virtool-workflow)
-
-A framework for developing bioinformatic workflows in Python.
-
-```python
-from virtool_workflow import step
-
-
-@step
-def step_function():
-    ...
-
-
-@step
-def step_function_2():
-    ...
-```
-
-- [Documentation](https://workflow.virtool.ca)
-- [Website](https://www.virtool.ca/)
-
-## Contributing
-
-### Commits
-
-All commits must follow the [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0) specification.
-
-These standardized commit messages are used to automatically publish releases using [`semantic-release`](https://semantic-release.gitbook.io/semantic-release)
-after commits are merged to `main` from successful PRs.
-
-**Example**
-
-```text
-feat: add API support for assigning labels to existing samples
-```
-
-Descriptive bodies and footers are required where necessary to describe the impact of the commit. Use bullets where appropriate.
-
-Additional Requirements
-
-1. **Write in the imperative**. For example, _"fix bug"_, not _"fixed bug"_ or _"fixes bug"_.
-2. **Don't refer to issues or code reviews**. For example, don't write something like this: _"make style changes requested in review"_.
-   Instead, _"update styles to improve accessibility"_.
-3. **Commits are not your personal journal**. For example, don't write something like this: _"got server running again"_
-   or _"oops. fixed my code smell"_.
-
-From Tim Pope: [A Note About Git Commit Messages](https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html)
-
-### Tests
-
-[Pytest](https://docs.pytest.org/en/7.1.x/ "Pytest") is used to implement unit
-and integration tests.
-
-A pytest plugin,
-[pytest-docker-compose](https://github.com/pytest-docker-compose/pytest-docker-compose)
-handles starting and stopping any required external services for integration
-tests. [docker-compose](https://docs.docker.com/compose/) will need to be
-installed on your system for this to work. It might also be necessary to setup a
-`docker` user group on your system, so you can [use docker without
-sudo](https://linoxide.com/use-docker-without-sudo-ubuntu/).
-
-`virtool-workflow` depends on some external bioinformatics tools such as [Bowtie
-2](http://bowtie-bio.sourceforge.net/bowtie2/index.shtml),
-[FastQC](https://www.bioinformatics.babraham.ac.uk/projects/fastqc/), and
-[Skewer](https://github.com/relipmoc/skewer). Installation of these tools can be
-somewhat involved, so it's best to run the test suite using `docker`. The
-[virtool/workflow-tools](https://github.com/virtool/workflow-tools) image
-provides a base with all of the external dependencies pre-installed.
-
-[./tests/docker-compose.yml](./tests/docker-compose.yml) will run the test suite
-inside a container based on
-[virtool/workflow-tools](https://github.com/virtool/workflow-tools) and mount
-the local docker socket so that `pytest`, running inside the container, can
-manage the other services required by the integration tests.
-
-To run the entire test suite:
-
-```sh
-cd tests
-docker-compose up --exit-code-from pytest
-```
-
-To run a subset of the tests, `tests/integration` only for example:
-
-```sh
-cd tests
-TEST_PATH=tests/integration docker-compose up --exit-code-from pytest
-```
-
-:warning: The `TEST_PATH` is a relative path from the repository root, not the `tests` directory.
-

virtool_workflow-6.0.0a11/README.md

@@ -1,93 +0,0 @@
-# Virtool Workflow
-
-![Tests](https://github.com/virtool/virtool-workflow/workflows/ci/badge.svg?branch=main)
-[![PyPI version](https://badge.fury.io/py/virtool-workflow.svg)](https://badge.fury.io/py/virtool-workflow)
-
-A framework for developing bioinformatic workflows in Python.
-
-```python
-from virtool_workflow import step
-
-
-@step
-def step_function():
-    ...
-
-
-@step
-def step_function_2():
-    ...
-```
-
-- [Documentation](https://workflow.virtool.ca)
-- [Website](https://www.virtool.ca/)
-
-## Contributing
-
-### Commits
-
-All commits must follow the [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0) specification.
-
-These standardized commit messages are used to automatically publish releases using [`semantic-release`](https://semantic-release.gitbook.io/semantic-release)
-after commits are merged to `main` from successful PRs.
-
-**Example**
-
-```text
-feat: add API support for assigning labels to existing samples
-```
-
-Descriptive bodies and footers are required where necessary to describe the impact of the commit. Use bullets where appropriate.
-
-Additional Requirements
-
-1. **Write in the imperative**. For example, _"fix bug"_, not _"fixed bug"_ or _"fixes bug"_.
-2. **Don't refer to issues or code reviews**. For example, don't write something like this: _"make style changes requested in review"_.
-   Instead, _"update styles to improve accessibility"_.
-3. **Commits are not your personal journal**. For example, don't write something like this: _"got server running again"_
-   or _"oops. fixed my code smell"_.
-
-From Tim Pope: [A Note About Git Commit Messages](https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html)
-
-### Tests
-
-[Pytest](https://docs.pytest.org/en/7.1.x/ "Pytest") is used to implement unit
-and integration tests.
-
-A pytest plugin,
-[pytest-docker-compose](https://github.com/pytest-docker-compose/pytest-docker-compose)
-handles starting and stopping any required external services for integration
-tests. [docker-compose](https://docs.docker.com/compose/) will need to be
-installed on your system for this to work. It might also be necessary to setup a
-`docker` user group on your system, so you can [use docker without
-sudo](https://linoxide.com/use-docker-without-sudo-ubuntu/).
-
-`virtool-workflow` depends on some external bioinformatics tools such as [Bowtie
-2](http://bowtie-bio.sourceforge.net/bowtie2/index.shtml),
-[FastQC](https://www.bioinformatics.babraham.ac.uk/projects/fastqc/), and
-[Skewer](https://github.com/relipmoc/skewer). Installation of these tools can be
-somewhat involved, so it's best to run the test suite using `docker`. The
-[virtool/workflow-tools](https://github.com/virtool/workflow-tools) image
-provides a base with all of the external dependencies pre-installed.
-
-[./tests/docker-compose.yml](./tests/docker-compose.yml) will run the test suite
-inside a container based on
-[virtool/workflow-tools](https://github.com/virtool/workflow-tools) and mount
-the local docker socket so that `pytest`, running inside the container, can
-manage the other services required by the integration tests.
-
-To run the entire test suite:
-
-```sh
-cd tests
-docker-compose up --exit-code-from pytest
-```
-
-To run a subset of the tests, `tests/integration` only for example:
-
-```sh
-cd tests
-TEST_PATH=tests/integration docker-compose up --exit-code-from pytest
-```
-
-:warning: The `TEST_PATH` is a relative path from the repository root, not the `tests` directory.

virtool_workflow-6.0.0a11/virtool_workflow/errors.py

@@ -1,61 +0,0 @@
-from subprocess import SubprocessError
-
-
-class IllegalJobArguments(ValueError):
-    """The `job.args` dict is in an illegal state."""
-
-
-
-class InsufficientJobRights(Exception):
-    ...
-
-
-class JobAlreadyAcquired(Exception):
-    def __init__(self, job_id: str):
-        super(JobAlreadyAcquired, self).__init__(
-            f"Job {job_id} is has already been acquired.",
-        )
-
-
-class JobAlreadyFinalized(Exception):
-    ...
-
-
-class JobsAPIError(Exception):
-    """A base exception for errors due to HTTP errors from the jobs API."""
-
-
-
-class JobsAPIBadRequest(JobsAPIError):
-    """A ``400 Bad Request`` response from the jobs API."""
-
-    status = 400
-
-
-class JobsAPIForbidden(JobsAPIError):
-    status = 403
-
-
-class JobsAPINotFound(JobsAPIError):
-    status = 404
-
-
-class JobsAPIConflict(JobsAPIError):
-    status = 409
-
-
-class JobsAPIServerError(JobsAPIError):
-    status = 500
-
-
-class MissingJobArgument(ValueError):
-    """The `job.args` dict is missing a required key for some funcionality."""
-
-
-
-class NotFound(KeyError):
-    ...
-
-
-class SubprocessFailed(SubprocessError):
-    """Subprocess exited with non-zero status during a workflow."""

virtool_workflow-6.0.0a11/virtool_workflow/files.py

@@ -1,24 +0,0 @@
-from dataclasses import dataclass
-from datetime import datetime
-from typing import Literal
-
-VirtoolFileFormat = Literal[
-    "sam",
-    "bam",
-    "fasta",
-    "fastq",
-    "csv",
-    "tsv",
-    "json",
-    "unknown",
-]
-
-
-@dataclass
-class VirtoolFile:
-    id: int
-    name: str
-    size: int
-    format: VirtoolFileFormat
-    name_on_disk: str = None
-    uploaded_at: datetime = None