virtool-workflow 6.0.0a10__tar.gz → 7.0.0__tar.gz

virtool_workflow-7.0.0/PKG-INFO

@@ -0,0 +1,76 @@
+Metadata-Version: 2.1
+Name: virtool-workflow
+Version: 7.0.0
+Summary: A framework for developing bioinformatics workflows for Virtool.
+Home-page: https://github.com/virtool/virtool-workflow
+License: MIT
+Author: Ian Boyes
+Maintainer: Ian Boyes
+Requires-Python: >=3.10,<3.13
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: aiofiles (>=0.7.0,<0.8.0)
+Requires-Dist: aiohttp (>=3.8.1,<4.0.0)
+Requires-Dist: biopython (>=1.81,<2.0)
+Requires-Dist: click (>=8.1.7,<9.0.0)
+Requires-Dist: orjson (>=3.9.9,<4.0.0)
+Requires-Dist: pydantic-factories (>=1.17.3,<2.0.0)
+Requires-Dist: pyfixtures (>=1.0.0,<2.0.0)
+Requires-Dist: sentry-sdk (>=2.3.1,<3.0.0)
+Requires-Dist: virtool-core (>=12.0.0,<13.0.0)
+Project-URL: Repository, https://github.com/virtool/virtool-workflow
+Description-Content-Type: text/markdown
+
+# Virtool Workflow
+
+![Tests](https://github.com/virtool/virtool-workflow/workflows/ci/badge.svg?branch=main)
+[![PyPI version](https://badge.fury.io/py/virtool-workflow.svg)](https://badge.fury.io/py/virtool-workflow)
+
+A framework for developing bioinformatic workflows in Python.
+
+```python
+from virtool_workflow import step
+
+
+@step
+def step_function():
+    ...
+
+
+@step
+def step_function_2():
+    ...
+```
+
+## Contributing
+
+### Commits
+
+We require specific commit formatting. Any commit that does not follow the guidelines
+will be squashed at our discretion.
+
+Read our [commit and release](https://dev.virtool.ca/en/latest/commits_releases.html)
+documentation for more information.
+
+### Tests
+
+Run tests with:
+
+```shell
+# Bring up Redis and the test container.
+docker compose up -d
+
+# Run tests in the test container.
+docker compose exec test poetry run pytest
+
+```
+
+Run specific tests like:
+
+```shell
+docker compose exec test poetry run pytest tests/test_status.py
+```
+

virtool_workflow-7.0.0/README.md

@@ -0,0 +1,49 @@
+# Virtool Workflow
+
+![Tests](https://github.com/virtool/virtool-workflow/workflows/ci/badge.svg?branch=main)
+[![PyPI version](https://badge.fury.io/py/virtool-workflow.svg)](https://badge.fury.io/py/virtool-workflow)
+
+A framework for developing bioinformatic workflows in Python.
+
+```python
+from virtool_workflow import step
+
+
+@step
+def step_function():
+    ...
+
+
+@step
+def step_function_2():
+    ...
+```
+
+## Contributing
+
+### Commits
+
+We require specific commit formatting. Any commit that does not follow the guidelines
+will be squashed at our discretion.
+
+Read our [commit and release](https://dev.virtool.ca/en/latest/commits_releases.html)
+documentation for more information.
+
+### Tests
+
+Run tests with:
+
+```shell
+# Bring up Redis and the test container.
+docker compose up -d
+
+# Run tests in the test container.
+docker compose exec test poetry run pytest
+
+```
+
+Run specific tests like:
+
+```shell
+docker compose exec test poetry run pytest tests/test_status.py
+```

{virtool_workflow-6.0.0a10 → virtool_workflow-7.0.0}/pyproject.toml

@@ -1,32 +1,44 @@
 [tool.poetry]
 name = "virtool-workflow"
-version = "6.0.0-alpha.10"
+version = "7.0.0"
 description = "A framework for developing bioinformatics workflows for Virtool."
-authors = ["Ian Boyes", "Blake Smith", "Ryan Fang"]
+authors = [
+    "Ian Boyes",
+    "Blake Smith",
+    "Ryan Fang",
+    "Matt Curtis",
+    "Aman Monga",
+    "Bryce Davidson",
+    "Christine Wong Chong",
+    "Lilly Roberts",
+    "Markus Swoveland",
+    "Ryan Fang",
+]
 license = "MIT"
+maintainers = [
+    "Ian Boyes",
+    "Reece Hoffman"
+]
 readme = "README.md"
 repository = "https://github.com/virtool/virtool-workflow"
 classifiers = [
-    "Topic :: Software Development :: Libraries",
-    "Programming Language :: Python :: 3.10"
+    "Programming Language :: Python :: 3.12",
 ]
 packages = [
     { include = "virtool_workflow" },
 ]
 
 [tool.poetry.dependencies]
-python = "~3.10"
-click = "^8.1.7"
-aiohttp = "^3.8.1"
+python = "^3.10,<3.13"
 aiofiles = "^0.7.0"
-virtool-core = "^11.0.0"
-aioredis = "1.3.1"
-sentry-sdk = "^1.5.7"
-pyfixtures = "^1.0.0"
-orjson = "^3.9.9"
+aiohttp = "^3.8.1"
 biopython = "^1.81"
+click = "^8.1.7"
+orjson = "^3.9.9"
 pydantic-factories = "^1.17.3"
-
+pyfixtures = "^1.0.0"
+sentry-sdk = "^2.3.1"
+virtool-core = "^12.0.0"
 
 [tool.poetry.scripts]
 run-workflow = "virtool_workflow.cli:cli_main"
@@ -36,13 +48,14 @@ pytest = "^7.4.2"
 pytest-aiohttp = "^1.0.0"
 pytest-asyncio = "^0.21.0"
 pytest-mock = "^3.10.0"
-pre-commit = "^2.18.1"
 sphinx = "^4.0.2"
 syrupy = "^3.0.5"
 sphinx-autobuild = "^2021.3.14"
 sphinx-nameko-theme = "^0.0.3"
 pytest-structlog = "^0.6"
 sphinx-toolbox = "^3.5.0"
+ruff = "^0.4.6"
+piccolo-theme = "^0.22.0"
 
 [tool.pytest.ini_options]
 asyncio_mode = "auto"

{virtool_workflow-6.0.0a10 → virtool_workflow-7.0.0}/virtool_workflow/runtime/redis.py

@@ -2,8 +2,8 @@ import asyncio
 from asyncio import CancelledError
 from typing import Callable
 
-from aioredis import Redis
 from structlog import get_logger
+from virtool_core.redis import Redis
 
 logger = get_logger("redis")
 
@@ -11,10 +11,11 @@ CANCELLATION_CHANNEL = "channel:cancel"
 
 
 async def get_next_job_with_timeout(
-    list_name: str, redis: Redis, timeout: int = None
+    list_name: str,
+    redis: Redis,
+    timeout: int | None = None,
 ) -> str:
-    """
-    Get the next job ID from a Redis list and raise a  :class:``Timeout`` error if one
+    """Get the next job ID from a Redis list and raise a  :class:``Timeout`` error if one
     is not found in ``timeout`` seconds.
 
     :param list_name: the name of the list to pop from
@@ -24,35 +25,39 @@ async def get_next_job_with_timeout(
 
     """
     logger.info(
-        "Waiting for a job", timeout=f"{timeout if timeout else 'infinity'} seconds"
+        "Waiting for a job",
+        timeout=f"{timeout if timeout else 'infinity'} seconds",
     )
 
     return await asyncio.wait_for(get_next_job(list_name, redis), timeout)
 
 
 async def get_next_job(list_name: str, redis: Redis) -> str:
-    """
-    Get the next job ID from a Redis list.
+    """Get the next job ID from a Redis list.
 
     :param list_name: the name of the list to pop from
     :param redis: the Redis client
     :return: the next job ID
 
     """
-    result = await redis.blpop(list_name)
-
-    if result is not None:
-        job_id = str(result[1], encoding="utf-8")
+    if (job_id := await redis.blpop(list_name)) is not None:
         logger.info("pulled job id from redis", id=job_id)
         return job_id
 
+    raise ValueError("Unexpected None from job id list")
+
 
 async def wait_for_cancellation(redis: Redis, job_id: str, func: Callable):
-    (channel,) = await redis.subscribe(CANCELLATION_CHANNEL)
+    """Call a function ``func`` when a job matching ``job_id`` is cancelled.
 
+    :param redis: the Redis client
+    :param job_id: the job ID to watch for
+    :param func: the function to call when the job is cancelled
+
+    """
     try:
-        async for cancelled_job_id in channel.iter():
-            if cancelled_job_id.decode() == job_id:
+        async for cancelled_job_id in redis.subscribe(CANCELLATION_CHANNEL):
+            if cancelled_job_id == job_id:
                 return func()
 
     except CancelledError:

{virtool_workflow-6.0.0a10 → virtool_workflow-7.0.0}/virtool_workflow/runtime/run.py

@@ -6,12 +6,11 @@ from asyncio import CancelledError
 from pathlib import Path
 from typing import Callable
 
-import pkg_resources
 import structlog
 from pyfixtures import FixtureScope, runs_in_new_fixture_context
 from structlog import get_logger
 from virtool_core.models.job import JobState
-from virtool_core.redis import configure_redis
+from virtool_core.redis import Redis
 
 from virtool_workflow.api.acquire import acquire_job_by_id
 from virtool_workflow.api.client import api_client
@@ -42,6 +41,7 @@ from virtool_workflow.runtime.redis import (
     wait_for_cancellation,
 )
 from virtool_workflow.runtime.sentry import configure_sentry
+from virtool_workflow.utils import get_virtool_workflow_version
 from virtool_workflow.workflow import Workflow
 
 logger = get_logger("runtime")
@@ -128,6 +128,8 @@ async def execute(workflow: Workflow, scope: FixtureScope, events: Events):
         scope["_error"] = error
         scope["_state"] = JobState.ERROR
 
+        logger.exception(error)
+
         await asyncio.gather(on_error.trigger(scope), on_failure.trigger(scope))
 
         if isinstance(error, asyncio.CancelledError):
@@ -160,7 +162,9 @@ async def run_workflow(
     job = await acquire_job_by_id(config.jobs_api_connection_string, job_id)
 
     async with api_client(
-        config.jobs_api_connection_string, job.id, job.key,
+        config.jobs_api_connection_string,
+        job.id,
+        job.key,
     ) as api, FixtureScope() as scope:
         # These fixtures should not be used directly by the workflow. They are used
         # by other built-in fixtures.
@@ -225,7 +229,7 @@ async def start_runtime(
 
     logger.info(
         "found virtool-workflow",
-        version=pkg_resources.get_distribution("virtool-workflow").version,
+        version=get_virtool_workflow_version(),
     )
 
     workflow = workflow_loader()
@@ -235,7 +239,7 @@ async def start_runtime(
 
     configure_sentry(sentry_dsn)
 
-    async with configure_redis(redis_connection_string, timeout=15) as redis:
+    async with Redis(redis_connection_string) as redis:
         try:
             job_id = await get_next_job_with_timeout(redis_list_name, redis, timeout)
         except asyncio.TimeoutError:
@@ -273,7 +277,7 @@ async def start_runtime(
         events.cancelled.set()
         run_workflow_task.cancel()
 
-    async with configure_redis(redis_connection_string) as redis:
+    async with Redis(redis_connection_string) as redis:
         cancellation_task = asyncio.create_task(
             wait_for_cancellation(redis, job_id, cancel_workflow),
         )

{virtool_workflow-6.0.0a10 → virtool_workflow-7.0.0}/virtool_workflow/runtime/run_subprocess.py

@@ -2,7 +2,7 @@
 import asyncio
 from asyncio.subprocess import Process
 from pathlib import Path
-from typing import Awaitable, Callable, Coroutine, Protocol
+from typing import Callable, Coroutine, Protocol
 
 from pyfixtures import fixture
 from structlog import get_logger
@@ -14,7 +14,7 @@ logger = get_logger("subprocess")
 
 
 class LineOutputHandler(Protocol):
-    async def __call__(self, line: str):
+    async def __call__(self, line: bytes):
         """Handle input from stdin, or stderr, line by line.
 
         :param line: A line of output from the stream.
@@ -45,7 +45,8 @@ class RunSubprocess(Protocol):
 
 
 async def watch_pipe(
-    stream: asyncio.StreamReader, handler: Callable[[bytes], Awaitable[None]],
+    stream: asyncio.StreamReader,
+    handler: LineOutputHandler,
 ):
     """Watch the stdout or stderr stream and pass lines to the `handler` callback function.
 
@@ -62,6 +63,21 @@ async def watch_pipe(
         await handler(line)
 
 
+def stderr_logger(line: bytes):
+    """Log a line of stderr output and try to decode it as UTF-8.
+
+    If the line is not decodable, log it as a string.
+
+    :param line: a line of stderr output
+    """
+    line = line.rstrip()
+
+    try:
+        logger.info("stderr", line=line.decode())
+    except UnicodeDecodeError:
+        logger.info("stderr", line=line)
+
+
 async def _run_subprocess(
     command: list[str],
     stdout_handler: LineOutputHandler | None = None,
@@ -75,12 +91,17 @@ async def _run_subprocess(
 
     stdout = asyncio.subprocess.PIPE if stdout_handler else asyncio.subprocess.DEVNULL
 
-    async def _stderr_handler(line):
-        logger.info("stderr", line=line.rstrip())
+    if stderr_handler:
 
-        if stderr_handler:
+        async def _stderr_handler(line):
+            stderr_logger(line)
             await stderr_handler(line)
 
+    else:
+
+        async def _stderr_handler(line):
+            stderr_logger(line)
+
     process = await asyncio.create_subprocess_exec(
         *(str(arg) for arg in command),
         cwd=cwd,
@@ -106,9 +127,12 @@ async def _run_subprocess(
 
         process.terminate()
 
-        # Have to do this to avoid Event loop closed error.
+        # Have to do this in Python 3.10 to avoid Event loop closed error.
         # https://github.com/python/cpython/issues/88050
-        process._transport.close()
+        try:
+            process._transport.close()
+        except AttributeError:
+            pass
 
         await process.wait()
         logger.info("subprocess exited", code=process.returncode)

{virtool_workflow-6.0.0a10 → virtool_workflow-7.0.0}/virtool_workflow/runtime/sentry.py

@@ -1,16 +1,16 @@
 import logging
 
-import pkg_resources
 import sentry_sdk
 from sentry_sdk.integrations.logging import LoggingIntegration
 from structlog import get_logger
 
+from virtool_workflow.utils import get_virtool_workflow_version
+
 logger = get_logger("runtime")
 
 
 def configure_sentry(dsn: str):
-    """Initialize Sentry for log aggregation.
-    """
+    """Initialize Sentry for log aggregation."""
     if dsn:
         logger.info("initializing sentry", dsn=f"{dsn[:15]}...")
 
@@ -21,7 +21,7 @@ def configure_sentry(dsn: str):
                     event_level=logging.WARNING,
                 ),
             ],
-            release=pkg_resources.get_distribution("virtool-workflow").version,
+            release=get_virtool_workflow_version(),
             traces_sample_rate=0.2,
         )
     else:

{virtool_workflow-6.0.0a10 → virtool_workflow-7.0.0}/virtool_workflow/utils.py

@@ -2,6 +2,7 @@ import asyncio
 import tarfile
 from collections.abc import Callable
 from functools import wraps
+from importlib import metadata
 from inspect import iscoroutinefunction
 from pathlib import Path
 
@@ -18,6 +19,14 @@ def coerce_to_coroutine_function(func: Callable):
     return _func
 
 
+def get_virtool_workflow_version() -> str:
+    """Get the version of the installed virtool-workflow package."""
+    try:
+        return metadata.version("virtool-workflow")
+    except metadata.PackageNotFoundError:
+        return "0.0.0"
+
+
 async def make_directory(path: Path):
     await asyncio.to_thread(path.mkdir, exist_ok=True, parents=True)
 

virtool_workflow-6.0.0a10/PKG-INFO

@@ -1,119 +0,0 @@
-Metadata-Version: 2.1
-Name: virtool-workflow
-Version: 6.0.0a10
-Summary: A framework for developing bioinformatics workflows for Virtool.
-Home-page: https://github.com/virtool/virtool-workflow
-License: MIT
-Author: Ian Boyes
-Requires-Python: >=3.10,<3.11
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Topic :: Software Development :: Libraries
-Requires-Dist: aiofiles (>=0.7.0,<0.8.0)
-Requires-Dist: aiohttp (>=3.8.1,<4.0.0)
-Requires-Dist: aioredis (==1.3.1)
-Requires-Dist: biopython (>=1.81,<2.0)
-Requires-Dist: click (>=8.1.7,<9.0.0)
-Requires-Dist: orjson (>=3.9.9,<4.0.0)
-Requires-Dist: pydantic-factories (>=1.17.3,<2.0.0)
-Requires-Dist: pyfixtures (>=1.0.0,<2.0.0)
-Requires-Dist: sentry-sdk (>=1.5.7,<2.0.0)
-Requires-Dist: virtool-core (>=11.0.0,<12.0.0)
-Project-URL: Repository, https://github.com/virtool/virtool-workflow
-Description-Content-Type: text/markdown
-
-# Virtool Workflow
-
-![Tests](https://github.com/virtool/virtool-workflow/workflows/ci/badge.svg?branch=main)
-[![PyPI version](https://badge.fury.io/py/virtool-workflow.svg)](https://badge.fury.io/py/virtool-workflow)
-
-A framework for developing bioinformatic workflows in Python.
-
-```python
-from virtool_workflow import step
-
-
-@step
-def step_function():
-    ...
-
-
-@step
-def step_function_2():
-    ...
-```
-
-- [Documentation](https://workflow.virtool.ca)
-- [Website](https://www.virtool.ca/)
-
-## Contributing
-
-### Commits
-
-All commits must follow the [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0) specification.
-
-These standardized commit messages are used to automatically publish releases using [`semantic-release`](https://semantic-release.gitbook.io/semantic-release)
-after commits are merged to `main` from successful PRs.
-
-**Example**
-
-```text
-feat: add API support for assigning labels to existing samples
-```
-
-Descriptive bodies and footers are required where necessary to describe the impact of the commit. Use bullets where appropriate.
-
-Additional Requirements
-
-1. **Write in the imperative**. For example, _"fix bug"_, not _"fixed bug"_ or _"fixes bug"_.
-2. **Don't refer to issues or code reviews**. For example, don't write something like this: _"make style changes requested in review"_.
-   Instead, _"update styles to improve accessibility"_.
-3. **Commits are not your personal journal**. For example, don't write something like this: _"got server running again"_
-   or _"oops. fixed my code smell"_.
-
-From Tim Pope: [A Note About Git Commit Messages](https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html)
-
-### Tests
-
-[Pytest](https://docs.pytest.org/en/7.1.x/ "Pytest") is used to implement unit
-and integration tests.
-
-A pytest plugin,
-[pytest-docker-compose](https://github.com/pytest-docker-compose/pytest-docker-compose)
-handles starting and stopping any required external services for integration
-tests. [docker-compose](https://docs.docker.com/compose/) will need to be
-installed on your system for this to work. It might also be necessary to setup a
-`docker` user group on your system, so you can [use docker without
-sudo](https://linoxide.com/use-docker-without-sudo-ubuntu/).
-
-`virtool-workflow` depends on some external bioinformatics tools such as [Bowtie
-2](http://bowtie-bio.sourceforge.net/bowtie2/index.shtml),
-[FastQC](https://www.bioinformatics.babraham.ac.uk/projects/fastqc/), and
-[Skewer](https://github.com/relipmoc/skewer). Installation of these tools can be
-somewhat involved, so it's best to run the test suite using `docker`. The
-[virtool/workflow-tools](https://github.com/virtool/workflow-tools) image
-provides a base with all of the external dependencies pre-installed.
-
-[./tests/docker-compose.yml](./tests/docker-compose.yml) will run the test suite
-inside a container based on
-[virtool/workflow-tools](https://github.com/virtool/workflow-tools) and mount
-the local docker socket so that `pytest`, running inside the container, can
-manage the other services required by the integration tests.
-
-To run the entire test suite:
-
-```sh
-cd tests
-docker-compose up --exit-code-from pytest
-```
-
-To run a subset of the tests, `tests/integration` only for example:
-
-```sh
-cd tests
-TEST_PATH=tests/integration docker-compose up --exit-code-from pytest
-```
-
-:warning: The `TEST_PATH` is a relative path from the repository root, not the `tests` directory.
-

virtool_workflow-6.0.0a10/README.md

@@ -1,93 +0,0 @@
-# Virtool Workflow
-
-![Tests](https://github.com/virtool/virtool-workflow/workflows/ci/badge.svg?branch=main)
-[![PyPI version](https://badge.fury.io/py/virtool-workflow.svg)](https://badge.fury.io/py/virtool-workflow)
-
-A framework for developing bioinformatic workflows in Python.
-
-```python
-from virtool_workflow import step
-
-
-@step
-def step_function():
-    ...
-
-
-@step
-def step_function_2():
-    ...
-```
-
-- [Documentation](https://workflow.virtool.ca)
-- [Website](https://www.virtool.ca/)
-
-## Contributing
-
-### Commits
-
-All commits must follow the [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0) specification.
-
-These standardized commit messages are used to automatically publish releases using [`semantic-release`](https://semantic-release.gitbook.io/semantic-release)
-after commits are merged to `main` from successful PRs.
-
-**Example**
-
-```text
-feat: add API support for assigning labels to existing samples
-```
-
-Descriptive bodies and footers are required where necessary to describe the impact of the commit. Use bullets where appropriate.
-
-Additional Requirements
-
-1. **Write in the imperative**. For example, _"fix bug"_, not _"fixed bug"_ or _"fixes bug"_.
-2. **Don't refer to issues or code reviews**. For example, don't write something like this: _"make style changes requested in review"_.
-   Instead, _"update styles to improve accessibility"_.
-3. **Commits are not your personal journal**. For example, don't write something like this: _"got server running again"_
-   or _"oops. fixed my code smell"_.
-
-From Tim Pope: [A Note About Git Commit Messages](https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html)
-
-### Tests
-
-[Pytest](https://docs.pytest.org/en/7.1.x/ "Pytest") is used to implement unit
-and integration tests.
-
-A pytest plugin,
-[pytest-docker-compose](https://github.com/pytest-docker-compose/pytest-docker-compose)
-handles starting and stopping any required external services for integration
-tests. [docker-compose](https://docs.docker.com/compose/) will need to be
-installed on your system for this to work. It might also be necessary to setup a
-`docker` user group on your system, so you can [use docker without
-sudo](https://linoxide.com/use-docker-without-sudo-ubuntu/).
-
-`virtool-workflow` depends on some external bioinformatics tools such as [Bowtie
-2](http://bowtie-bio.sourceforge.net/bowtie2/index.shtml),
-[FastQC](https://www.bioinformatics.babraham.ac.uk/projects/fastqc/), and
-[Skewer](https://github.com/relipmoc/skewer). Installation of these tools can be
-somewhat involved, so it's best to run the test suite using `docker`. The
-[virtool/workflow-tools](https://github.com/virtool/workflow-tools) image
-provides a base with all of the external dependencies pre-installed.
-
-[./tests/docker-compose.yml](./tests/docker-compose.yml) will run the test suite
-inside a container based on
-[virtool/workflow-tools](https://github.com/virtool/workflow-tools) and mount
-the local docker socket so that `pytest`, running inside the container, can
-manage the other services required by the integration tests.
-
-To run the entire test suite:
-
-```sh
-cd tests
-docker-compose up --exit-code-from pytest
-```
-
-To run a subset of the tests, `tests/integration` only for example:
-
-```sh
-cd tests
-TEST_PATH=tests/integration docker-compose up --exit-code-from pytest
-```
-
-:warning: The `TEST_PATH` is a relative path from the repository root, not the `tests` directory.