virtool-workflow 5.2.1__tar.gz → 5.3.1__tar.gz

{virtool_workflow-5.2.1 → virtool_workflow-5.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: virtool-workflow
-Version: 5.2.1
+Version: 5.3.1
 Summary: A framework for developing bioinformatics workflows for Virtool.
 Home-page: https://github.com/virtool/virtool-workflow
 License: MIT
@@ -10,9 +10,7 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.10
 Classifier: Topic :: Software Development :: Libraries
-Provides-Extra: test
 Requires-Dist: aiofiles (>=0.7.0,<0.8.0)
 Requires-Dist: aiohttp (>=3.8.1,<4.0.0)
 Requires-Dist: aioredis (==1.3.1)

{virtool_workflow-5.2.1 → virtool_workflow-5.3.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "virtool-workflow"
-version = "5.2.1"
+version = "5.3.1"
 description = "A framework for developing bioinformatics workflows for Virtool."
 authors = ["Ian Boyes", "Blake Smith", "Ryan Fang"]
 license = "MIT"
@@ -24,9 +24,6 @@ aioredis = "1.3.1"
 sentry-sdk = "^1.5.7"
 pyfixtures = "^1.0.0"
 
-[tool.poetry.extras]
-test = ["virtool"]
-
 [tool.poetry.scripts]
 run-workflow = "virtool_workflow.cli:cli_main"
 

{virtool_workflow-5.2.1 → virtool_workflow-5.3.1}/virtool_workflow/api/client.py

@@ -1,12 +1,17 @@
+from functools import wraps
+
 import aiohttp
 from pyfixtures import fixture
-from functools import wraps
 
 
 @fixture
 async def http():
     """:class:`Aiohttp.ClientSession` instance to be used for workflows."""
-    async with aiohttp.ClientSession(auto_decompress=False) as session:
+    connector = aiohttp.TCPConnector(force_close=True, limit=100)
+
+    async with aiohttp.ClientSession(
+        auto_decompress=False, connector=connector
+    ) as session:
         yield JobApiHttpSession(session)
 
 

{virtool_workflow-5.2.1 → virtool_workflow-5.3.1}/virtool_workflow/api/jobs.py

@@ -65,14 +65,23 @@ def acquire_job(http: ClientSession, jobs_api_connection_string: str):
 
 
 async def ping(http: ClientSession, jobs_api_connection_string: str, job_id: str):
-    async with http.patch(
-        f"{jobs_api_connection_string}/jobs/{job_id}/ping", json={"acquired": True}
-    ) as response:
+    """
+    Send a ping to the jobs API to indicate that the job is still running.
 
-        async with raising_errors_by_status_code(
-            response, status_codes_to_exceptions={400: JobAlreadyAcquired}
-        ) as resp_json:
-            return WFJob(**resp_json)
+    :param http: An :class:`aiohttp.ClientSession` to use to make the request.
+    :param jobs_api_connection_string: The url for the jobs API.
+    :param job_id: The id of the job to ping.
+    :return: The job.
+    """
+    print(http)
+
+    async with http.put(f"{jobs_api_connection_string}/jobs/{job_id}/ping") as resp:
+        print(resp.status)
+        print(await resp.text())
+
+    print("PING")
+
+    logger.info("Sent ping")
 
 
 @fixture(scope="function")
@@ -109,8 +118,10 @@ async def _push_status(
     state: str,
     progress: float,
     error: Optional[Exception] = None,
-    max_tb: int = 50,
+    max_tb: int = 500,
 ):
+    if error:
+        logger.critical("Reporting error to API", exc_info=error)
 
     payload = {
         "state": state,

{virtool_workflow-5.2.1 → virtool_workflow-5.3.1}/virtool_workflow/runtime/run.py

@@ -1,16 +1,19 @@
 import asyncio
 import signal
 import sys
+from asyncio import CancelledError
 from logging import getLogger
 from pathlib import Path
 from typing import Any, Dict
 
 import pkg_resources
+from aiohttp import ClientOSError, ServerDisconnectedError
 from pyfixtures import FixtureScope, runs_in_new_fixture_context
 from virtool_core.logging import configure_logs
 from virtool_core.redis import configure_redis
 
 from virtool_workflow import execute
+from virtool_workflow.api.jobs import ping
 from virtool_workflow.hooks import (
     on_failure,
     on_cancelled,
@@ -83,6 +86,33 @@ def cleanup_builtin_status_hooks():
     on_terminated.clear()
 
 
+async def ping_periodically(http, job, jobs_api_connection_string, job_id):
+    """
+    Ping the API to keep the job alive.
+
+    """
+    retries = 0
+
+    try:
+        while True:
+            if retries > 5:
+                logger.warning("Failed to ping server")
+                break
+
+            await asyncio.sleep(0.1)
+
+            try:
+                await ping(http, jobs_api_connection_string, job_id)
+            except (ClientOSError, ServerDisconnectedError):
+                await asyncio.sleep(0.3)
+                retries += 1
+                continue
+
+            await asyncio.sleep(5)
+    except CancelledError:
+        logger.info("Stopped pinging server")
+
+
 async def run_workflow(
     config: Dict[str, Any],
     job_id: str,
@@ -96,13 +126,19 @@ async def run_workflow(
         scope["config"] = config
         scope["job_id"] = job_id
 
+        bound_ping = await scope.bind(ping_periodically)
+
         execute_task = asyncio.create_task(execute(workflow, scope, events))
+        ping_task = asyncio.create_task(bound_ping())
 
         try:
             await execute_task
         except asyncio.CancelledError:
             execute_task.cancel()
 
+        ping_task.cancel()
+
+        await ping_task
         await execute_task
 
         cleanup_builtin_status_hooks()

{virtool_workflow-5.2.1 → virtool_workflow-5.3.1}/virtool_workflow/runtime/sentry.py

@@ -9,7 +9,7 @@ from sentry_sdk.integrations.logging import LoggingIntegration
 logger = getLogger("runtime")
 
 
-def configure_sentry(dsn: Optional[str], event_level: int = logging.ERROR):
+def configure_sentry(dsn: Optional[str]):
     """
     Initialize Sentry for log aggregation.
     """
@@ -22,7 +22,7 @@ def configure_sentry(dsn: Optional[str], event_level: int = logging.ERROR):
         dsn=dsn,
         integrations=[
             LoggingIntegration(
-                event_level=event_level,
+                event_level=logging.WARNING,
             )
         ],
         release=pkg_resources.get_distribution("virtool-workflow").version,

virtool_workflow-5.2.1/setup.py

@@ -1,45 +0,0 @@
-# -*- coding: utf-8 -*-
-from setuptools import setup
-
-packages = \
-['virtool_workflow',
- 'virtool_workflow.analysis',
- 'virtool_workflow.api',
- 'virtool_workflow.data_model',
- 'virtool_workflow.runtime',
- 'virtool_workflow.testing']
-
-package_data = \
-{'': ['*']}
-
-install_requires = \
-['aiofiles>=0.7.0,<0.8.0',
- 'aiohttp>=3.8.1,<4.0.0',
- 'aioredis==1.3.1',
- 'click>=8.0.0,<9.0.0',
- 'pyfixtures>=1.0.0,<2.0.0',
- 'sentry-sdk>=1.5.7,<2.0.0',
- 'virtool-core>=3.0.0,<4.0.0']
-
-entry_points = \
-{'console_scripts': ['run-workflow = virtool_workflow.cli:cli_main']}
-
-setup_kwargs = {
-    'name': 'virtool-workflow',
-    'version': '5.2.1',
-    'description': 'A framework for developing bioinformatics workflows for Virtool.',
-    'long_description': '# Virtool Workflow\n\n![Tests](https://github.com/virtool/virtool-workflow/workflows/Tests/badge.svg?branch=master)\n[![PyPI version](https://badge.fury.io/py/virtool-workflow.svg)](https://badge.fury.io/py/virtool-workflow)\n\nA framework for developing bioinformatic workflows in Python.\n\n```python\nfrom virtool_workflow import step\n\n\n@step\ndef step_function():\n    ...\n\n\n@step\ndef step_function_2():\n    ...\n```\n\n- [Documentation](https://workflow.virtool.ca)\n- [Website](https://www.virtool.ca/)\n\n## Contributing\n\n### Commits\n\nAll commits must follow the [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0) specification.\n\nThese standardized commit messages are used to automatically publish releases using [`semantic-release`](https://semantic-release.gitbook.io/semantic-release)\nafter commits are merged to `main` from successful PRs.\n\n**Example**\n\n```text\nfeat: add API support for assigning labels to existing samples\n```\n\nDescriptive bodies and footers are required where necessary to describe the impact of the commit. Use bullets where appropriate.\n\nAdditional Requirements\n\n1. **Write in the imperative**. For example, _"fix bug"_, not _"fixed bug"_ or _"fixes bug"_.\n2. **Don\'t refer to issues or code reviews**. For example, don\'t write something like this: _"make style changes requested in review"_.\n   Instead, _"update styles to improve accessibility"_.\n3. **Commits are not your personal journal**. For example, don\'t write something like this: _"got server running again"_\n   or _"oops. fixed my code smell"_.\n\nFrom Tim Pope: [A Note About Git Commit Messages](https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html)\n\n### Poetry\n\nDependencies & virtual environments are managed with [Poetry](https://python-poetry.org/ "Poetry")\n\nTo install `poetry`:\n\n```sh\nsudo pip install poetry\n```\n\nTo install dependencies, and the `virtool-workflow` package, into a virtual environment:\n\n```sh\ngit clone https://github.com/virtool/virtool-workflow\ncd virtool-workflow\n\npoetry install\n```\n\nTo run commands in the virtual environment:\n\n```sh\npoetry run <<command>>\n```\n\n### Tests\n\n[Pytest](https://docs.pytest.org/en/7.1.x/ "Pytest") is used to implement unit\nand integration tests.\n\nA pytest plugin,\n[pytest-docker-compose](https://github.com/pytest-docker-compose/pytest-docker-compose)\nhandles starting and stopping any required external services for integration\ntests. [docker-compose](https://docs.docker.com/compose/) will need to be\ninstalled on your system for this to work. It might also be necessary to setup a\n`docker` user group on your system, so you can [use docker without\nsudo](https://linoxide.com/use-docker-without-sudo-ubuntu/).\n\n`virtool-workflow` depends on some external bioinformatics tools such as [Bowtie\n2](http://bowtie-bio.sourceforge.net/bowtie2/index.shtml),\n[FastQC](https://www.bioinformatics.babraham.ac.uk/projects/fastqc/), and\n[Skewer](https://github.com/relipmoc/skewer). Installation of these tools can be\nsomewhat involved, so it\'s best to run the test suite using `docker`. The\n[virtool/workflow-tools](https://github.com/virtool/workflow-tools) image\nprovides a base with all of the external dependencies pre-installed.\n\n[./tests/docker-compose.yml](./tests/docker-compose.yml) will run the test suite\ninside a container based on\n[virtool/workflow-tools](https://github.com/virtool/workflow-tools) and mount\nthe local docker socket so that `pytest`, running inside the container, can\nmanage the other services required by the integration tests.\n\nTo run the entire test suite:\n\n```sh\ncd tests\ndocker-compose up --exit-code-from pytest\n```\n\nTo run a subset of the tests, `tests/integration` only for example:\n\n```sh\ncd tests\nTEST_PATH=tests/integration docker-compose up --exit-code-from pytest\n```\n\n:warning: The `TEST_PATH` is a relative path from the repository root, not the `tests` directory.\n',
-    'author': 'Ian Boyes',
-    'author_email': 'None',
-    'maintainer': 'None',
-    'maintainer_email': 'None',
-    'url': 'https://github.com/virtool/virtool-workflow',
-    'packages': packages,
-    'package_data': package_data,
-    'install_requires': install_requires,
-    'entry_points': entry_points,
-    'python_requires': '>=3.10,<4.0',
-}
-
-
-setup(**setup_kwargs)