virtool-workflow 5.2.1__py3-none-any.whl → 5.3.1__py3-none-any.whl

virtool_workflow/api/client.py

@@ -1,12 +1,17 @@
+from functools import wraps
+
 import aiohttp
 from pyfixtures import fixture
-from functools import wraps
 
 
 @fixture
 async def http():
     """:class:`Aiohttp.ClientSession` instance to be used for workflows."""
-    async with aiohttp.ClientSession(auto_decompress=False) as session:
+    connector = aiohttp.TCPConnector(force_close=True, limit=100)
+
+    async with aiohttp.ClientSession(
+        auto_decompress=False, connector=connector
+    ) as session:
         yield JobApiHttpSession(session)
 
 

virtool_workflow/api/jobs.py

@@ -65,14 +65,23 @@ def acquire_job(http: ClientSession, jobs_api_connection_string: str):
 
 
 async def ping(http: ClientSession, jobs_api_connection_string: str, job_id: str):
-    async with http.patch(
-        f"{jobs_api_connection_string}/jobs/{job_id}/ping", json={"acquired": True}
-    ) as response:
+    """
+    Send a ping to the jobs API to indicate that the job is still running.
 
-        async with raising_errors_by_status_code(
-            response, status_codes_to_exceptions={400: JobAlreadyAcquired}
-        ) as resp_json:
-            return WFJob(**resp_json)
+    :param http: An :class:`aiohttp.ClientSession` to use to make the request.
+    :param jobs_api_connection_string: The url for the jobs API.
+    :param job_id: The id of the job to ping.
+    :return: The job.
+    """
+    print(http)
+
+    async with http.put(f"{jobs_api_connection_string}/jobs/{job_id}/ping") as resp:
+        print(resp.status)
+        print(await resp.text())
+
+    print("PING")
+
+    logger.info("Sent ping")
 
 
 @fixture(scope="function")
@@ -109,8 +118,10 @@ async def _push_status(
     state: str,
     progress: float,
     error: Optional[Exception] = None,
-    max_tb: int = 50,
+    max_tb: int = 500,
 ):
+    if error:
+        logger.critical("Reporting error to API", exc_info=error)
 
     payload = {
         "state": state,

virtool_workflow/runtime/run.py

@@ -1,16 +1,19 @@
 import asyncio
 import signal
 import sys
+from asyncio import CancelledError
 from logging import getLogger
 from pathlib import Path
 from typing import Any, Dict
 
 import pkg_resources
+from aiohttp import ClientOSError, ServerDisconnectedError
 from pyfixtures import FixtureScope, runs_in_new_fixture_context
 from virtool_core.logging import configure_logs
 from virtool_core.redis import configure_redis
 
 from virtool_workflow import execute
+from virtool_workflow.api.jobs import ping
 from virtool_workflow.hooks import (
     on_failure,
     on_cancelled,
@@ -83,6 +86,33 @@ def cleanup_builtin_status_hooks():
     on_terminated.clear()
 
 
+async def ping_periodically(http, job, jobs_api_connection_string, job_id):
+    """
+    Ping the API to keep the job alive.
+
+    """
+    retries = 0
+
+    try:
+        while True:
+            if retries > 5:
+                logger.warning("Failed to ping server")
+                break
+
+            await asyncio.sleep(0.1)
+
+            try:
+                await ping(http, jobs_api_connection_string, job_id)
+            except (ClientOSError, ServerDisconnectedError):
+                await asyncio.sleep(0.3)
+                retries += 1
+                continue
+
+            await asyncio.sleep(5)
+    except CancelledError:
+        logger.info("Stopped pinging server")
+
+
 async def run_workflow(
     config: Dict[str, Any],
     job_id: str,
@@ -96,13 +126,19 @@ async def run_workflow(
         scope["config"] = config
         scope["job_id"] = job_id
 
+        bound_ping = await scope.bind(ping_periodically)
+
         execute_task = asyncio.create_task(execute(workflow, scope, events))
+        ping_task = asyncio.create_task(bound_ping())
 
         try:
             await execute_task
         except asyncio.CancelledError:
             execute_task.cancel()
 
+        ping_task.cancel()
+
+        await ping_task
         await execute_task
 
         cleanup_builtin_status_hooks()

virtool_workflow/runtime/sentry.py

@@ -9,7 +9,7 @@ from sentry_sdk.integrations.logging import LoggingIntegration
 logger = getLogger("runtime")
 
 
-def configure_sentry(dsn: Optional[str], event_level: int = logging.ERROR):
+def configure_sentry(dsn: Optional[str]):
     """
     Initialize Sentry for log aggregation.
     """
@@ -22,7 +22,7 @@ def configure_sentry(dsn: Optional[str], event_level: int = logging.ERROR):
         dsn=dsn,
         integrations=[
             LoggingIntegration(
-                event_level=event_level,
+                event_level=logging.WARNING,
             )
         ],
         release=pkg_resources.get_distribution("virtool-workflow").version,

{virtool_workflow-5.2.1.dist-info → virtool_workflow-5.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: virtool-workflow
-Version: 5.2.1
+Version: 5.3.1
 Summary: A framework for developing bioinformatics workflows for Virtool.
 Home-page: https://github.com/virtool/virtool-workflow
 License: MIT
@@ -10,9 +10,7 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.10
 Classifier: Topic :: Software Development :: Libraries
-Provides-Extra: test
 Requires-Dist: aiofiles (>=0.7.0,<0.8.0)
 Requires-Dist: aiohttp (>=3.8.1,<4.0.0)
 Requires-Dist: aioredis (==1.3.1)

{virtool_workflow-5.2.1.dist-info → virtool_workflow-5.3.1.dist-info}/RECORD

@@ -13,11 +13,11 @@ virtool_workflow/analysis/trimming.py,sha256=u-M7UFnJUrjxZ_vwkjuPFzIuIVRsCXpWSi5
 virtool_workflow/analysis/utils.py,sha256=sxFRlJnablszwwWfiZHbytn23D7cLRtOIC-Ivh4GlLQ,992
 virtool_workflow/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 virtool_workflow/api/analysis.py,sha256=koPVudEjnepdggcx-w9Y9mHKJStFaW3HBc6sgSVKfqs,4789
-virtool_workflow/api/client.py,sha256=3EDE6Rn7ZBC-NW5W_8oackdXOERCENzWVSCFIeNEv_E,1425
+virtool_workflow/api/client.py,sha256=1cZOQf8ZEdviM50th3Jky4vyCq3c4J2wBXsdwacKU5U,1528
 virtool_workflow/api/errors.py,sha256=jehj160u-Cy5vJR714YQ-LRlR6T4yqoJcl156MnN4Eo,2476
 virtool_workflow/api/hmm.py,sha256=OU8jc00qm-_IaFjlidu6PktDjk0Syi7MHcwCWvvAtPY,2106
 virtool_workflow/api/indexes.py,sha256=L14yfV1wZXep4HwQ1YTF9846XChuXfOcUI2jgOPUqB4,3616
-virtool_workflow/api/jobs.py,sha256=27o4lOwRvORT35siYyZPUNe08BsNe90GlliM9MeFFfc,3970
+virtool_workflow/api/jobs.py,sha256=qGTxsL4qPrHAgMCVFGfLWB2PA12pj-Nq3zXMd0ojokg,4248
 virtool_workflow/api/samples.py,sha256=NJs_P0MGmSllx3-LIN0EpYIqiGF5nedIEqynb10KL1I,3168
 virtool_workflow/api/subtractions.py,sha256=NJhNMhwkXGWBIQejBRQfReHQd7eTZl5-VSPTWjy-s5U,4016
 virtool_workflow/api/uploads.py,sha256=XjUUMVV2LkSG1AhYBps-0k4L9PeochXH3CmJTw-8Mdg,1861
@@ -41,17 +41,17 @@ virtool_workflow/runtime/executor.py,sha256=jh5s8xGnc6bQ3Cqvi_mRyauXH8pm2qCDLGA4
 virtool_workflow/runtime/hook.py,sha256=rCWADxp-nPv4vqKVhGgaxAW6eQOqPcC0rlvM-hBuh1w,4389
 virtool_workflow/runtime/providers.py,sha256=lT0gLAa6pWlu5v-wYQsxwXl0uy7gWYXK4cai3iiZgak,2504
 virtool_workflow/runtime/redis.py,sha256=l_p3LMF9BDLKUAygJZE4a9zXvX7zLj1oO1BSyL4Vxpk,1566
-virtool_workflow/runtime/run.py,sha256=BtGq3w5svbj2ZIIPIeznreqeMY9rDzhfcLGV9gMUIH0,4640
+virtool_workflow/runtime/run.py,sha256=3PZwHIS2g_J3KezXxCYreoZy7CpJwsunbcW5Sb-YUsU,5618
 virtool_workflow/runtime/run_subprocess.py,sha256=fDuKRunXBff9wGbFBgh0_-kVXzIsk39d5Bo_RPxkdHI,4636
-virtool_workflow/runtime/sentry.py,sha256=BBVijAfo_nHZyph5jSRUERz5ZvBvto--wVC9p_3CqkQ,714
+virtool_workflow/runtime/sentry.py,sha256=5FFerIiKwVhLMWrQKpK2nP8KQtO7qIT7nf2DcaOSU1Q,684
 virtool_workflow/runtime/states.py,sha256=sKNsSJ0cbU31pZVTq_GHPBiVamhx2JmCvyl5RiaGc58,203
 virtool_workflow/runtime/step.py,sha256=N1f8FP_AmXyYTRu0wxiiKYbI_S-Ehma_zxafZ7C-G5Y,2437
 virtool_workflow/runtime/utils.py,sha256=ZzIhA08PsQHUxHycSP8JnTd9b3x1fyCp8fC8NslleKs,432
 virtool_workflow/testing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 virtool_workflow/testing/fixtures.py,sha256=_UuKeUEhC6a0v4WE3gSfrplj2Rrl5GSgHjXicyBuLgs,1609
 virtool_workflow/workflow.py,sha256=S45QrnhInGulR6wezBVFJbZl3XZ351b6dLnEwmdpbE4,752
-virtool_workflow-5.2.1.dist-info/entry_points.txt,sha256=d4MA8ZDTJOU0jKZ3ymtHZbfLVoRPgItEIN5U4uIqay8,62
-virtool_workflow-5.2.1.dist-info/LICENSE,sha256=nkoVQw9W4aoQM9zgtNzHDmztap5TuXZ1L2-87vNr3w8,1097
-virtool_workflow-5.2.1.dist-info/WHEEL,sha256=vxFmldFsRN_Hx10GDvsdv1wroKq8r5Lzvjp6GZ4OO8c,88
-virtool_workflow-5.2.1.dist-info/METADATA,sha256=hochvbqB3ikwXRfIPSzfn04tPM8DRje6mDhaGIgauN0,4846
-virtool_workflow-5.2.1.dist-info/RECORD,,
+virtool_workflow-5.3.1.dist-info/LICENSE,sha256=nkoVQw9W4aoQM9zgtNzHDmztap5TuXZ1L2-87vNr3w8,1097
+virtool_workflow-5.3.1.dist-info/METADATA,sha256=GG8GuZTC5dsqrqhhcuRLAl1cZgczVSZp3Ob27fVyVLw,4774
+virtool_workflow-5.3.1.dist-info/WHEEL,sha256=Zb28QaM1gQi8f4VCBhsUklF61CTlNYfs9YAZn-TOGFk,88
+virtool_workflow-5.3.1.dist-info/entry_points.txt,sha256=d4MA8ZDTJOU0jKZ3ymtHZbfLVoRPgItEIN5U4uIqay8,62
+virtool_workflow-5.3.1.dist-info/RECORD,,

{virtool_workflow-5.2.1.dist-info → virtool_workflow-5.3.1.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.3.2
+Generator: poetry-core 1.6.1
 Root-Is-Purelib: true
 Tag: py3-none-any