toil 8.0.0__py3-none-any.whl → 8.2.0__py3-none-any.whl

toil/server/cli/wes_cwl_runner.py

@@ -10,13 +10,13 @@ from io import BytesIO
 from typing import Any, Optional, cast
 from urllib.parse import urldefrag, urljoin, urlparse
 
-import requests
 import ruamel.yaml
 import schema_salad
 from configargparse import ArgumentParser
 from wes_client.util import WESClient  # type: ignore
 from wes_client.util import wes_reponse as wes_response
 
+from toil.lib.web import web_session
 from toil.wdl.utils import get_version as get_wdl_version
 
 """
@@ -117,7 +117,7 @@ class WESClientWithWorkflowEngineParameters(WESClient):  # type: ignore
         proto, host = endpoint.split("://")
         super().__init__(
             {
-                # TODO: use the auth argument in requests.post so we don't need to encode it ourselves
+                # TODO: use the auth argument in requests' post so we don't need to encode it ourselves
                 "auth": (
                     {
                         "Authorization": "Basic "
@@ -144,7 +144,8 @@ class WESClientWithWorkflowEngineParameters(WESClient):  # type: ignore
             return "3.8"
         elif extension == "cwl":
             with open(workflow_file) as f:
-                return str(ruamel.yaml.safe_load(f)["cwlVersion"])
+                yaml = ruamel.yaml.YAML(typ='safe', pure=True)
+                return str(yaml.load(f)["cwlVersion"])
         elif extension == "wdl":
             with open(workflow_file) as f:
                 return get_wdl_version(f)
@@ -314,7 +315,7 @@ class WESClientWithWorkflowEngineParameters(WESClient):  # type: ignore
         data, files = self.build_wes_request(
             workflow_file, workflow_params_file, attachments, workflow_engine_parameters
         )
-        post_result = requests.post(
+        post_result = web_session.post(
             urljoin(f"{self.proto}://{self.host}", "/ga4gh/wes/v1/runs"),
             data=data,
             files=files,

toil/server/utils.py

@@ -19,10 +19,9 @@ from datetime import datetime
 from typing import Optional
 from urllib.parse import urlparse
 
-import requests
-
 from toil.lib.io import AtomicFileCreate
 from toil.lib.retry import retry
+from toil.lib.web import web_session
 
 try:
     from toil.lib.aws import get_current_aws_region
@@ -63,7 +62,7 @@ def download_file_from_internet(
     """
     Download a file from the Internet and write it to dest.
     """
-    response = requests.get(src)
+    response = web_session.get(src)
 
     if not response.ok:
         raise RuntimeError("Request failed with a client error or a server error.")

toil/statsAndLogging.py

@@ -24,6 +24,7 @@ from typing import IO, TYPE_CHECKING, Any, Callable, Optional, Union
 
 from toil.lib.conversions import strtobool
 from toil.lib.expando import Expando
+from toil.lib.history import HistoryManager
 from toil.lib.resources import ResourceMonitor
 
 if TYPE_CHECKING:
@@ -168,6 +169,7 @@ class StatsAndLogging:
         The following function is used for collating stats/reporting log messages from the workers.
         Works inside of a thread, collates as long as the stop flag is not True.
         """
+
         #  Overall timing
         startTime = time.time()
         startClock = ResourceMonitor.get_total_cpu_time()
@@ -231,8 +233,40 @@ class StatsAndLogging:
                 )
                 cls.writeLogFiles(jobNames, messages, config=config)
 
+            try:
+                jobs = stats.jobs
+            except AttributeError:
+                pass
+            else:
+                for job in jobs:
+                    try:
+                        # Here we're talking to job._executor which fills in these stats.
+
+                        # Convince MyPy we won't be sent any job stats without
+                        # a workflow ID. You can't set up the job store without
+                        # one, but if we're somehow missing one, keep the stats
+                        # and logging thread up.
+                        assert config.workflowID is not None
+
+                        # TODO: Use better job names!
+                        HistoryManager.record_job_attempt(
+                            config.workflowID,
+                            config.workflowAttemptNumber,
+                            job.class_name,
+                            job.succeeded == "True",
+                            float(job.start),
+                            float(job.time),
+                            cores=float(job.requested_cores),
+                            cpu_seconds=float(job.clock),
+                            memory_bytes=int(job.memory) * 1024,
+                            disk_bytes=int(job.disk)
+                        )
+                    except:
+                        logger.exception("Could not record job attempt in history!")
+                        # Keep going. Don't fail the workflow for history-related issues.
+
         while True:
-            # This is a indirect way of getting a message to the thread to exit
+            # This is an indirect way of getting a message to the thread to exit
             if stop.is_set():
                 jobStore.read_logs(callback)
                 break