toil 7.0.0__py3-none-any.whl → 8.0.0__py3-none-any.whl

toil/server/app.py

@@ -29,49 +29,103 @@ logger = logging.getLogger(__name__)
 def parser_with_server_options() -> argparse.ArgumentParser:
     parser = ArgumentParser(description="Toil server mode.")
 
-    parser.add_argument("--debug", action="store_true", default=False,
-                        help="Enable debug mode.")
-    parser.add_argument("--bypass_celery", action="store_true", default=False,
-                        help="Skip sending workflows to Celery and just run them under the "
-                             "server. For testing.")
-    parser.add_argument("--host", type=str, default="127.0.0.1",
-                        help="The host interface that the Toil server binds on. (default: '127.0.0.1').")
-    parser.add_argument("--port", type=int, default=8080,
-                        help="The port that the Toil server listens on. (default: 8080).")
-    parser.add_argument("--swagger_ui", action="store_true", default=False,
-                        help="If True, the swagger UI will be enabled and hosted on the "
-                             "`{api_base_path}/ui` endpoint. (default: False)")
+    parser.add_argument(
+        "--debug", action="store_true", default=False, help="Enable debug mode."
+    )
+    parser.add_argument(
+        "--bypass_celery",
+        action="store_true",
+        default=False,
+        help="Skip sending workflows to Celery and just run them under the "
+        "server. For testing.",
+    )
+    parser.add_argument(
+        "--host",
+        type=str,
+        default="127.0.0.1",
+        help="The host interface that the Toil server binds on. (default: '127.0.0.1').",
+    )
+    parser.add_argument(
+        "--port",
+        type=int,
+        default=8080,
+        help="The port that the Toil server listens on. (default: 8080).",
+    )
+    parser.add_argument(
+        "--swagger_ui",
+        action="store_true",
+        default=False,
+        help="If True, the swagger UI will be enabled and hosted on the "
+        "`{api_base_path}/ui` endpoint. (default: False)",
+    )
     # CORS
-    parser.add_argument("--cors", action="store_true", default=False,
-                        help="Enable Cross Origin Resource Sharing (CORS). This should only be turned on "
-                             "if the server is intended to be used by a website or domain.")
-    parser.add_argument("--cors_origins", type=str, default="*",
-                        help="Ignored if --cors is False. This sets the allowed origins for CORS. "
-                             "For details about CORS and its security risks, see: "
-                             "https://w3id.org/ga4gh/product-approval-support/cors. (default: '*')")
+    parser.add_argument(
+        "--cors",
+        action="store_true",
+        default=False,
+        help="Enable Cross Origin Resource Sharing (CORS). This should only be turned on "
+        "if the server is intended to be used by a website or domain.",
+    )
+    parser.add_argument(
+        "--cors_origins",
+        type=str,
+        default="*",
+        help="Ignored if --cors is False. This sets the allowed origins for CORS. "
+        "For details about CORS and its security risks, see: "
+        "https://w3id.org/ga4gh/product-approval-support/cors. (default: '*')",
+    )
     # production only
-    parser.add_argument("-w", "--workers", dest='workers', type=int, default=2,
-                        help="Ignored if --debug is True. The number of worker processes launched by the "
-                             "WSGI server. (default: 2).")
-
-    parser.add_argument("--work_dir", type=str, default=os.path.join(os.getcwd(), "workflows"),
-                        help="The directory where workflows should be stored. This directory should be "
-                             "empty or only contain previous workflows. (default: './workflows').")
-    parser.add_argument("--state_store", type=str, default=None,
-                        help="The local path or S3 URL where workflow state metadata should be stored. "
-                             "(default: in --work_dir)")
-    parser.add_argument("--opt", "-o", type=str, action="append", default=[],
-                        help="Specify the default parameters to be sent to the workflow engine for each "
-                             "run. Options taking arguments must use = syntax. Accepts multiple values.\n"
-                             "Example: '--opt=--logLevel=CRITICAL --opt=--workDir=/tmp'.")
-    parser.add_argument("--dest_bucket_base", type=str, default=None,
-                        help="Direct CWL workflows to save output files to dynamically generated "
-                             "unique paths under the given URL. Supports AWS S3.")
-    parser.add_argument("--wes_dialect", type=str, default="standard", choices=["standard", "agc"],
-                        help="Restrict WES responses to a dialect compatible with clients that do not fully "
-                             "implement the WES standard. (default: 'standard')")
-
-    parser.add_argument("--version", action='version', version=version)
+    parser.add_argument(
+        "-w",
+        "--workers",
+        dest="workers",
+        type=int,
+        default=2,
+        help="Ignored if --debug is True. The number of worker processes launched by the "
+        "WSGI server. (default: 2).",
+    )
+
+    parser.add_argument(
+        "--work_dir",
+        type=str,
+        default=os.path.join(os.getcwd(), "workflows"),
+        help="The directory where workflows should be stored. This directory should be "
+        "empty or only contain previous workflows. (default: './workflows').",
+    )
+    parser.add_argument(
+        "--state_store",
+        type=str,
+        default=None,
+        help="The local path or S3 URL where workflow state metadata should be stored. "
+        "(default: in --work_dir)",
+    )
+    parser.add_argument(
+        "--opt",
+        "-o",
+        type=str,
+        action="append",
+        default=[],
+        help="Specify the default parameters to be sent to the workflow engine for each "
+        "run. Options taking arguments must use = syntax. Accepts multiple values.\n"
+        "Example: '--opt=--logLevel=CRITICAL --opt=--workDir=/tmp'.",
+    )
+    parser.add_argument(
+        "--dest_bucket_base",
+        type=str,
+        default=None,
+        help="Direct CWL workflows to save output files to dynamically generated "
+        "unique paths under the given URL. Supports AWS S3.",
+    )
+    parser.add_argument(
+        "--wes_dialect",
+        type=str,
+        default="standard",
+        choices=["standard", "agc"],
+        help="Restrict WES responses to a dialect compatible with clients that do not fully "
+        "implement the WES standard. (default: 'standard')",
+    )
+
+    parser.add_argument("--version", action="version", version=version)
     return parser
 
 
@@ -79,34 +133,43 @@ def create_app(args: argparse.Namespace) -> "connexion.FlaskApp":
     """
     Create a "connexion.FlaskApp" instance with Toil server configurations.
     """
-    flask_app = connexion.FlaskApp(__name__,
-                                   specification_dir='api_spec/',
-                                   options={"swagger_ui": args.swagger_ui})
+    flask_app = connexion.FlaskApp(
+        __name__, specification_dir="api_spec/", options={"swagger_ui": args.swagger_ui}
+    )
 
-    flask_app.app.config['JSON_SORT_KEYS'] = False
+    flask_app.app.config["JSON_SORT_KEYS"] = False
 
     if args.cors:
         # enable cross origin resource sharing
         from flask_cors import CORS
+
         CORS(flask_app.app, resources={r"/ga4gh/*": {"origins": args.cors_origins}})
 
     # add workflow execution service (WES) API endpoints
-    backend = ToilBackend(work_dir=args.work_dir,
-                          state_store=args.state_store,
-                          options=args.opt,
-                          dest_bucket_base=args.dest_bucket_base,
-                          bypass_celery=args.bypass_celery,
-                          wes_dialect=args.wes_dialect)
-
-    flask_app.add_api('workflow_execution_service.swagger.yaml',
-                      resolver=connexion.Resolver(backend.resolve_operation_id))  # noqa
+    backend = ToilBackend(
+        work_dir=args.work_dir,
+        state_store=args.state_store,
+        options=args.opt,
+        dest_bucket_base=args.dest_bucket_base,
+        bypass_celery=args.bypass_celery,
+        wes_dialect=args.wes_dialect,
+    )
+
+    flask_app.add_api(
+        "workflow_execution_service.swagger.yaml",
+        resolver=connexion.Resolver(backend.resolve_operation_id),
+    )  # noqa
 
     # add custom endpoints
     if isinstance(backend, ToilBackend):
         # We extend the WES API to allow presenting log data
         base_url = "/toil/wes/v1"
-        flask_app.app.add_url_rule(f"{base_url}/logs/<run_id>/stdout", view_func=backend.get_stdout)
-        flask_app.app.add_url_rule(f"{base_url}/logs/<run_id>/stderr", view_func=backend.get_stderr)
+        flask_app.app.add_url_rule(
+            f"{base_url}/logs/<run_id>/stdout", view_func=backend.get_stdout
+        )
+        flask_app.app.add_url_rule(
+            f"{base_url}/logs/<run_id>/stderr", view_func=backend.get_stderr
+        )
         # To be a well-behaved AGC engine we can implement the default status check endpoint
         flask_app.app.add_url_rule("/engine/v1/status", view_func=backend.get_health)
         # And we can provide lost humans some information on what they are looking at
@@ -116,7 +179,7 @@ def create_app(args: argparse.Namespace) -> "connexion.FlaskApp":
 
 
 def start_server(args: argparse.Namespace) -> None:
-    """ Start a Toil server."""
+    """Start a Toil server."""
 
     # Explain a bit about who and where we are
     logger.info("Toil WES server version %s starting...", version)
@@ -137,7 +200,10 @@ def start_server(args: argparse.Namespace) -> None:
         flask_app.run(host=host, port=port)
     else:
         # start a production WSGI server
-        run_app(flask_app.app, options={
-            "bind": f"{host}:{port}",
-            "workers": args.workers,
-        })
+        run_app(
+            flask_app.app,
+            options={
+                "bind": f"{host}:{port}",
+                "workers": args.workers,
+            },
+        )

toil/server/celery_app.py

@@ -11,7 +11,9 @@ Also the entrypoint for starting celery workers.
 
 def create_celery_app() -> Celery:
     """ """
-    broker = os.environ.get("TOIL_WES_BROKER_URL", "amqp://guest:guest@localhost:5672//")
+    broker = os.environ.get(
+        "TOIL_WES_BROKER_URL", "amqp://guest:guest@localhost:5672//"
+    )
     app = Celery("toil_wes", broker=broker)
 
     # Celery configurations

toil/server/cli/wes_cwl_runner.py

@@ -5,8 +5,9 @@ import subprocess
 import sys
 import time
 from base64 import b64encode
+from collections.abc import Iterable
 from io import BytesIO
-from typing import Any, Dict, Iterable, List, Optional, Tuple, cast
+from typing import Any, Optional, cast
 from urllib.parse import urldefrag, urljoin, urlparse
 
 import requests
@@ -56,7 +57,7 @@ cwltest --verbose \
 logger = logging.getLogger(__name__)
 
 
-def generate_attachment_path_names(paths: List[str]) -> Tuple[str, List[str]]:
+def generate_attachment_path_names(paths: list[str]) -> tuple[str, list[str]]:
     """
     Take in a list of path names and return a list of names with the common path
     name stripped out, while preserving the input order. This guarantees that
@@ -105,7 +106,8 @@ class WESClientWithWorkflowEngineParameters(WESClient):  # type: ignore
 
     TODO: Propose a PR in wes-service to include workflow_engine_params.
     """
-    def __init__(self, endpoint: str, auth: Optional[Tuple[str, str]] = None) -> None:
+
+    def __init__(self, endpoint: str, auth: Optional[tuple[str, str]] = None) -> None:
         """
         :param endpoint: The http(s) URL of the WES server. Must include the
                          protocol.
@@ -113,12 +115,21 @@ class WESClientWithWorkflowEngineParameters(WESClient):  # type: ignore
                      request to the WES server.
         """
         proto, host = endpoint.split("://")
-        super().__init__({
-            # TODO: use the auth argument in requests.post so we don't need to encode it ourselves
-            "auth": {"Authorization": "Basic " + b64encode(f"{auth[0]}:{auth[1]}".encode()).decode("utf-8")} if auth else {},
-            "proto": proto,
-            "host": host
-        })
+        super().__init__(
+            {
+                # TODO: use the auth argument in requests.post so we don't need to encode it ourselves
+                "auth": (
+                    {
+                        "Authorization": "Basic "
+                        + b64encode(f"{auth[0]}:{auth[1]}".encode()).decode("utf-8")
+                    }
+                    if auth
+                    else {}
+                ),
+                "proto": proto,
+                "host": host,
+            }
+        )
 
     def get_version(self, extension: str, workflow_file: str) -> str:
         """Determines the version of a .py, .wdl, or .cwl file."""
@@ -140,7 +151,7 @@ class WESClientWithWorkflowEngineParameters(WESClient):  # type: ignore
         else:
             raise RuntimeError(f"Invalid workflow extension: {extension}.")
 
-    def parse_params(self, workflow_params_file: str) -> Dict[str, Any]:
+    def parse_params(self, workflow_params_file: str) -> dict[str, Any]:
         """
         Parse the CWL input file into a dictionary to be attached to the body of
         the WES run request.
@@ -155,9 +166,11 @@ class WESClientWithWorkflowEngineParameters(WESClient):  # type: ignore
         workflow_params: Any
         workflow_params, _ = loader.resolve_ref(workflow_params_file, checklinks=False)
 
-        return cast(Dict[str, Any], workflow_params)
+        return cast(dict[str, Any], workflow_params)
 
-    def modify_param_paths(self, base_dir: str, workflow_params: Dict[str, Any]) -> None:
+    def modify_param_paths(
+        self, base_dir: str, workflow_params: dict[str, Any]
+    ) -> None:
         """
         Modify the file paths in the input workflow parameters to be relative
         to base_dir.
@@ -168,7 +181,7 @@ class WESClientWithWorkflowEngineParameters(WESClient):  # type: ignore
         :param workflow_params: A dict containing the workflow parameters.
         """
 
-        def replace(field: str, file_obj: Dict[str, str]) -> None:
+        def replace(field: str, file_obj: dict[str, str]) -> None:
             """
             Given a file object with the "location" or "path" field, replace it
             to be relative to base_dir.
@@ -192,15 +205,16 @@ class WESClientWithWorkflowEngineParameters(WESClient):  # type: ignore
                     replace_paths(file.values())
                 elif isinstance(file, list):
                     replace_paths(file)
+
         replace_paths(workflow_params.values())
 
     def build_wes_request(
-            self,
-            workflow_file: str,
-            workflow_params_file: Optional[str],
-            attachments: Optional[List[str]],
-            workflow_engine_parameters: Optional[List[str]] = None
-    ) -> Tuple[Dict[str, str], Iterable[Tuple[str, Tuple[str, BytesIO]]]]:
+        self,
+        workflow_file: str,
+        workflow_params_file: Optional[str],
+        attachments: Optional[list[str]],
+        workflow_engine_parameters: Optional[list[str]] = None,
+    ) -> tuple[dict[str, str], Iterable[tuple[str, tuple[str, BytesIO]]]]:
         """
         Build the workflow run request to submit to WES.
 
@@ -233,21 +247,21 @@ class WESClientWithWorkflowEngineParameters(WESClient):  # type: ignore
 
         workflow_type = wf_url.lower().split(".")[-1]  # Grab the file extension
         workflow_type_version = self.get_version(workflow_type, wf_url)
-        data: Dict[str, str] = {
+        data: dict[str, str] = {
             "workflow_url": workflow_file,
             "workflow_params": "",  # to be set after attachments are processed
             "workflow_type": workflow_type,
-            "workflow_type_version": workflow_type_version
+            "workflow_type_version": workflow_type_version,
         }
 
         # Convert engine arguments into a JSON object
         if workflow_engine_parameters:
             params = {}
             for param in workflow_engine_parameters:
-                if '=' not in param:  # flags like "--logDebug"
+                if "=" not in param:  # flags like "--logDebug"
                     k, v = param, None
                 else:
-                    k, v = param.split('=', 1)
+                    k, v = param.split("=", 1)
                 params[k] = v
             data["workflow_engine_parameters"] = json.dumps(params)
 
@@ -278,12 +292,12 @@ class WESClientWithWorkflowEngineParameters(WESClient):  # type: ignore
         return data, [("workflow_attachment", val) for val in workflow_attachments]
 
     def run_with_engine_options(
-            self,
-            workflow_file: str,
-            workflow_params_file: Optional[str],
-            attachments: Optional[List[str]],
-            workflow_engine_parameters: Optional[List[str]]
-    ) -> Dict[str, Any]:
+        self,
+        workflow_file: str,
+        workflow_params_file: Optional[str],
+        attachments: Optional[list[str]],
+        workflow_engine_parameters: Optional[list[str]],
+    ) -> dict[str, Any]:
         """
         Composes and sends a post request that signals the WES server to run a
         workflow.
@@ -297,10 +311,9 @@ class WESClientWithWorkflowEngineParameters(WESClient):  # type: ignore
 
         :return: The body of the post result as a dictionary.
         """
-        data, files = self.build_wes_request(workflow_file,
-                                             workflow_params_file,
-                                             attachments,
-                                             workflow_engine_parameters)
+        data, files = self.build_wes_request(
+            workflow_file, workflow_params_file, attachments, workflow_engine_parameters
+        )
         post_result = requests.post(
             urljoin(f"{self.proto}://{self.host}", "/ga4gh/wes/v1/runs"),
             data=data,
@@ -308,10 +321,10 @@ class WESClientWithWorkflowEngineParameters(WESClient):  # type: ignore
             headers=self.auth,
         )
 
-        return cast(Dict[str, Any], wes_response(post_result))
+        return cast(dict[str, Any], wes_response(post_result))
 
 
-def get_deps_from_cwltool(cwl_file: str, input_file: Optional[str] = None) -> List[str]:
+def get_deps_from_cwltool(cwl_file: str, input_file: Optional[str] = None) -> list[str]:
     """
     Return a list of dependencies of the given workflow from cwltool.
 
@@ -320,19 +333,21 @@ def get_deps_from_cwltool(cwl_file: str, input_file: Optional[str] = None) -> Li
                        this returns the dependencies from the input file.
     """
 
-    option = '--print-input-deps' if input_file else '--print-deps'
+    option = "--print-input-deps" if input_file else "--print-deps"
 
-    args = ['cwltool', option, '--relative-deps', 'cwd', cwl_file]
+    args = ["cwltool", option, "--relative-deps", "cwd", cwl_file]
     if input_file:
         args.append(input_file)
 
-    p = subprocess.run(args=args, check=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
+    p = subprocess.run(
+        args=args, check=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL
+    )
 
     result = p.stdout.decode()
     if not result:
         return []
 
-    json_result: Dict[str, Any] = json.loads(result)
+    json_result: dict[str, Any] = json.loads(result)
     deps = []
 
     def get_deps(obj: Any) -> None:
@@ -368,10 +383,12 @@ def get_deps_from_cwltool(cwl_file: str, input_file: Optional[str] = None) -> Li
     return deps
 
 
-def submit_run(client: WESClientWithWorkflowEngineParameters,
-               cwl_file: str,
-               input_file: Optional[str] = None,
-               engine_options: Optional[List[str]] = None) -> str:
+def submit_run(
+    client: WESClientWithWorkflowEngineParameters,
+    cwl_file: str,
+    input_file: Optional[str] = None,
+    engine_options: Optional[list[str]] = None,
+) -> str:
     """
     Given a CWL file, its input files, and an optional list of engine options,
     submit the CWL workflow to the WES server via the WES client.
@@ -391,23 +408,32 @@ def submit_run(client: WESClientWithWorkflowEngineParameters,
     if input_file:
         attachments.extend(get_deps_from_cwltool(cwl_file, input_file))
 
-    run_result: Dict[str, Any] = client.run_with_engine_options(
+    run_result: dict[str, Any] = client.run_with_engine_options(
         cwl_file,
         input_file,
         attachments=attachments,
-        workflow_engine_parameters=engine_options)
+        workflow_engine_parameters=engine_options,
+    )
     return str(run_result["run_id"])
 
 
 def poll_run(client: WESClientWithWorkflowEngineParameters, run_id: str) -> bool:
-    """ Return True if the given workflow run is in a finished state."""
+    """Return True if the given workflow run is in a finished state."""
     status_result = client.get_run_status(run_id)
     state = status_result.get("state")
 
-    return state in ("COMPLETE", "CANCELING", "CANCELED", "EXECUTOR_ERROR", "SYSTEM_ERROR")
+    return state in (
+        "COMPLETE",
+        "CANCELING",
+        "CANCELED",
+        "EXECUTOR_ERROR",
+        "SYSTEM_ERROR",
+    )
 
 
-def print_logs_and_exit(client: WESClientWithWorkflowEngineParameters, run_id: str) -> None:
+def print_logs_and_exit(
+    client: WESClientWithWorkflowEngineParameters, run_id: str
+) -> None:
     """
     Fetch the workflow logs from the WES server, print the results, then exit
     the program with the same exit code as the workflow run.
@@ -431,9 +457,11 @@ def main() -> None:
     parser.add_argument("cwl_file", type=str)
     parser.add_argument("input_file", type=str, nargs="?", default=None)
     # arguments used by the WES runner
-    parser.add_argument("--wes_endpoint",
-                        default=os.environ.get("TOIL_WES_ENDPOINT", "http://localhost:8080"),
-                        help="The http(s) URL of the WES server.  (default: %(default)s)")
+    parser.add_argument(
+        "--wes_endpoint",
+        default=os.environ.get("TOIL_WES_ENDPOINT", "http://localhost:8080"),
+        help="The http(s) URL of the WES server.  (default: %(default)s)",
+    )
     # the rest of the arguments are passed as engine options to the WES server
     options, rest = parser.parse_known_args()
 
@@ -449,7 +477,8 @@ def main() -> None:
 
     client = WESClientWithWorkflowEngineParameters(
         endpoint=endpoint,
-        auth=(wes_user, wes_password) if wes_user and wes_password else None)
+        auth=(wes_user, wes_password) if wes_user and wes_password else None,
+    )
 
     run_id = submit_run(client, cwl_file, input_file, engine_options=rest)
     assert run_id
@@ -462,5 +491,5 @@ def main() -> None:
     print_logs_and_exit(client, run_id)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()