seqslab-cli 3.3.2.post1__tar.gz → 3.3.3__tar.gz

{seqslab-cli-3.3.2.post1/python/seqslab_cli.egg-info → seqslab-cli-3.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: seqslab-cli
-Version: 3.3.2.post1
+Version: 3.3.3
 Summary: Atgenomix SeqsLab Command Line Tool
 Home-page: https://github.com/AnomeGAP/seqslab-cli
 Author: Allen Chang

{seqslab-cli-3.3.2.post1 → seqslab-cli-3.3.3}/python/seqslab/__init__.py

@@ -21,6 +21,6 @@ name = "seqslab"
 __all__ = []
 
 
-__version__ = "3.3.2.post1"
+__version__ = "3.3.3"
 
 LOGGING = {"DIR_PATH": "/var/log/seqslab"}

{seqslab-cli-3.3.2.post1 → seqslab-cli-3.3.3}/python/seqslab/drs/api/azure.py

@@ -8,6 +8,7 @@ from typing import List
 import requests
 from nubia import context
 from seqslab.auth.commands import BaseAuth
+from seqslab.drs.storage.azure import BlobStorage
 from seqslab.drs.utils.biomimetype import get_mime_type
 from tenacity import retry, stop_after_attempt, wait_fixed
 from termcolor import cprint
@@ -396,3 +397,14 @@ class AzureDRSregister(DRSregister):
                 )
 
         return dsts_list
+
+    def common_root(self, dsts: List[URL], **kwargs) -> str:
+        root_dst = super().common_root(dsts)
+        blob_root = BlobStorage(kwargs.get("workspace")).get_token(path=None)[
+            "register_url"
+        ]
+        if URL(root_dst).path == URL(blob_root).path:
+            raise ValueError(
+                "access_url pointing to cloud storage root path is not allowed"
+            )
+        return root_dst

{seqslab-cli-3.3.2.post1 → seqslab-cli-3.3.3}/python/seqslab/drs/api/base.py

@@ -335,7 +335,7 @@ class DRSregister:
                 folder_stdin["access_methods"].append(access_method_infos[i])
                 folder_stdin["access_methods"][i]["access_url"][
                     "url"
-                ] = self.common_root(dsts=dsts)
+                ] = self.common_root(dsts=dsts, workspace=kwargs.get("workspace"))
 
         folder_stdin["name"] = kwargs.get(
             "name",
@@ -360,7 +360,7 @@ class DRSregister:
             raise ValueError("Make sure all the sizes are filled.")
         return sum(sizes)
 
-    def common_root(self, dsts: List[URL]) -> str:
+    def common_root(self, dsts: List[URL], **kwargs) -> str:
         """
         e.g. http://user:password@example.com:8042/over/there?name=ferret#nose
         URL.scheme: http

{seqslab-cli-3.3.2.post1 → seqslab-cli-3.3.3}/python/seqslab/drs/commands.py

@@ -173,7 +173,7 @@ class BaseDatahub:
         "dst",
         type=str,
         positional=False,
-        description="Specify the destination directory, file path, or DRS URL (optional, default =' /'). ",
+        description="Specify the destination directory, file path, or DRS URL (optional, default ='MyDir/'). ",
     )
     @argument(
         "recursive",
@@ -223,7 +223,7 @@ class BaseDatahub:
         self,
         workspace: str,
         src: str,
-        dst: str = "/",
+        dst: str = "MyDir/",
         recursive: bool = False,
         output: str = "json",
         concurrency: int = 0,
@@ -706,7 +706,7 @@ class BaseDatahub:
         backend = drs_register().load_register(workspace)
         if stdin:
             payloads = backend.create_payload(
-                stdin=BaseDatahub._stdin(), type=drs_type, **kwargs
+                stdin=BaseDatahub._stdin(), type=drs_type, workspace=workspace, **kwargs
             )
         else:
             if not kwargs["is_integrity"]:

{seqslab-cli-3.3.2.post1 → seqslab-cli-3.3.3}/python/seqslab/trs/template/base.py

@@ -1,4 +1,5 @@
 # Standard Library
+import re
 from typing import List
 
 import pydot
@@ -21,7 +22,7 @@ class TrsCreateTemplate:
             primary_descriptor=primary_descriptor, zip_file=zip_file
         )
         workflow = self.desc_template(parameter)
-        calls = self.call_template(parameter)
+        calls = self.workflow_call_template(parameter)
         inputs_connections = self.inputs_connections(
             parameter=parameter, inputs_json=inputs_json
         )
@@ -58,6 +59,32 @@ class TrsCreateTemplate:
                     calls.add(node.get_name().replace("CALL_", ""))
         return calls
 
+    def workflow_call_template(self, parameter: dict) -> list:
+        # adding main workflow
+        runtime_cell_select = [
+            pydot.graph_from_dot_data(
+                TrsCreateTemplate.dot_cleaning(parameter["graph"])
+            )[0].get_name()
+        ]
+
+        # parsing workflow>call information from graph string
+        graphRex = re.compile(r'graph\s\w+|}|label="call\s\w+', re.IGNORECASE)
+        wnames = []
+        matches = graphRex.findall(parameter.get("graph"))
+        if matches:
+            for e in matches:
+                if e.startswith("graph "):
+                    wnames.append(e.split(" ")[1].replace("cluster_", ""))
+                elif e.startswith("}"):
+                    wnames.pop()
+                elif e.startswith("label="):
+                    call = e.replace('label="', "").strip('"')
+                    if call.lower().startswith("call"):
+                        runtime_cell_select.append(
+                            "" + wnames[-1] + ">" + call.replace("call ", "")
+                        )
+        return runtime_cell_select
+
     def call_template(self, parameter: dict) -> list:
         calls = [
             pydot.graph_from_dot_data(

{seqslab-cli-3.3.2.post1 → seqslab-cli-3.3.3}/python/seqslab/user/resource/base.py

@@ -112,7 +112,9 @@ class BaseResource:
     def is_global_admin(self, **kwargs) -> bool:
         token = BaseAuth.get_token()
         response = self.get_user(token.get("attrs").get("user_id"))
-        if "Global administrator" in response.get("roles"):
+        if "Global administrator" in [
+            item.get("name") for item in response.get("roles")
+        ]:
             return True
 
         return False

{seqslab-cli-3.3.2.post1 → seqslab-cli-3.3.3}/python/seqslab/wes/commands.py

@@ -458,11 +458,11 @@ class BaseJobs:
     @argument(
         "runtimes",
         type=str,
-        description="Key:value pairs indicating the workflow name -> SeqsLab supported runtime_options names. "
-        "Multiple configuration pairs can be provided using ':' as separator, "
-        "e.g. main=m4-cluster:subworkflow=m4-8xcluster "
-        "(optional, default = None, which indicates running the whole workflow.wdl using m4-cluster for "
-        "a single node cluster on the Azure backend).",
+        description="String of key-value pairs using : as a separator, indicating the execution runtimes for each task "
+        "or workflow defined in workflow-url. For example, Main=m4-cluster:Main>Fq2Bam=m4-8xcluster. To "
+        "list tasks/workflows for a given workflow, use the execs command. To find available runtime "
+        "options, use the request runtimes_options command. (Optional, defaults to None, which runs the "
+        "entire workflow using m4-cluster.)",
     )
     @argument(
         "integrity",
@@ -661,6 +661,101 @@ class BaseJobs:
 
         return 0
 
+    @command(aliases=["rt"])
+    @argument(
+        "page",
+        type=int,
+        positional=False,
+        description="Specify the page number in the set of paginated records (optional, default = 1).",
+    )
+    @argument(
+        "page_size",
+        type=int,
+        positional=False,
+        description="Specify the number of records to return in each page (optional, default = 10).",
+    )
+    @command
+    @argument(
+        "output",
+        type=str,
+        positional=False,
+        description="Specify the output format of the stdout file (optional, default = json).",
+        choices=["json", "table"],
+    )
+    def runtime_options(
+        self, workspace: str, page: int = 1, page_size: int = 10, output="json"
+    ) -> int:
+        """
+        List registered cluster runtimes settings.
+        """
+        resource = get_factory().load_resource(workspace)
+        r = resource.list_runtime_options(page=page, page_size=page_size)
+
+        if isinstance(r, int):
+            return r
+
+        self._stdout(r["results"], output)
+        return 0
+
+    @command(aliases=["op"])
+    @argument(
+        "page",
+        type=int,
+        positional=False,
+        description="Specify the page number in the set of paginated records (optional, default = 1).",
+    )
+    @argument(
+        "page_size",
+        type=int,
+        positional=False,
+        description="Specify the number of records to return in each page (optional, default = 10).",
+    )
+    @command
+    @argument(
+        "output",
+        type=str,
+        positional=False,
+        description="Specify the output format of the stdout file (optional, default = json).",
+        choices=["json", "table"],
+    )
+    def operator_pipelines(
+        self, workspace: str, page: int = 1, page_size: int = 10, output="json"
+    ) -> int:
+        """
+        List registered operator pipelines.
+        """
+        resource = get_factory().load_resource(workspace)
+        r = resource.list_runtime_options(page=page, page_size=page_size)
+
+        if isinstance(r, int):
+            return r
+
+        self._stdout(r["results"], output)
+        return 0
+
+    @staticmethod
+    def _stdout(results, output: str) -> int:
+        # Standard Library
+        import csv
+        from io import StringIO
+
+        from tabulate import tabulate
+
+        """
+            stdout:: TODO: support different format ex: json, tsv, table
+        """
+        if output == "json":
+            cprint(json.dumps(results, indent=4))
+        elif output == "table":
+            table_header = list(results[0].keys())
+            table_datas = [result.values() for result in results]
+            cprint(
+                tabulate(
+                    tabular_data=table_datas, headers=table_header, tablefmt="pipe"
+                )
+            )
+        return 0
+
 
 @command
 class Jobs(BaseJobs):

{seqslab-cli-3.3.2.post1 → seqslab-cli-3.3.3}/python/seqslab/wes/internal/parameters.py

@@ -230,13 +230,33 @@ def workflow_backend_params(
             raise RuntimeError(
                 f"given call name {k} not in TRS registered call name list {calls}!"
             )
-        clusters.append(
-            WorkflowBackendParamsClusterTemplate(
-                run_time=resource.get_runtime_setting(v),
-                workflow_name=k,
-                kernel_version=kernel_version,
+
+        # general workflow>call handling
+        ids = k.split(">")
+        if len(ids) == 2:
+            clusters.append(
+                WorkflowBackendParamsClusterTemplate(
+                    run_time=resource.get_runtime_setting(v),
+                    call_name=ids[1],
+                    workflow_name=ids[0],
+                    kernel_version=kernel_version,
+                )
             )
-        )
+        else:
+            # main workflow name
+            if k == primary_workflow_name:
+                clusters.append(
+                    WorkflowBackendParamsClusterTemplate(
+                        run_time=resource.get_runtime_setting(v),
+                        call_name=k,
+                        kernel_version=kernel_version,
+                    )
+                )
+            else:
+                # raise exception for incorrect format of k
+                raise RuntimeError(
+                    f"given identifier {k} does not in format of in TRS registered call name list {calls}!"
+                )
 
     bk_template = WorkflowBackendParamsTemplate(
         graph=execs.get("graph"),

{seqslab-cli-3.3.2.post1 → seqslab-cli-3.3.3}/python/seqslab/wes/resource/base.py

@@ -50,6 +50,8 @@ class BaseResource(ABC):
     WES_RUNS_DRY_URL = f"{WES_BASE_URL}runs/dryrun/?backend={{backend}}"
     WES_RUNS_FILE_URL = f"{WES_BASE_URL}runs/{{id}}/files/?backend={{backend}}"
     WES_RUNS_STATUS_URL = f"{WES_BASE_URL}runs/{{id}}/status/?backend={{backend}}"
+    WES_RUNTIME_OPTIONS_BASE_URL = f"{WES_BASE_URL}runtime-options/"
+    WES_OPERATOR_PIPELINES_BASE_URL = f"{WES_BASE_URL}operator-pipelines/"
     WES_RUNTIME_OPTIONS_URL = (
         f"{WES_BASE_URL}runtime-options/{{name}}?backend={{backend}}"
     )
@@ -310,3 +312,35 @@ class BaseResource(ABC):
             if response.status_code not in [requests.codes.ok]:
                 raise requests.HTTPError(response.text)
             return json.loads(response.content)
+
+    def list_runtime_options(self, page=1, page_size=10):
+        try:
+            token = BaseAuth.get_token().get("tokens").get("access")
+            with requests.get(
+                url=f"{self.WES_RUNTIME_OPTIONS_BASE_URL}?page={page}&page_size={page_size}",
+                headers={"Authorization": f"Bearer {token}"},
+            ) as response:
+                if response.status_code not in [requests.codes.ok]:
+                    raise requests.HTTPError(
+                        f"{response.status_code}: {repr(response.text)}"
+                    )
+        except Exception as err:
+            print(err)
+            raise err
+        return response.json()
+
+    def list_operator_pipelines(self, page=1, page_size=10):
+        try:
+            token = BaseAuth.get_token().get("tokens").get("access")
+            with requests.get(
+                url=f"{self.WES_OPERATOR_PIPELINES_BASE_URL}?page={page}&page_size={page_size}",
+                headers={"Authorization": f"Bearer {token}"},
+            ) as response:
+                if response.status_code not in [requests.codes.ok]:
+                    raise requests.HTTPError(
+                        f"{response.status_code}: {repr(response.text)}"
+                    )
+        except Exception as err:
+            print(err)
+            raise err
+        return response.json()

{seqslab-cli-3.3.2.post1 → seqslab-cli-3.3.3}/python/seqslab/wes/template/base.py

@@ -34,6 +34,8 @@ class WorkflowParamsTemplate:
         tasks = {}
         pl_keys = [pipeline["id"] for pipeline in ex_template["operator_pipelines"]]
         for k, v in dict(ex_template["i_configs"], **ex_template["o_configs"]).items():
+            if not v:
+                continue
             assert (
                 v in pl_keys
             ), f"given operator pipeline ID {v} for FQN {k} not in operator pipeline list from execs: {pl_keys}"
@@ -113,11 +115,14 @@ def WorkflowBackendParamsTemplate(
 
 
 def WorkflowBackendParamsClusterTemplate(
-    run_time: dict, workflow_name: str, kernel_version: str
+    run_time: dict, call_name: str, kernel_version: str, workflow_name: str = ""
 ) -> dict:
     if kernel_version:
         opts = run_time["options"]
         opts.append(f"seqslab.kernel.version {kernel_version}")
         run_time.update({"options": opts})
-    run_time.update({"call": workflow_name})
+    if workflow_name:
+        run_time.update({"call": call_name, "workflow": workflow_name})
+    else:
+        run_time.update({"call": call_name})
     return run_time

{seqslab-cli-3.3.2.post1 → seqslab-cli-3.3.3/python/seqslab_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: seqslab-cli
-Version: 3.3.2.post1
+Version: 3.3.3
 Summary: Atgenomix SeqsLab Command Line Tool
 Home-page: https://github.com/AnomeGAP/seqslab-cli
 Author: Allen Chang

{seqslab-cli-3.3.2.post1 → seqslab-cli-3.3.3}/python/seqslab_cli.egg-info/requires.txt

@@ -2,7 +2,7 @@ pip>=22.0.4
 click==8.1.7
 tabulate==0.9.0
 terminaltables==3.1.10
-cryptography==41.0.7
+cryptography==42.0.6
 jeepney==0.8.0
 secretstorage==3.3.3
 dbus-python==1.2.16
@@ -12,7 +12,7 @@ python-nubia==0.2b5
 msal==1.26.0
 tenacity==8.2.3
 aiofiles==23.2.1
-aiohttp[speedups]==3.9.1
+aiohttp[speedups]==3.9.5
 aioretry==5.0.2
 uvloop==0.19.0
 arrow==1.3.0
@@ -22,9 +22,9 @@ PyJWT==2.8.0
 django-environ==0.11.2
 validators==0.22.0
 pydot==2.0.0
-orjson==3.9.10
-pydantic~=2.5.3
+orjson==3.10.3
+pydantic~=2.7.1
 aiohttp_retry==2.8.3
 requests~=2.31.0
-setuptools~=69.0.3
+setuptools~=69.5.1
 jsonpath-ng==1.6.0

{seqslab-cli-3.3.2.post1 → seqslab-cli-3.3.3}/requirements.txt

@@ -1,4 +1,4 @@
-cryptography==41.0.7
+cryptography==42.0.6
 jeepney==0.8.0
 secretstorage==3.3.3
 dbus-python==1.2.16
@@ -8,7 +8,7 @@ python-nubia==0.2b5
 msal==1.26.0
 tenacity==8.2.3
 aiofiles==23.2.1
-aiohttp[speedups]==3.9.1
+aiohttp[speedups]==3.9.5
 aioretry==5.0.2
 uvloop==0.19.0
 arrow==1.3.0
@@ -18,9 +18,9 @@ PyJWT==2.8.0
 django-environ==0.11.2
 validators==0.22.0
 pydot==2.0.0
-orjson==3.9.10
-pydantic~=2.5.3
+orjson==3.10.3
+pydantic~=2.7.1
 aiohttp_retry==2.8.3
 requests~=2.31.0
-setuptools~=69.0.3
+setuptools~=69.5.1
 jsonpath-ng==1.6.0