seqslab-cli 3.3.5__tar.gz → 3.3.7__tar.gz

{seqslab-cli-3.3.5/python/seqslab_cli.egg-info → seqslab-cli-3.3.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: seqslab-cli
-Version: 3.3.5
+Version: 3.3.7
 Summary: Atgenomix SeqsLab Command Line Tool
 Home-page: https://github.com/AnomeGAP/seqslab-cli
 Author: Allen Chang

{seqslab-cli-3.3.5 → seqslab-cli-3.3.7}/python/seqslab/__init__.py

@@ -21,6 +21,6 @@ name = "seqslab"
 __all__ = []
 
 
-__version__ = "3.3.5"
+__version__ = "3.3.7"
 
 LOGGING = {"DIR_PATH": "/var/log/seqslab"}

{seqslab-cli-3.3.5 → seqslab-cli-3.3.7}/python/seqslab/drs/commands.py

@@ -886,52 +886,6 @@ class BaseDatahub:
 
         return results
 
-    @command(aliases=["deregister"])
-    @argument(
-        "id",
-        type=List[str],
-        positional=False,
-        description="Specify the IDs of the DRS objects that you want to delete "
-        "(optional).",
-    )
-    @argument(
-        "names",
-        type=List[str],
-        positional=False,
-        description="Specify the names of the DRS objects that you want to delete "
-        "(optional).",
-    )
-    @argument(
-        "tags",
-        type=List[str],
-        positional=False,
-        description="Specify the labels of the DRS objects that you want to delete "
-        "(optional).",
-    )
-    def deregister(
-        self,
-        id: List[str] = [],
-        tags: List[str] = [],
-        names: List[str] = [],
-    ) -> int:
-        """
-        Deregister DRS objects by DRS ID, DRS name, or DRS tag.
-        """
-        if not id and not names and not tags:
-            cprint(
-                "Must specify one of the IDs, names or tags to identify DRS objects for deletion",
-                "red",
-            )
-            return errno.ENOENT
-
-        resps = asyncio.run(
-            utils.drs_delete(id, names, tags, query_opts="?backend_content=false")
-        )
-        for r in resps:
-            cprint(r, "yellow")
-
-        return 0
-
     @command(aliases=["clean"])
     @argument(
         "id",
@@ -974,6 +928,10 @@ class BaseDatahub:
         resps = asyncio.run(
             utils.drs_delete(id, names, tags, query_opts="?backend_content=true")
         )
+        if isinstance(resps, int):
+            cprint(f"Delete with return code {resps}", "red")
+            return resps
+
         for r in resps:
             cprint(r, "yellow")
 
@@ -1090,7 +1048,9 @@ class BaseDatahub:
             fq_paths = BaseDatahub.find_fastq_paths(fastq_path, fastq_signature)
 
             if run_sheet.SampleSheet.is_single_end:
-                assert len(fq_paths) == 1
+                assert (
+                    len(fq_paths) == 1
+                ), f"{str(fq_paths)} does not contain exactly one fastq path"
                 sample_meta["Pair"] = "1"
                 upload_info.append(
                     {
@@ -1110,7 +1070,9 @@ class BaseDatahub:
                     )
 
             if run_sheet.SampleSheet.is_paired_end:
-                assert len(fq_paths) == 2
+                assert (
+                    len(fq_paths) == 2
+                ), f"{str(fq_paths)} does not contain exactly two fastq paths"
                 fq_paths.sort()
                 sample_meta["Pair"] = "1"
                 upload_info.append(
@@ -1455,6 +1417,121 @@ class BaseDatahub:
         api_backend = drs_register().load_register(workspace)
         return api_backend.get_drs(drs_id=id)
 
+    @exception_handler
+    def _patch_add_read_drs(self, add_reads_id: str, workspace: str, tag: str):
+        results = self._get(add_reads_id, workspace)
+        if isinstance(results, int):
+            return results
+        cprint(f"Found Add Reads target DRS object {add_reads_id}", color="yellow")
+        tags = [item["name"] for item in results["tags"]] + [tag]
+        patch_payload = {"tags": tags}
+        results = self._change(add_reads_id, **patch_payload)
+        if isinstance(results, int):
+            return results
+        cprint(
+            f"Add Reads target DRS object {add_reads_id} patched with tags {tags}",
+            color="yellow",
+        )
+        return add_reads_id
+
+    @command(aliases=["add-reads"])
+    @argument(
+        "workspace",
+        type=str,
+        positional=False,
+        description="Specify the workspace based on the signed in account (required).",
+    )
+    @argument(
+        "run_sheet",
+        type=str,
+        positional=False,
+        description="Specify the Run Sheet file path (required).",
+    )
+    @argument(
+        "seq_run_id",
+        type=str,
+        positional=False,
+        description="Specify a runsheet header field as a sequencer run identifier; the specified value will be "
+        "used as a sequencer run specific label for future dataset management (optional).",
+    )
+    def add_reads_runsheet(
+        self,
+        workspace,
+        run_sheet: str,
+        seq_run_id: str = "",
+    ) -> int:
+        """
+        Identify and Patch existing DRS object with add-reads runsheet label for runsheet-based run triggering for
+        DRS auto-matching
+        """
+        if not self._valide_workspace(workspace):
+            logging.error("Workspace not found")
+            cprint("Workspace not found.", "red")
+            return errno.EINVAL
+
+        # SampleSheetV2 based RunSheet SeqsLab fields
+        seqslab_section = "SeqsLabRunSheet"
+        seqslab_format = "SeqsLabColumnFormat"
+        seqslab_sep = "#"
+        rs = RunSheet(run_sheet, seqslab_section, seqslab_format, seqslab_sep)
+
+        # set base tags
+        base_tgs = ""
+        try:
+            if seq_run_id:
+                base_tgs = f"{rs.SampleSheet.Header[seq_run_id]}/"
+        except KeyError:
+            print(
+                f"Given base_label_field not found in SampleSheet Header {seq_run_id}"
+            )
+            return errno.EINVAL
+
+        add_reads = False
+        for sa in rs.SampleSheet.samples:
+            if rs.SampleSheet.is_single_end:
+                assert not sa.to_json().get("Add_Read2_ID") and not sa.to_json().get(
+                    "Add_Read2_Label"
+                ), "columns Add_Read2_ID and Add_Read2_Label should be blank for single_end sequencer run"
+                if (add_read1_id := sa.to_json().get("Add_Read1_ID")) and (
+                    add_read1_label := sa.to_json().get("Add_Read1_Label")
+                ):
+                    add_reads = True
+                    ret = self._patch_add_read_drs(
+                        add_read1_id,
+                        workspace,
+                        f"{base_tgs}{sa.get('Run_Name', rs.SampleSheet.Header.Date)}/"
+                        f"{add_read1_label}",
+                    )
+                    if isinstance(ret, int):
+                        return ret
+            if rs.SampleSheet.is_paired_end:
+                if (add_read1_id := sa.to_json().get("Add_Read1_ID")) and (
+                    add_read1_label := sa.to_json().get("Add_Read1_Label")
+                ):
+                    add_reads = True
+                    ret = self._patch_add_read_drs(
+                        add_read1_id,
+                        workspace,
+                        f"{base_tgs}{sa.get('Run_Name', rs.SampleSheet.Header.Date)}/{add_read1_label}",
+                    )
+                    if isinstance(ret, int):
+                        return ret
+                if (add_read2_id := sa.to_json().get("Add_Read2_ID")) and (
+                    add_read2_label := sa.to_json().get("Add_Read2_Label")
+                ):
+                    add_reads = True
+                    ret = self._patch_add_read_drs(
+                        add_read2_id,
+                        workspace,
+                        f"{base_tgs}{sa.get('Run_Name', rs.SampleSheet.Header.Date)}/{add_read2_label}",
+                    )
+                    if isinstance(ret, int):
+                        return ret
+
+        if not add_reads:
+            cprint("No DRS object is patched in this operation", color="yellow")
+        return 0
+
 
 @command
 class Datahub(BaseDatahub):

{seqslab-cli-3.3.5 → seqslab-cli-3.3.7}/python/seqslab/runsheet/runsheet.py

@@ -16,18 +16,25 @@ class Run:
         - ``"Run_Name"``
         - ``"Workflow_URL"``
         - ``"Runtimes"``
+        - ``"Run_Schedule_ID``
 
     A run may include multiple samples.  For samples in a Run Sheet, samples with the same Run_Name will be clustered
     as a single run.
     """
 
     def __init__(
-        self, samples: List[Sample], run_name: str, workflow_url: str, runtimes: str
+        self,
+        samples: List[Sample],
+        run_name: str,
+        workflow_url: str,
+        runtimes: str,
+        run_schedule_id: str = None,
     ) -> None:
         self.sample_sheet: Optional[SampleSheet] = None
         self.run_name = run_name
         self.workflow_url = workflow_url
         self.runtimes = runtimes
+        self.run_schedule_id = run_schedule_id
         self.samples = []
         if not workflow_url.endswith("/"):
             raise ValueError(
@@ -40,6 +47,7 @@ class Run:
                     s.get("Run_Name") == self.run_name
                     and s.get("Workflow_URL") == self.workflow_url
                     and s.get("Runtimes") == self.runtimes
+                    and s.get("Run_Schedule_ID") == self.run_schedule_id
                 ):
                     self.samples.append(s)
 
@@ -162,6 +170,7 @@ class RunSheet:
                 sample.get("Run_Name"),
                 sample.get("Workflow_URL"),
                 sample.get("Runtimes"),
+                sample.get("Run_Schedule_ID"),
             )
             rn = sample.get("Run_Name")
             if rsig in runs and rn in run_name_set:
@@ -174,7 +183,7 @@ class RunSheet:
                     f"Inconsistent run_name/run_name_set {rn}/{run_name_set} and run sig/runs {rsig}/{runs.keys()}"
                 )
         for k, v in runs.items():
-            self._runs.append(Run(v, k[0], k[1], k[2]))
+            self._runs.append(Run(v, k[0], k[1], k[2], k[3]))
 
     @property
     def runs(self) -> List:

{seqslab-cli-3.3.5 → seqslab-cli-3.3.7}/python/seqslab/trs/commands.py

@@ -408,9 +408,7 @@ class BaseTools:
         positional=False,
         description="Specify whether to force reload system cache for SCR (optional, default = False).",
     )
-    def images(
-        self, scr_id: str, repositories: List[str] = [], reload: bool = False
-    ) -> int:
+    def images(self, scr_id: str, repositories: List[str], reload: bool = False) -> int:
         """
         List image tags and details of given repositories in an SCR.
         """

{seqslab-cli-3.3.5 → seqslab-cli-3.3.7}/python/seqslab/wes/commands.py

@@ -19,13 +19,13 @@ from seqslab.auth.commands import BaseAuth
 from seqslab.exceptions import exception_handler
 from seqslab.runsheet.runsheet import Run, RunSheet
 from seqslab.trs.register.common import trs_register
-from seqslab.wes import API_HOSTNAME, __version__
-from seqslab.wes.internal import parameters
 from seqslab.workspace.internal.common import get_factory as get_workspace_factory
 from tenacity import retry, stop_after_attempt, wait_fixed
 from termcolor import cprint
 from tzlocal import get_localzone
 
+from . import API_HOSTNAME, __version__
+from .internal import parameters
 from .resource.common import get_factory
 
 """
@@ -494,38 +494,32 @@ class BaseJobs:
         request_path = f"{working_dir}/{rpath}-request.json"
         wf_info = run.workflow_url.split("versions")[1].strip("/").split("/")
 
-        if not execs:
-            trs_register().load_resource().get_execs_json(
-                workflow_url=run.workflow_url, download_path=execs_path
-            )
+        resource = get_factory().load_resource()
+        if run.run_schedule_id:
+            req = resource.get_schedule(run.run_schedule_id).get("request")
+            params = req.get("workflow_params")
+            backend_params = req.get("workflow_backend_params")
         else:
-            execs_path = f"{working_dir}/{execs}"
 
-        resource = get_factory().load_resource()
-        ops = resource.list_operator_pipelines(page=1, page_size=1000)["results"]
-        opp_w_args = [
-            op["id"]
-            for op in ops
-            for operator in op["operators"]
-            if isinstance(operator, dict) and operator.get("arguments")
-        ]
-        params = parameters.workflow_params(
-            execs_path,
-            run,
-            is_runsheet_template,
-            is_single_end,
-            fq_signature,
-            opp_w_args,
-        )
-        if not isinstance(params, dict):
-            raise Exception(
-                f"Unable to generate workflow_params based on given exec_path, with error code {params}"
+            if not execs:
+                trs_register().load_resource().get_execs_json(
+                    workflow_url=run.workflow_url, download_path=execs_path
+                )
+            else:
+                execs_path = f"{working_dir}/{execs}"
+
+            ops = resource.list_operator_pipelines(page=1, page_size=1000)["results"]
+            opp_w_args = [
+                op["id"]
+                for op in ops
+                for operator in op["operators"]
+                if isinstance(operator, dict) and operator.get("arguments")
+            ]
+            params = parameters.workflow_params(
+                execs_path,
+                opp_w_args,
             )
-
-        request = {
-            "name": run.run_name,
-            "workflow_params": params,
-            "workflow_backend_params": parameters.workflow_backend_params(
+            backend_params = parameters.workflow_backend_params(
                 execs_path,
                 workspace,
                 run.runtimes,
@@ -533,7 +527,22 @@ class BaseJobs:
                 trust,
                 kernel_version,
                 token_lifetime,
-            ),
+            )
+
+        if is_runsheet_template:
+            params = parameters.runsheet_rendering(
+                params, run, fq_signature, is_single_end
+            )
+
+        if not isinstance(params, dict):
+            raise Exception(
+                f"Unable to generate workflow_params based on given exec_path, with error code {params}"
+            )
+
+        request = {
+            "name": run.run_name,
+            "workflow_params": params,
+            "workflow_backend_params": backend_params,
             "workflow_url": run.workflow_url,
             "workflow_type_version": "1.0",
             "workflow_type": wf_info[1],

{seqslab-cli-3.3.5 → seqslab-cli-3.3.7}/python/seqslab/wes/internal/parameters.py

@@ -31,26 +31,38 @@ laws and state trade secret laws, punishable by civil and criminal penalties.
 """
 
 
-def validate_label_column(run: Run):
+def label2fqn(run: Run):
     r1fqn = set(
-        [
-            ".".join(re.sub(r"(/\d+)*$", "", sample.Read1_Label).split("/"))
-            for sample in run.samples
-        ]
+        list(
+            map(
+                lambda sample: (
+                    re.sub(r"(/\d+)*$", "", sample.Read1_Label),
+                    ".".join(re.sub(r"(/\d+)*$", "", sample.Read1_Label).split("/")),
+                ),
+                run.samples,
+            )
+        )
     )
-    assert len(r1fqn) == 1
     r2fqn = set(
-        [
-            ".".join(re.sub(r"(/\d+)*$", "", sample.Read2_Label).split("/"))
-            for sample in run.samples
-        ]
+        list(
+            map(
+                lambda sample: (
+                    re.sub(r"(/\d+)*$", "", sample.Read2_Label),
+                    ".".join(re.sub(r"(/\d+)*$", "", sample.Read2_Label).split("/")),
+                ),
+                run.samples,
+            )
+        )
     )
-    assert len(r2fqn) == 1
     if not r1fqn and not r2fqn:
         raise Exception(
             "runsheet template without Read1_Label and Read2_Label assignment"
         )
-    return r1fqn.pop(), r2fqn.pop()
+    # normalize r1fqn and r2fqn length
+    list_r2fqn = (
+        list(r2fqn) if len(r2fqn) == len(r1fqn) else [("", "") for item in r1fqn]
+    )
+    return list(r1fqn), list_r2fqn
 
 
 def is_castable_to_int(s):
@@ -113,13 +125,68 @@ def _fastq_expr(rule: str, meta: dict) -> str:
     return expr
 
 
+def validate_execs(exec_content: dict):
+    inputs = exec_content.get("inputs")
+    datasets = exec_content.get("connections")
+    for fqn in datasets:
+        if fqn not in inputs:
+            cprint(f"datasets FQN {fqn} not in inputs.json", "red")
+            return False
+    return True
+
+
+def runsheet_rendering(params: dict, run: Run, fq_sig: str, is_single_end: bool):
+    # inputs.json rendering based on runsheet info for both normal runs and add-reads runs, also reset datasets section
+    tpl_r1fqn, tpl_r2fqn = label2fqn(run)
+    for idx in range(0, len(tpl_r1fqn)):
+        r1lbpfx = tpl_r1fqn[idx][0]
+        r2lbpfx = tpl_r2fqn[idx][0]
+        lb1 = []
+        lb2 = []
+        na1 = []
+        na2 = []
+        for sa in run.samples:
+            info_dic = {k.replace(" ", "_"): v for k, v in sa.to_json().items()}
+
+            lbo1 = sa.Read1_Label
+            lba1 = sa.to_json().get("Add_Read1_Label", "")
+            if r1lbpfx in lbo1:
+                lb1.append(lbo1)
+                na1.append(f"{_fastq_expr(fq_sig, info_dic)}_r1")
+            if r1lbpfx in lba1:
+                lb1.append(lba1)
+                na1.append(f"{_fastq_expr(fq_sig, info_dic)}_org_r1")
+
+            lbo2 = sa.Read2_Label
+            lba2 = sa.to_json().get("Add_Read2_Label", "")
+            if r2lbpfx in lbo2:
+                lb2.append(lbo2)
+                na2.append(f"{_fastq_expr(fq_sig, info_dic)}_r2")
+            if r2lbpfx in lba2:
+                lb2.append(lba2)
+                na2.append(f"{_fastq_expr(fq_sig, info_dic)}_org_r2")
+
+        r1fqn = tpl_r1fqn[idx][1]
+        r2fqn = tpl_r2fqn[idx][1]
+        if r1fqn == r2fqn:
+            params["inputs"][r1fqn] = populate_fqn(lb1 + lb2, na1 + na2)
+            params["datasets"][r1fqn] = None
+            return params
+        else:
+            params["inputs"][r1fqn] = populate_fqn(lb1, na1)
+            params["datasets"][r1fqn] = None
+
+        if r2fqn:
+            assert is_single_end is False, "R2FQN should not be set for single end run"
+            params["inputs"][r2fqn] = populate_fqn(lb2, na2)
+            params["datasets"][r2fqn] = None
+
+    return params
+
+
 def workflow_params(
     execs_json: str,
-    run: Run,
-    is_runsheet_template: bool,
-    is_single_end: bool,
-    fq_sig: str,
-    opp_w_args: dict,
+    opp_w_args: list,
 ) -> dict:
     """
     Create workflow_params.json.
@@ -132,33 +199,12 @@ def workflow_params(
     try:
         with open(execs_json, "r") as f:
             t_content = json.loads(f.read())
+            if not validate_execs(t_content):
+                return errno.EINVAL
 
         params = WorkflowParamsTemplate().create(
             ex_template=t_content, opp_w_args=opp_w_args
         )
-
-        if is_runsheet_template:
-            r1fqn, r2fqn = validate_label_column(run)
-            lb1 = [sa.Read1_Label for sa in run.samples]
-            lb2 = [sa.Read2_Label for sa in run.samples]
-            na1 = [
-                f'{_fastq_expr(fq_sig, {k.replace(" ", "_"): v for k, v in sa.to_json().items()})}_r1'
-                for sa in run.samples
-            ]
-            na2 = [
-                f'{_fastq_expr(fq_sig, {k.replace(" ", "_"): v for k, v in sa.to_json().items()})}_r2'
-                for sa in run.samples
-            ]
-
-            if r1fqn == r2fqn:
-                params["inputs"][r1fqn] = populate_fqn(lb1 + lb2, na1 + na2)
-                return params
-            else:
-                params["inputs"][r1fqn] = populate_fqn(lb1, na1)
-
-            if r2fqn:
-                assert is_single_end is False
-                params["inputs"][r2fqn] = populate_fqn(lb2, na2)
         return params
     except zipfile.BadZipfile as error:
         cprint(f"{error}", "red")

{seqslab-cli-3.3.5 → seqslab-cli-3.3.7}/python/seqslab/wes/resource/base.py

@@ -56,6 +56,9 @@ class BaseResource(ABC):
         f"{WES_BASE_URL}runtime-options/{{name}}?backend={{backend}}"
     )
     WES_SCHEDULES_URL = f"{WES_BASE_URL}schedules/?backend={{backend}}"
+    WES_SCHEDULES_OBJECT_URL = (
+        f"{WES_BASE_URL}schedules/{{obj_id}}/?backend={{backend}}"
+    )
 
     class Response(NamedTuple):
         status: int
@@ -376,3 +379,15 @@ class BaseResource(ABC):
             raise err
 
         return response
+
+    def get_schedule(self, schedule_id) -> dict:
+        ctx = context.get_context()
+        backend = ctx.args.backend
+        token = BaseAuth.get_token().get("tokens").get("access")
+        with requests.get(
+            url=f"{self.WES_SCHEDULES_OBJECT_URL.format(obj_id=schedule_id, backend=backend)}",
+            headers={"Authorization": f"Bearer {token}"},
+        ) as response:
+            if response.status_code not in [requests.codes.ok]:
+                raise requests.HTTPError(response.text)
+            return json.loads(response.content)

{seqslab-cli-3.3.5 → seqslab-cli-3.3.7}/python/seqslab/wes/template/base.py

@@ -1,6 +1,10 @@
 class WorkflowParamsTemplate:
     def create(self, ex_template: dict, opp_w_args: dict) -> dict:
-        operator_pipelines = self.operator_pipelines(ex_template, opp_w_args)
+        operator_pipelines = (
+            self.operator_pipelines(ex_template, opp_w_args)
+            if "tasks" not in ex_template
+            else (ex_template["tasks"])
+        )
         return {
             "inputs": ex_template.get("inputs"),
             "datasets": ex_template.get("connections", None),

{seqslab-cli-3.3.5 → seqslab-cli-3.3.7/python/seqslab_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: seqslab-cli
-Version: 3.3.5
+Version: 3.3.7
 Summary: Atgenomix SeqsLab Command Line Tool
 Home-page: https://github.com/AnomeGAP/seqslab-cli
 Author: Allen Chang