pybioos 0.0.5__py3-none-any.whl → 0.0.7__py3-none-any.whl

bioos/__about__.py

@@ -1,4 +1,4 @@
 # coding:utf-8
 
 # Package version
-__version__ = "0.0.5"
+__version__ = "0.0.7"

bioos/bioos_workflow.py

@@ -2,12 +2,13 @@ import argparse
 import json
 import logging
 import os
+import re
 import time
 
 import pandas as pd
 
 from bioos import bioos
-from bioos.errors import NotFoundError
+from bioos.errors import NotFoundError, ParameterError
 
 
 def recognize_files_from_input_json(workflow_input_json: dict) -> dict:
@@ -159,6 +160,100 @@ class Bioos_workflow:
 
         return self.params_submit
 
+    def preprocess2(self,
+                    input_json_file: str,
+                    data_model_name: str = "dm",
+                    submission_desc: str = "Submit by pybioos",
+                    call_caching: bool = True,
+                    force_reupload: bool = False):
+        if not os.path.isfile(input_json_file):
+            raise ParameterError('Input_json_file is not found.')
+
+        input_json = json.load(open(input_json_file))
+        self.logger.info("Load json input successfully.")
+
+        # putative files
+        input_json_str = json.dumps(input_json)
+
+        # capture strings containing "/" the test if the file exists
+        putative_files = [
+            s.strip('"\'') for s in re.findall(
+                r'''"[-_\w./:]+?/[-_\w./:]+?"''', input_json_str)
+            if os.path.isfile(s.strip('"\''))
+        ]
+
+        putative_files = set(putative_files)
+        file_str = ''
+        for putative_file in putative_files:
+            file_str = file_str + '\t' + putative_file + '\n'
+
+        self.logger.info(
+            f"Putative files need to upload includes:\n{file_str}")
+
+        # provision upload and file path replace
+        df = self.ws.files.list('input_provision')
+        uploaded_files = [] if df.empty else df.key.to_list()
+        for putative_file in putative_files:
+            target = f"input_provision/{os.path.basename(putative_file)}"
+
+            if not force_reupload and target in uploaded_files:
+                self.logger.info(
+                    f"Skip target site already existed file {putative_file}.")
+            else:
+                self.logger.info(f"Start upload {putative_file}.")
+                self.ws.files.upload(putative_file,
+                                     target="input_provision/",
+                                     flatten=True)
+                self.logger.info(f"Finish upload {putative_file}.")
+            s3_location = self.ws.files.s3_urls(target)[0]
+            input_json_str = re.sub(putative_file, s3_location, input_json_str)
+
+        # start build params_submit
+        self.params_submit = {
+            "outputs": "{}",
+            "submission_desc": submission_desc,
+            "call_caching": call_caching,
+        }
+
+        # if the input json is a batch or singleton submission
+        input_json = json.loads(input_json_str)
+        if isinstance(input_json, list):  # batch mode
+            self.logger.info("Batch mode found.")
+
+            # build data model for batch mode
+            inputs_list = input_json
+            df = pd.DataFrame(inputs_list)
+            id_col = f"{data_model_name}_id"
+            columns = [
+                id_col,
+            ]
+            columns.extend(df.columns)
+            df[id_col] = [f"tmp_{x}" for x in list(range(len(df)))]
+            df = df.reindex(columns=columns)
+            columns = [key.split(".")[-1] for key in df.columns.to_list()]
+            df.columns = pd.Index(columns)
+
+            # write data models
+            self.ws.data_models.write({data_model_name: df.map(str)},
+                                      force=True)
+            self.logger.info("Set data model successfully.")
+
+            # match the batch sytax of Bio-OS
+            unupdate_dict = inputs_list[0]
+            for key, _ in unupdate_dict.items():
+                unupdate_dict[key] = f'this.{key.split(".")[-1]}'
+
+            self.params_submit["inputs"] = json.dumps(unupdate_dict)
+            self.params_submit["data_model_name"] = data_model_name
+            self.params_submit["row_ids"] = df[id_col].to_list()
+
+        else:  # singleton mode
+            self.logger.info("Singleton mode found.")
+            self.params_submit["inputs"] = json.dumps(input_json)
+
+        self.logger.info("Build params dict successfully.")
+        return self.params_submit
+
     def postprocess(self, download=False):
         # 假设全部执行完毕
         #  对运行完成的目录进行下载
@@ -266,11 +361,11 @@ def bioos_workflow():
                 secret_key=parsed_args.sk)
     bw = Bioos_workflow(workspace_name=parsed_args.workspace_name,
                         workflow_name=parsed_args.workflow_name)
-    bw.preprocess(input_json_file=parsed_args.input_json,
-                  data_model_name=parsed_args.data_model_name,
-                  submission_desc=parsed_args.submission_desc,
-                  call_caching=parsed_args.call_caching,
-                  force_reupload=parsed_args.force_reupload)
+    bw.preprocess2(input_json_file=parsed_args.input_json,
+                   data_model_name=parsed_args.data_model_name,
+                   submission_desc=parsed_args.submission_desc,
+                   call_caching=parsed_args.call_caching,
+                   force_reupload=parsed_args.force_reupload)
     bw.submit_workflow_bioosapi()
 
     # moniter
@@ -291,4 +386,9 @@ def bioos_workflow():
             bw.monitor_workflow()
 
         time.sleep(60)
+        bw.logger.info("Submission finished. Print final status for runs.")
+        print(bw.runs)
+
+        bw.logger.info("Start to postprocess.")
         bw.postprocess(download=parsed_args.download_results)
+        bw.logger.info("Postprocess finished.")

bioos/resource/workflows.py

@@ -292,10 +292,11 @@ class Submission(metaclass=SingletonType):  # 与run class行为相同
             'WorkspaceID':
             self.workspace_id,
         }).get("Items")
-        for model in models:
-            if model["ID"] == item["DataModelID"]:
-                self.data_model = model.get("Name")
-                break
+        if "DataModelID" in item.keys():
+            for model in models:
+                if model["ID"] == item["DataModelID"]:
+                    self.data_model = model.get("Name")
+                    break
 
         self.call_cache = item.get("ExposedOptions").get("ReadFromCache")
         self.outputs = item.get("Outputs")
@@ -513,9 +514,14 @@ class Workflow(metaclass=SingletonType):
             return ""
         return res["ID"].iloc[0]
 
-    def submit(self, data_model_name: str, row_ids: List[str], inputs: str, outputs: str,
-               submission_desc: str, call_caching: bool, submission_name_suffix: str = "") \
-            -> List[Run]:
+    def submit(self,
+               inputs: str,
+               outputs: str,
+               submission_desc: str,
+               call_caching: bool,
+               submission_name_suffix: str = "",
+               row_ids: List[str] = [],
+               data_model_name: str = '') -> List[Run]:
         """Submit an existed workflow.
 
         *Example*:
@@ -547,44 +553,40 @@ class Workflow(metaclass=SingletonType):
         :return: Result Runs corresponding to submitted workflows
         :rtype: List[Run]
         """
-        if not row_ids:
-            raise ParameterError("row_ids")
+
         if not inputs and not is_json(inputs):
             raise ParameterError('inputs')
         if not outputs and not is_json(outputs):
             raise ParameterError('outputs')
-
-        data_model_id = self.query_data_model_id(data_model_name)
-        if not data_model_id:
-            raise ParameterError("data_model_name")
-
         if not submission_name_suffix:
             submission_name_suffix = datetime.now().strftime(
                 '%Y-%m-%d-%H-%M-%S')
-        submission_id = Config.service().create_submission({
-            "ClusterID":
-            self.get_cluster,
-            'WorkspaceID':
-            self.workspace_id,
-            'WorkflowID':
-            self.id,
-            'Name':
-            workflows.submission_name(self.name, submission_name_suffix),
-            'Description':
-            submission_desc,
-            'DataModelID':
-            data_model_id,
-            'DataModelRowIDs':
-            row_ids,
-            'Inputs':
-            inputs,
+
+        params = {
+            "ClusterID": self.get_cluster,
+            'WorkspaceID': self.workspace_id,
+            'WorkflowID': self.id,
+            'Name': workflows.submission_name(self.name,
+                                              submission_name_suffix),
+            'Description': submission_desc,
+            'Inputs': inputs,
             'ExposedOptions': {
                 "ReadFromCache": call_caching,
                 # TODO this may change in the future
                 "ExecutionRootDir": f"s3://{self.bucket}"
             },
-            'Outputs':
-            outputs,
-        }).get("ID")
+            'Outputs': outputs,
+        }
+
+        # It is batch mode when data_model_name and row_ids are specified.
+        if data_model_name and row_ids:
+            data_model_id = self.query_data_model_id(data_model_name)
+            if not data_model_id:
+                raise ParameterError("data_model_name")
+
+            params['DataModelID'] = data_model_id
+            params['DataModelRowIDs'] = row_ids
+
+        submission_id = Config.service().create_submission(params).get("ID")
 
         return Submission(self.workspace_id, submission_id).runs

{pybioos-0.0.5.dist-info → pybioos-0.0.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pybioos
-Version: 0.0.5
+Version: 0.0.7
 Summary: BioOS SDK for Python
 Home-page: https://github.com/GBA-BI/pybioos
 Author: Jilong Liu

{pybioos-0.0.5.dist-info → pybioos-0.0.7.dist-info}/RECORD

@@ -1,7 +1,7 @@
-bioos/__about__.py,sha256=AbfDZKyySPiWBjcsh8N93m6C2wXayn43ZS5G_U3Cx5A,56
+bioos/__about__.py,sha256=okNssbP5M6Wx_3bJKWb7uNW9yTV_ylNEEhx1uaNcEAc,56
 bioos/__init__.py,sha256=4GZKi13lDTD25YBkGakhZyEQZWTER_OWQMNPoH_UM2c,22
 bioos/bioos.py,sha256=fHzOb1l5wYxw6NVYYZDiFcgk4V28BAgWEc3ev12reWs,2409
-bioos/bioos_workflow.py,sha256=QOyoFXFkE8dMAuTUT5hf8vXy2l4EEt5YzztsBu_s9Fk,10070
+bioos/bioos_workflow.py,sha256=ZhlgIqsCMUBnNvzW1hQIlagJtZpSeDVFp99yXpIU358,14122
 bioos/config.py,sha256=CvFabYqV1BkFWO8fnr5vBf6xNtNzA8hAEVeEIbvAOm8,4307
 bioos/errors.py,sha256=Lzz2rkjDOTR2X9CnVkmsmqeOgmNqbi46WAxnC6LEGm0,2459
 bioos/log.py,sha256=twiCvf5IgJB7uvzANwBluSlztJN8ZrxbGZUBGlZ0vps,3204
@@ -13,7 +13,7 @@ bioos/resource/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 bioos/resource/data_models.py,sha256=enKp8yyQI8IbRqe--0Xtyg1XzOwQQPQzoQsx_hNuZ6E,5089
 bioos/resource/files.py,sha256=zMxsLg1CfZyrgvMesphMz-qROU-38fE_E8XtN9DlWtE,8281
 bioos/resource/utility.py,sha256=emY7qVLLLvGmQYlVj-_bLAxU7i1GfQOUybdRkfEDwVA,1300
-bioos/resource/workflows.py,sha256=06e9KS3Bm73mh9U6cDI7g8TmReM62EcgIjAxPGaqXaw,19541
+bioos/resource/workflows.py,sha256=A-fEUqPGuCmqZNNaW1zc8cozFJoQcg8bzXSyI2ZLypM,19724
 bioos/resource/workspaces.py,sha256=Gmr8y_sjK7TQbhMhQ_7rxqR1KFcwU72I95YYCFrrLBQ,3995
 bioos/service/BioOsService.py,sha256=HuYUEwomHCLpA1MYgVqGyWAQWHM-_BHB-jmy9VsOlnQ,6724
 bioos/service/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
@@ -30,9 +30,9 @@ bioos/tests/workspaces.py,sha256=LuuRrTs2XqfE5mGQyJNl9RBtuMb4NZHBJFoO8HMZVYQ,522
 bioos/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 bioos/utils/common_tools.py,sha256=fgMoE_-qZjgfQtUj_pmCTyYDtbJasyfH4Gm3VQsbgBQ,1651
 bioos/utils/workflows.py,sha256=zRbwTUigoM5V5LFOgzQPm3kwxt5Ogz95OFfefJc6Fjo,133
-pybioos-0.0.5.dist-info/LICENSE,sha256=cPkGXsgfPgEhIns7Lt3Avxx0Uy-VbdsoP8jvNGuj3cE,1063
-pybioos-0.0.5.dist-info/METADATA,sha256=NiirXZl6Vyh3dnxRhrgMit-xyu6NCNdR5kIHeKM6nMM,770
-pybioos-0.0.5.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-pybioos-0.0.5.dist-info/entry_points.txt,sha256=8TRx1zyu7ja3x5RNaeFxeiYTj_-tiWydbuSulxi3TM0,59
-pybioos-0.0.5.dist-info/top_level.txt,sha256=llpzydkKVDSaWZgz3bsTUsQmhoQpc_JcRJg2-H-5a2U,6
-pybioos-0.0.5.dist-info/RECORD,,
+pybioos-0.0.7.dist-info/LICENSE,sha256=cPkGXsgfPgEhIns7Lt3Avxx0Uy-VbdsoP8jvNGuj3cE,1063
+pybioos-0.0.7.dist-info/METADATA,sha256=a-HFZWmOAZg1U9jIc5v2I-1mh7hJHA25pZbEeU3uPu8,770
+pybioos-0.0.7.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+pybioos-0.0.7.dist-info/entry_points.txt,sha256=8TRx1zyu7ja3x5RNaeFxeiYTj_-tiWydbuSulxi3TM0,59
+pybioos-0.0.7.dist-info/top_level.txt,sha256=llpzydkKVDSaWZgz3bsTUsQmhoQpc_JcRJg2-H-5a2U,6
+pybioos-0.0.7.dist-info/RECORD,,