pybioos 0.0.15__py3-none-any.whl → 0.0.19__py3-none-any.whl

bioos/__about__.py

@@ -1,4 +1,4 @@
 # coding:utf-8
 
 # Package version
-__version__ = "0.0.15"
+__version__ = "0.0.19"

bioos/bioos.py

@@ -1,7 +1,7 @@
 from pandas import DataFrame
 from volcengine.const.Const import REGION_CN_NORTH1
 
-from bioos.config import Config
+from bioos.config import Config,DEFAULT_ENDPOINT
 from bioos.resource.utility import UtilityResource
 from bioos.resource.workspaces import Workspace
 
@@ -20,9 +20,9 @@ def status() -> Config.LoginInfo:
     return Config.login_info()
 
 
-def login(endpoint: str,
-          access_key: str,
+def login(access_key: str,
           secret_key: str,
+          endpoint: str = DEFAULT_ENDPOINT,
           region: str = REGION_CN_NORTH1) -> bool:
     """Login to the given endpoint using specified account and password.
 
@@ -35,14 +35,16 @@ def login(endpoint: str,
     *Example*:
     ::
 
-        bioos.login(endpoint="https://cloud.xxxxx.xxx.cn",access_key="xxxxxxxx",secret_key="xxxxxxxx")
+        bioos.login(access_key="xxxxxxxx", secret_key="xxxxxxxx")
+        # or specify endpoint explicitly:
+        bioos.login(access_key="xxxxxxxx", secret_key="xxxxxxxx", endpoint="https://cloud.xxxxx.xxx.cn")
 
-    :param endpoint: The environment to be logged in
-    :type endpoint: str
     :param access_key: The specified account's access key
     :type access_key: str
     :param secret_key: Corresponding secret key of the access key
     :type secret_key: str
+    :param endpoint: The environment to be logged in (optional, defaults to Config._endpoint)
+    :type endpoint: str
     :param region: The region to be logged in
     :type region: str
     :return: Login result
@@ -50,7 +52,8 @@ def login(endpoint: str,
     """
     Config.set_access_key(access_key)
     Config.set_secret_key(secret_key)
-    Config.set_endpoint(endpoint)
+    if endpoint is not None:
+        Config.set_endpoint(endpoint)
     Config.set_region(region)
     return Config.login_info().login_status == "Already logged in"
 
@@ -70,6 +73,28 @@ def list_workspaces() -> DataFrame:
     }).get("Items"))
 
 
+def create_workspace(name: str, description: str) -> dict:
+    """Creates a new workspace in the login environment.
+
+    *Example*:
+    ::
+
+        bioos.create_workspace(name="My Workspace", description="This is my new workspace")
+
+    :param name: Name of the workspace to create
+    :type name: str
+    :param description: Description of the workspace
+    :type description: str
+    :return: Creation result containing workspace information
+    :rtype: dict
+    """
+    params = {
+        "Name": name,
+        "Description": description
+    }
+    return Config.service().create_workspace(params)
+
+
 def workspace(id_: str) -> Workspace:  # 这里是workspace的入口
     """Returns the workspace for the given name .
 

bioos/bioos_workflow.py

@@ -4,11 +4,77 @@ import logging
 import os
 import re
 import time
+from typing import Dict, Any
 
 import pandas as pd
 
 from bioos import bioos
 from bioos.errors import NotFoundError, ParameterError
+from bioos.config import DEFAULT_ENDPOINT
+
+def uniquify_columns(cols: list[str]) -> list[str]:
+    seen, out = {}, []
+    for col in cols:
+        base = col.split(".")[-1]
+        if base not in seen:
+            seen[base] = 0
+            out.append(base)
+        else:
+            seen[base] += 1
+            out.append(f"{base}_{seen[base]}")   # fastq → fastq_1 → fastq_2
+    return out
+
+def handle_null_values(input_dict):
+    """
+    处理输入字典中的null值，避免在通过pybioos后台传输时引发错误。
+    
+    Args:
+        input_dict: 输入的字典，可能包含null值
+        
+    Returns:
+        处理后的字典，null值已被适当处理
+    """
+    if not isinstance(input_dict, dict):
+        return input_dict
+        
+    result = {}
+    for key, value in input_dict.items():
+        if value is None:
+            
+            key_lower = key.lower()
+            
+            # 明显的文件路径或字符串相关
+            if any(keyword in key_lower for keyword in ["file", "path", "dir", "name", "str", "text", "url", "uri"]):
+                result[key] = ""
+            # 明显的数字相关
+            elif any(keyword in key_lower for keyword in ["num", "count", "size", "int", "float", "length", "depth", "width", "height"]):
+                continue
+            # 布尔值相关
+            elif any(keyword in key_lower for keyword in ["flag", "bool", "enable", "disable", "is_", "has_"]):
+                continue
+            # 数组或列表相关
+            elif any(keyword in key_lower for keyword in ["list", "array", "items", "samples"]):
+                # 对于数组，null通常表示空数组
+                result[key] = []
+            else:
+                # 默认情况下，替换为空字符串，因为这是最安全的处理方式
+                result[key] = ""
+        elif isinstance(value, dict):
+            result[key] = handle_null_values(value)
+        elif isinstance(value, list):
+            processed_list = []
+            for item in value:
+                if isinstance(item, dict):
+                    processed_list.append(handle_null_values(item))
+                elif item is None:
+                    continue
+                else:
+                    processed_list.append(item)
+            result[key] = processed_list
+        else:
+            result[key] = value
+            
+    return result
 
 
 def recognize_files_from_input_json(workflow_input_json: dict) -> dict:
@@ -121,6 +187,8 @@ class Bioos_workflow:
         # 处理provision，更新inputs_list
         inputs_list_update = []
         for input_dict in inputs_list:
+            # 处理null值
+            input_dict = handle_null_values(input_dict)
             input_dict_update = self.input_provision(input_dict,
                                                      force_reupload)
             inputs_list_update.append(input_dict_update)
@@ -165,12 +233,19 @@ class Bioos_workflow:
                     data_model_name: str = "dm",
                     submission_desc: str = "Submit by pybioos",
                     call_caching: bool = True,
-                    force_reupload: bool = False):
+                    force_reupload: bool = False,
+                    mount_tos: bool = False):
         if not os.path.isfile(input_json_file):
             raise ParameterError('Input_json_file is not found.')
+        #给每一个data_model加一个uuid，保证不重复
+        if data_model_name == "dm":
+            data_model_name = f"dm_{int(time.time())}"
 
         input_json = json.load(open(input_json_file))
         self.logger.info("Load json input successfully.")
+        
+        # 处理null值
+        input_json = handle_null_values(input_json)
 
         # putative files
         input_json_str = json.dumps(input_json)
@@ -213,6 +288,7 @@ class Bioos_workflow:
             "outputs": "{}",
             "submission_desc": submission_desc,
             "call_caching": call_caching,
+            "mount_tos": mount_tos,
         }
 
         # if the input json is a batch or singleton submission
@@ -231,7 +307,8 @@ class Bioos_workflow:
             df[id_col] = [f"tmp_{x}" for x in list(range(len(df)))]
             df = df.reindex(columns=columns)
             columns = [key.split(".")[-1] for key in df.columns.to_list()]
-            df.columns = pd.Index(columns)
+            #df.columns = pd.Index(columns)
+            df.columns = pd.Index(uniquify_columns(df.columns.to_list()))
 
             # write data models
             self.ws.data_models.write({data_model_name: df.applymap(str)},
@@ -243,6 +320,7 @@ class Bioos_workflow:
             for key, _ in unupdate_dict.items():
                 unupdate_dict[key] = f'this.{key.split(".")[-1]}'
 
+
             self.params_submit["inputs"] = json.dumps(unupdate_dict)
             self.params_submit["data_model_name"] = data_model_name
             self.params_submit["row_ids"] = df[id_col].to_list()
@@ -254,7 +332,7 @@ class Bioos_workflow:
         self.logger.info("Build params dict successfully.")
         return self.params_submit
 
-    def postprocess(self, download=False):
+    def postprocess(self, download=False,download_dir="."):
         # 假设全部执行完毕
         #  对运行完成的目录进行下载
         # 证实bioos包只能对文件的list进行下载，不支持文件夹
@@ -268,9 +346,10 @@ class Bioos_workflow:
 
                     files.append(file)
 
-        if download:
+        if download and files:
+            os.makedirs(download_dir, exist_ok=True)
             try:
-                self.ws.files.download(files, ".", flatten=False)
+                self.ws.files.download(files, download_dir, flatten=False)
             except Exception as e:
                 print(f'Some file can not download. \n {e}')
 
@@ -305,7 +384,7 @@ def bioos_workflow():
     parser.add_argument("--endpoint",
                         type=str,
                         help="Bio-OS instance platform endpoint",
-                        default="https://bio-top.miracle.ac.cn")
+                        default=DEFAULT_ENDPOINT)
     parser.add_argument(
         "--ak",
         type=str,
@@ -344,6 +423,12 @@ def bioos_workflow():
                         action='store_true',
                         help="Force reupolad tos existed files.")
 
+    parser.add_argument(
+        "--mount_tos",
+        action='store_true',
+        help="是否挂载tos",
+        default=False)
+
     parser.add_argument(
         "--monitor",
         action='store_true',
@@ -357,6 +442,11 @@ def bioos_workflow():
         "--download_results",
         action='store_true',
         help="Download the submission run result files to local current path.")
+    parser.add_argument(
+        "--download_dir",
+        type=str,
+        default=".",
+        help="本地保存下载结果的目录（默认当前目录）")
 
     parsed_args = parser.parse_args()
 
@@ -370,7 +460,8 @@ def bioos_workflow():
                    data_model_name=parsed_args.data_model_name,
                    submission_desc=parsed_args.submission_desc,
                    call_caching=parsed_args.call_caching,
-                   force_reupload=parsed_args.force_reupload)
+                   force_reupload=parsed_args.force_reupload,
+                   mount_tos=parsed_args.mount_tos)
     bw.submit_workflow_bioosapi()
 
     # moniter
@@ -395,5 +486,5 @@ def bioos_workflow():
         print(bw.runs)
 
         bw.logger.info("Start to postprocess.")
-        bw.postprocess(download=parsed_args.download_results)
+        bw.postprocess(download=parsed_args.download_results,download_dir = parsed_args.download_dir)
         bw.logger.info("Postprocess finished.")

bioos/bw_import.py

@@ -1,17 +1,13 @@
 #!/usr/bin/env python3
 # coding: utf-8
-
 import argparse
 import logging
 import os
 import sys
 import time
-
 from bioos import bioos
-from bioos.config import Config
+from bioos.config import Config, DEFAULT_ENDPOINT
 from bioos.resource.workflows import WorkflowResource
-
-
 def get_logger():
     """Setup logger"""
     logger = logging.getLogger('bw_import')
@@ -23,14 +19,11 @@ def get_logger():
         logger.addHandler(handler)
         logger.setLevel(logging.INFO)
     return logger
-
-
 def bioos_workflow_import():
     """Command line entry point"""
     parser = argparse.ArgumentParser(
         description='Bio-OS Workflow Import Tool',
         formatter_class=argparse.RawDescriptionHelpFormatter)
-
     # 必需参数
     parser.add_argument(
         '--ak',
@@ -49,8 +42,10 @@ def bioos_workflow_import():
     parser.add_argument('--workflow_source',
                         required=True,
                         help='Local WDL file path or git repository URL')
-
     # 可选参数
+    parser.add_argument('--endpoint',
+                        help='Bio-OS instance platform endpoint',
+                        default=DEFAULT_ENDPOINT)
     parser.add_argument('--workflow_desc',
                         help='Description for the workflow',
                         default='')
@@ -67,16 +62,13 @@ def bioos_workflow_import():
         type=int,
         default=60,
         help='Time interval in seconds for checking workflow status')
-
     args = parser.parse_args()
     logger = get_logger()
-
     try:
         # 配置Bio-OS
         Config.set_access_key(args.ak)
         Config.set_secret_key(args.sk)
-        Config.set_endpoint("https://bio-top.miracle.ac.cn")
-
+        Config.set_endpoint(args.endpoint)
         # 获取workspace ID
         workspaces = bioos.list_workspaces()
         workspace_info = workspaces.query(f"Name=='{args.workspace_name}'")
@@ -84,10 +76,8 @@ def bioos_workflow_import():
             logger.error(f"Workspace {args.workspace_name} not found")
             sys.exit(1)
         workspace_id = workspace_info["ID"].iloc[0]
-
         # 创建WorkflowResource实例
         workflow_resource = WorkflowResource(workspace_id)
-
         # 导入workflow
         try:
             result = workflow_resource.import_workflow(
@@ -99,19 +89,15 @@ def bioos_workflow_import():
             logger.info(
                 f"Successfully uploaded workflow: {result}, validating..., please wait..."
             )
-
             # 如果设置了monitor参数，则监控工作流状态
             if args.monitor:
                 max_retries = 10  # 最大重试次数
                 retry_count = 0
-
                 while retry_count < max_retries:
                     df = workflow_resource.list()
                     workflow_info = df[df.Name == args.workflow_name]
-
                     if len(workflow_info) == 1:
                         status = workflow_info.iloc[0]["Status"]["Phase"]
-
                         if status == "Succeeded":
                             logger.info(
                                 f"Workflow {args.workflow_name} validated successfully"
@@ -138,7 +124,6 @@ def bioos_workflow_import():
                             f"Workflow {args.workflow_name} not found after import"
                         )
                         sys.exit(1)
-
                 logger.error(
                     f"Workflow validation timeout after {max_retries} retries")
                 sys.exit(1)
@@ -148,15 +133,11 @@ def bioos_workflow_import():
                     f"Workflow {args.workflow_name} is still validating, {result}, please wait and check the status later."
                 )
                 sys.exit(0)
-
         except Exception as e:
             logger.error(f"Failed to import workflow: {str(e)}")
             sys.exit(1)
-
     except Exception as e:
         logger.error(f"Error: {str(e)}")
         sys.exit(1)
-
-
 if __name__ == '__main__':
     bioos_workflow_import()

bioos/bw_import_status_check.py

@@ -6,7 +6,7 @@ import logging
 import sys
 
 from bioos import bioos
-from bioos.config import Config
+from bioos.config import Config, DEFAULT_ENDPOINT
 from bioos.resource.workflows import WorkflowResource
 
 
@@ -44,6 +44,11 @@ def bioos_workflow_status_check():
     parser.add_argument('--workflow_id',
                         required=True,
                         help='ID of the workflow to check')
+    
+    # 可选参数
+    parser.add_argument('--endpoint',
+                        help='Bio-OS instance platform endpoint',
+                        default=DEFAULT_ENDPOINT)
 
     args = parser.parse_args()
     logger = get_logger()
@@ -52,7 +57,7 @@ def bioos_workflow_status_check():
         # 配置Bio-OS
         Config.set_access_key(args.ak)
         Config.set_secret_key(args.sk)
-        Config.set_endpoint("https://bio-top.miracle.ac.cn")
+        Config.set_endpoint(args.endpoint)
 
         # 获取workspace ID
         workspaces = bioos.list_workspaces()

bioos/bw_status_check.py

@@ -6,7 +6,7 @@ import logging
 import sys
 
 from bioos import bioos
-from bioos.config import Config
+from bioos.config import Config, DEFAULT_ENDPOINT
 from bioos.resource.workflows import WorkflowResource
 
 
@@ -44,6 +44,11 @@ def bioos_workflow_status_check():
     parser.add_argument('--submission_id',
                         required=True,
                         help='ID of the submission to check')
+    
+    # 可选参数
+    parser.add_argument('--endpoint',
+                        help='Bio-OS instance platform endpoint',
+                        default=DEFAULT_ENDPOINT)
 
     args = parser.parse_args()
     logger = get_logger()
@@ -52,7 +57,7 @@ def bioos_workflow_status_check():
         # 配置Bio-OS
         Config.set_access_key(args.ak)
         Config.set_secret_key(args.sk)
-        Config.set_endpoint("https://bio-top.miracle.ac.cn")
+        Config.set_endpoint(args.endpoint)
 
         # 获取workspace ID
         workspaces = bioos.list_workspaces()

bioos/config.py

@@ -7,6 +7,9 @@ from bioos.errors import ConfigurationError
 from bioos.log import PyLogger
 from bioos.service.BioOsService import BioOsService
 
+# 默认的 Bio-OS endpoint
+DEFAULT_ENDPOINT = "https://bio-top.miracle.ac.cn"
+
 LOGIN_STATUS = Literal['Already logged in', 'Not logged in']
 
 
@@ -14,7 +17,7 @@ class Config:
     _service: BioOsService = None
     _access_key: str = os.environ.get('VOLC_ACCESSKEY')
     _secret_key: str = os.environ.get('VOLC_SECRETKEY')
-    _endpoint: str = os.environ.get('BIOOS_ENDPOINT')
+    _endpoint: str = os.environ.get('BIOOS_ENDPOINT', DEFAULT_ENDPOINT)
     _region: str = REGION_CN_NORTH1
     Logger = PyLogger()  # 这里是把类赋给了Logger变量
 

bioos/get_submission_logs.py

@@ -7,7 +7,7 @@ import os
 import sys
 
 from bioos import bioos
-from bioos.config import Config
+from bioos.config import Config, DEFAULT_ENDPOINT
 
 
 def get_logger():
@@ -48,6 +48,11 @@ def get_submission_logs():
         '--output_dir',
         default='.',
         help='Local directory to save the logs (default: current directory)')
+    
+    # 可选参数
+    parser.add_argument('--endpoint',
+                        help='Bio-OS instance platform endpoint',
+                        default=DEFAULT_ENDPOINT)
 
     args = parser.parse_args()
     logger = get_logger()
@@ -56,7 +61,7 @@ def get_submission_logs():
         # 配置Bio-OS
         Config.set_access_key(args.ak)
         Config.set_secret_key(args.sk)
-        Config.set_endpoint("https://bio-top.miracle.ac.cn")
+        Config.set_endpoint(args.endpoint)
 
         # 获取workspace ID
         workspaces = bioos.list_workspaces()

bioos/resource/__init__.py

@@ -1 +1 @@
-
+from .iesapp import WebInstanceApp, WebInstanceAppResource