pybioos 0.0.9__py3-none-any.whl → 0.0.11__py3-none-any.whl

bioos/__about__.py

@@ -1,4 +1,4 @@
 # coding:utf-8
 
 # Package version
-__version__ = "0.0.9"
+__version__ = "0.0.11"

bioos/bw_import.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+# coding: utf-8
+
+import argparse
+import logging
+import os
+import sys
+import time
+
+from bioos import bioos
+from bioos.config import Config
+from bioos.resource.workflows import WorkflowResource
+
+
+def get_logger():
+    """Setup logger"""
+    logger = logging.getLogger('bw_import')
+    if not logger.handlers:
+        handler = logging.StreamHandler()
+        formatter = logging.Formatter(
+            '%(asctime)s - %(levelname)s - %(message)s')
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+        logger.setLevel(logging.INFO)
+    return logger
+
+
+def bioos_workflow_import():
+    """Command line entry point"""
+    parser = argparse.ArgumentParser(
+        description='Bio-OS Workflow Import Tool',
+        formatter_class=argparse.RawDescriptionHelpFormatter)
+
+    # 必需参数
+    parser.add_argument(
+        '--ak',
+        required=True,
+        help='Access key for your Bio-OS instance platform account')
+    parser.add_argument(
+        '--sk',
+        required=True,
+        help='Secret key for your Bio-OS instance platform account')
+    parser.add_argument('--workspace_name',
+                        required=True,
+                        help='Target workspace name')
+    parser.add_argument('--workflow_name',
+                        required=True,
+                        help='Name for the workflow to be imported')
+    parser.add_argument('--workflow_source',
+                        required=True,
+                        help='Local WDL file path or git repository URL')
+
+    # 可选参数
+    parser.add_argument('--workflow_desc',
+                        help='Description for the workflow',
+                        default='')
+    parser.add_argument(
+        '--main_path',
+        help='Main workflow file path (required for git repository)',
+        default='')
+
+    args = parser.parse_args()
+    logger = get_logger()
+
+    try:
+        # 配置Bio-OS
+        Config.set_access_key(args.ak)
+        Config.set_secret_key(args.sk)
+        Config.set_endpoint("https://bio-top.miracle.ac.cn")
+
+        # 获取workspace ID
+        workspaces = bioos.list_workspaces()
+        workspace_info = workspaces.query(f"Name=='{args.workspace_name}'")
+        if workspace_info.empty:
+            logger.error(f"Workspace {args.workspace_name} not found")
+            sys.exit(1)
+        workspace_id = workspace_info["ID"].iloc[0]
+
+        # 创建WorkflowResource实例
+        workflow_resource = WorkflowResource(workspace_id)
+
+        # 导入workflow
+        try:
+            result = workflow_resource.import_workflow(
+                source=args.workflow_source,
+                name=args.workflow_name,
+                description=args.workflow_desc,
+                language="WDL",
+                main_workflow_path=args.main_path)
+            logger.info(
+                f"Successfully uploaded workflow: {result}, validating..., please wait..."
+            )
+
+            # 循环检查工作流状态
+            max_retries = 10  # 最大重试次数
+            retry_count = 0
+
+            while retry_count < max_retries:
+                df = workflow_resource.list()
+                workflow_info = df[df.Name == args.workflow_name]
+
+                if len(workflow_info) == 1:
+                    status = workflow_info.iloc[0]["Status"]["Phase"]
+
+                    if status == "Succeeded":
+                        logger.info(
+                            f"Workflow {args.workflow_name} validated successfully"
+                        )
+                        sys.exit(0)
+                    elif status == "Failed":
+                        logger.error(
+                            f"Workflow {args.workflow_name} validation failed")
+                        sys.exit(1)
+                    elif status == "Importing":
+                        logger.info(
+                            f"Workflow {args.workflow_name} is still validating, please wait..."
+                        )
+                        time.sleep(30)
+                        retry_count += 1
+                    else:
+                        logger.error(
+                            f"Workflow {args.workflow_name} has unknown status: {status}"
+                        )
+                        sys.exit(1)
+                else:
+                    logger.error(
+                        f"Workflow {args.workflow_name} not found after upload"
+                    )
+                    sys.exit(1)
+
+            logger.error(
+                f"Workflow validation timeout after {max_retries} retries")
+            sys.exit(1)
+
+        except Exception as e:
+            logger.error(f"Failed to import workflow: {str(e)}")
+            sys.exit(1)
+
+    except Exception as e:
+        logger.error(f"Error: {str(e)}")
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    bioos_workflow_import()

bioos/resource/files.py

@@ -25,8 +25,14 @@ class FileResource(metaclass=SingletonType):
         res = Config.service().get_tos_access({
             'WorkspaceID': self.workspace_id,
         })  # 这里触发的是后端的返回行为，返回值无定义python类型
-        self.endpoint = s3_endpoint_mapping(res["Endpoint"])
-        self.region = res["Region"]
+
+        if "Endpoint" in res and "Region" in res:
+            self.endpoint = s3_endpoint_mapping(res["Endpoint"])
+            self.region = res["Region"]
+        else:
+            self.endpoint = "https://tos.miracle.ac.cn"
+            self.region = "cn-guangzhou-swd"
+
         self.tos_handler = TOSHandler(  #需要注意
             tos.TosClientV2(  # 2.6.6中tos下还要有一层
                 ak=None,

bioos/resource/workflows.py

@@ -1,4 +1,8 @@
+import base64
+import os
+import zipfile
 from datetime import datetime
+from io import BytesIO
 from typing import List
 
 import pandas as pd
@@ -18,6 +22,31 @@ RUN_STATUS = Literal["Succeeded", "Failed", "Running", "Pending"]
 WORKFLOW_LANGUAGE = Literal["WDL"]
 
 
+def zip_files(source_files, zip_type='base64'):
+    # 创建一个内存中的字节流对象
+    buffer = BytesIO()
+
+    # 使用 zipfile 模块创建一个 ZIP 文件对象
+    with zipfile.ZipFile(buffer, 'w', zipfile.ZIP_DEFLATED,
+                         compresslevel=9) as zip_file:
+        for f in source_files:
+            # 将文件添加到 ZIP 中
+            # 假设每个 f 是一个字典，包含 'name' 和 'originFile' 键
+            zip_file.writestr(f['name'], f['originFile'])
+
+    # 获取 ZIP 数据
+    zip_data = buffer.getvalue()
+
+    if zip_type == 'base64':
+        # 将 ZIP 数据编码为 base64
+        return base64.b64encode(zip_data).decode('utf-8')
+    elif zip_type == 'blob':
+        # 直接返回二进制数据
+        return zip_data
+    else:
+        raise ValueError("zip_type must be 'base64' or 'blob'")
+
+
 class Run(metaclass=SingletonType):  # 单例模式，why
     """Represents a specific run of submission .
     """
@@ -345,11 +374,11 @@ class WorkflowResource(metaclass=SingletonType):
     def import_workflow(self,
                         source: str,
                         name: str,
-                        language: WORKFLOW_LANGUAGE,
-                        tag: str,
-                        main_workflow_path: str,
-                        description: str = "",
-                        token: str = "") -> str:
+                        description: str,
+                        language: WORKFLOW_LANGUAGE = "WDL",
+                        tag: str = "",
+                        main_workflow_path: str = "",
+                        token: str = "") -> dict:
         """Imports a workflow .
 
         *Example*:
@@ -387,23 +416,44 @@ class WorkflowResource(metaclass=SingletonType):
         else:
             raise ParameterError("name", name)
 
-        if not (source.startswith("http://") or source.startswith("https://")):
-            raise ParameterError("source", source)
         if language != "WDL":
             raise ParameterError("language", language)
 
-        params = {
-            "WorkspaceID": self.workspace_id,
-            "Name": name,
-            "Description": description,
-            "Language": language,
-            "Source": source,
-            "Tag": tag,
-            "MainWorkflowPath": main_workflow_path,
-        }
-        if token:
-            params["Token"] = token
-        return Config.service().create_workflow(params).get("ID")
+        if source.startswith("http://") or source.startswith("https://"):
+            params = {
+                "WorkspaceID": self.workspace_id,
+                "Name": name,
+                "Description": description,
+                "Language": language,
+                "Source": source,
+                "Tag": tag,
+                "MainWorkflowPath": main_workflow_path,
+                "SourceType": "git",
+            }
+            if token:
+                params["Token"] = token
+        elif os.path.exists(source):
+            source_files = [{
+                "name": os.path.basename(source),
+                "originFile": open(source, "rb").read()
+            }]
+            zip_base64 = zip_files(source_files, 'base64')
+
+            main_workflow_path = os.path.basename(source)
+            params = {
+                "WorkspaceID": self.workspace_id,
+                "Name": name,
+                "Description": description,
+                "Language": language,
+                # "Source": source,
+                "SourceType": "file",
+                "Content": zip_base64,
+                "MainWorkflowPath": main_workflow_path,
+            }
+        else:
+            raise ParameterError("source", source)
+
+        return Config.service().create_workflow(params)
 
     def list(self) -> DataFrame:
         """Lists all workflows' information .
@@ -430,7 +480,7 @@ class WorkflowResource(metaclass=SingletonType):
         res_df['UpdateTime'] = pd.to_datetime(
             res_df['UpdateTime'], unit='ms', origin=pd.Timestamp('2018-07-01'))
 
-        return res_df.drop("Status", axis=1)
+        return res_df
 
     def delete(self, target: str):
         """Deletes a workflow from the workspace .

{pybioos-0.0.9.dist-info → pybioos-0.0.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pybioos
-Version: 0.0.9
+Version: 0.0.11
 Summary: BioOS SDK for Python
 Home-page: https://github.com/GBA-BI/pybioos
 Author: Jilong Liu
@@ -13,13 +13,15 @@ Classifier: Intended Audience :: Science/Research
 Classifier: Topic :: Scientific/Engineering :: Bio-Informatics
 Classifier: Topic :: Scientific/Engineering :: Interface Engine/Protocol Translator
 License-File: LICENSE
-Requires-Dist: volcengine >=1.0.61
-Requires-Dist: tabulate >=0.8.10
-Requires-Dist: click >=8.0.0
-Requires-Dist: pandas >=1.3.0
-Requires-Dist: tos ==2.5.6
-Requires-Dist: cachetools >=5.2.0
-Requires-Dist: typing-extensions >=4.4.0
-Requires-Dist: apscheduler >=3.10.4
-Requires-Dist: colorama >=0.4.6
+Requires-Dist: volcengine (>=1.0.61)
+Requires-Dist: tabulate (>=0.8.10)
+Requires-Dist: click (>=8.0.0)
+Requires-Dist: pandas (>=1.3.0)
+Requires-Dist: tos (==2.5.6)
+Requires-Dist: cachetools (>=5.2.0)
+Requires-Dist: typing-extensions (>=4.4.0)
+Requires-Dist: apscheduler (>=3.10.4)
+Requires-Dist: colorama (>=0.4.6)
+
+UNKNOWN
 

{pybioos-0.0.9.dist-info → pybioos-0.0.11.dist-info}/RECORD

@@ -1,7 +1,8 @@
-bioos/__about__.py,sha256=-AYOsaAd5kYnSQP79FEXM4KfbqyfJZc7yILrgr9b1rU,56
+bioos/__about__.py,sha256=CQkBh68geNlt0TyhyDWhAJZD8BGTGXo4PMLurVugwKo,57
 bioos/__init__.py,sha256=4GZKi13lDTD25YBkGakhZyEQZWTER_OWQMNPoH_UM2c,22
 bioos/bioos.py,sha256=fHzOb1l5wYxw6NVYYZDiFcgk4V28BAgWEc3ev12reWs,2409
 bioos/bioos_workflow.py,sha256=LbKxK7Otzj9apJciI3jg_cfh92x9aZGlUyN5ZRAdZeU,14127
+bioos/bw_import.py,sha256=VFG-RWZL_0cuZPYCxapzDn8EFqT2mFFsykBt6hG9wGM,4860
 bioos/config.py,sha256=CvFabYqV1BkFWO8fnr5vBf6xNtNzA8hAEVeEIbvAOm8,4307
 bioos/errors.py,sha256=Lzz2rkjDOTR2X9CnVkmsmqeOgmNqbi46WAxnC6LEGm0,2459
 bioos/log.py,sha256=twiCvf5IgJB7uvzANwBluSlztJN8ZrxbGZUBGlZ0vps,3204
@@ -11,9 +12,9 @@ bioos/models/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 bioos/models/models.py,sha256=HPCLZ4jK2HDth4vrlVHba21CiW6y7y5im1kOjV4adc8,391
 bioos/resource/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
 bioos/resource/data_models.py,sha256=enKp8yyQI8IbRqe--0Xtyg1XzOwQQPQzoQsx_hNuZ6E,5089
-bioos/resource/files.py,sha256=zMxsLg1CfZyrgvMesphMz-qROU-38fE_E8XtN9DlWtE,8281
+bioos/resource/files.py,sha256=1HY0IHvq8H843VM2XZIHDdCuXXNcMrlEFhSNqWXmFzE,8456
 bioos/resource/utility.py,sha256=emY7qVLLLvGmQYlVj-_bLAxU7i1GfQOUybdRkfEDwVA,1300
-bioos/resource/workflows.py,sha256=A-fEUqPGuCmqZNNaW1zc8cozFJoQcg8bzXSyI2ZLypM,19724
+bioos/resource/workflows.py,sha256=DUAwq0AE7FZbnb5iJUGEKC5XzGFl_CcJy7Sh0KO-EvM,21387
 bioos/resource/workspaces.py,sha256=Gmr8y_sjK7TQbhMhQ_7rxqR1KFcwU72I95YYCFrrLBQ,3995
 bioos/service/BioOsService.py,sha256=HuYUEwomHCLpA1MYgVqGyWAQWHM-_BHB-jmy9VsOlnQ,6724
 bioos/service/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
@@ -30,9 +31,9 @@ bioos/tests/workspaces.py,sha256=LuuRrTs2XqfE5mGQyJNl9RBtuMb4NZHBJFoO8HMZVYQ,522
 bioos/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 bioos/utils/common_tools.py,sha256=fgMoE_-qZjgfQtUj_pmCTyYDtbJasyfH4Gm3VQsbgBQ,1651
 bioos/utils/workflows.py,sha256=zRbwTUigoM5V5LFOgzQPm3kwxt5Ogz95OFfefJc6Fjo,133
-pybioos-0.0.9.dist-info/LICENSE,sha256=cPkGXsgfPgEhIns7Lt3Avxx0Uy-VbdsoP8jvNGuj3cE,1063
-pybioos-0.0.9.dist-info/METADATA,sha256=7s3pBVuK5VLgPkOR15fXwr_29t6qjycy3Oh92uLsS_4,802
-pybioos-0.0.9.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-pybioos-0.0.9.dist-info/entry_points.txt,sha256=8TRx1zyu7ja3x5RNaeFxeiYTj_-tiWydbuSulxi3TM0,59
-pybioos-0.0.9.dist-info/top_level.txt,sha256=llpzydkKVDSaWZgz3bsTUsQmhoQpc_JcRJg2-H-5a2U,6
-pybioos-0.0.9.dist-info/RECORD,,
+pybioos-0.0.11.dist-info/LICENSE,sha256=cPkGXsgfPgEhIns7Lt3Avxx0Uy-VbdsoP8jvNGuj3cE,1063
+pybioos-0.0.11.dist-info/METADATA,sha256=30fgMYG5pN8_Jn_bY-utOWWXW7TmzgBzmsc3Mz3GCfA,830
+pybioos-0.0.11.dist-info/WHEEL,sha256=OqRkF0eY5GHssMorFjlbTIq072vpHpF60fIQA6lS9xA,92
+pybioos-0.0.11.dist-info/entry_points.txt,sha256=G8eh4umiCimlV9nNNeyA3vqLyG3jfUj8uVNrgvAuhAM,110
+pybioos-0.0.11.dist-info/top_level.txt,sha256=llpzydkKVDSaWZgz3bsTUsQmhoQpc_JcRJg2-H-5a2U,6
+pybioos-0.0.11.dist-info/RECORD,,

{pybioos-0.0.9.dist-info → pybioos-0.0.11.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.42.0)
+Generator: bdist_wheel (0.36.2)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{pybioos-0.0.9.dist-info → pybioos-0.0.11.dist-info}/entry_points.txt

@@ -1,2 +1,4 @@
 [console_scripts]
 bw = bioos.bioos_workflow:bioos_workflow
+bw_import = bioos.bw_import:bioos_workflow_import
+