pybioos 0.0.3__py3-none-any.whl

bioos/__about__.py

@@ -0,0 +1,4 @@
+# coding:utf-8
+
+# Package version
+__version__ = "0.0.3"

bioos/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.3"

bioos/bioos.py

@@ -0,0 +1,90 @@
+from pandas import DataFrame
+from volcengine.const.Const import REGION_CN_NORTH1
+
+from bioos.config import Config
+from bioos.resource.utility import UtilityResource
+from bioos.resource.workspaces import Workspace
+
+
+def status() -> Config.LoginInfo:
+    """Get the current login information.
+
+    *Example*:
+    ::
+
+        bioos.status()
+
+    :return: Current login information
+    :rtype: Config.LoginInfo
+    """
+    return Config.login_info()
+
+
+def login(endpoint: str,
+          access_key: str,
+          secret_key: str,
+          region: str = REGION_CN_NORTH1) -> bool:
+    """Login to the given endpoint using specified account and password.
+
+    **If bioos sdk runs inside the miracle private cloud env** such as on a notebook under a
+    workspace, the login procedure will be finished automatically.
+
+    **If bioos sdk runs outside the miracle cloud env** such as on user's local machine,
+    the login procedure should be explicitly executed.
+
+    *Example*:
+    ::
+
+        bioos.login(endpoint="https://cloud.xxxxx.xxx.cn",access_key="xxxxxxxx",secret_key="xxxxxxxx")
+
+    :param endpoint: The environment to be logged in
+    :type endpoint: str
+    :param access_key: The specified account's access key
+    :type access_key: str
+    :param secret_key: Corresponding secret key of the access key
+    :type secret_key: str
+    :param region: The region to be logged in
+    :type region: str
+    :return: Login result
+    :rtype: bool
+    """
+    Config.set_access_key(access_key)
+    Config.set_secret_key(secret_key)
+    Config.set_endpoint(endpoint)
+    Config.set_region(region)
+    return Config.login_info().login_status == "Already logged in"
+
+
+def list_workspaces() -> DataFrame:
+    """Lists all workspaces in the login environment .
+
+    *Example*:
+    ::
+
+        bioos.list_workspaces()
+
+    """
+    return DataFrame.from_records(Config.service().list_workspaces({
+        "PageSize":
+        0
+    }).get("Items"))
+
+
+def workspace(id_: str) -> Workspace:  # 这里是workspace的入口
+    """Returns the workspace for the given name .
+
+    :param id_: Workspace id
+    :type id_: str
+    :return: Specified workspace object
+    :rtype: Workspace
+    """
+    return Workspace(id_)
+
+
+def utility() -> UtilityResource:
+    """Returns Common tool collection Resource object
+
+    :return: Tool collection Resource object
+    :rtype: UtilityResource
+    """
+    return UtilityResource()

bioos/bioos_workflow.py

@@ -0,0 +1,284 @@
+import argparse
+import json
+import logging
+import os
+import time
+
+import pandas as pd
+
+from bioos import bioos
+from bioos.errors import NotFoundError
+
+
+def recognize_files_from_input_json(workflow_input_json: dict) -> dict:
+    putative_files = {}
+
+    # this version only support absolute path
+
+    for key, value in workflow_input_json.items():
+        if str(value).startswith("s3"):
+            continue
+
+        if "registry-vpc" in str(value):
+            continue
+
+        if "/" in str(value):
+            putative_files[key] = value
+
+    return putative_files
+
+
+def get_logger():
+    global LOGGER
+
+    LOGGER = logging.getLogger(__name__)
+    LOGGER.setLevel(logging.DEBUG)
+    formatter = logging.Formatter("%(asctime)s %(levelname)s: %(message)s")
+
+    handler = logging.StreamHandler()
+    handler.setLevel(logging.DEBUG)
+    handler.setFormatter(formatter)
+    LOGGER.addHandler(handler)
+
+    return LOGGER
+
+
+# 引入langchain，利用dash起一个界面？
+# 将启明的内容整合进来
+# 与Workspace SPEC做映射
+class Bioos_workflow:
+
+    def __init__(self, workspace_name: str, workflow_name: str) -> None:
+        # global LOGGER
+        self.logger = get_logger()
+
+        # get workspace id
+        df = bioos.list_workspaces()
+        ser = df[df.Name == workspace_name].ID
+        if len(ser) != 1:
+            raise NotFoundError("Workspace", workspace_name)
+        workspace_id = ser.to_list()[0]
+
+        self.ws = bioos.workspace(workspace_id)
+        self.wf = self.ws.workflow(name=workflow_name)
+
+    # 需要有推定上传目的地址的机制，由WES endpoint的配置来指定
+    def input_provision(self, workflow_input_json: dict):
+        # need to support different source and target
+        # 输入的是WDL的标准json，有两种形式，单例的{}和多例的[{}]，为简单表述，这里以单例形式处理
+
+        # find files
+        putative_files = recognize_files_from_input_json(workflow_input_json)
+
+        # upload files
+        update_dict = {}
+        for key, value in putative_files.items():
+            target = f"input_provision/{os.path.basename(value)}"
+            # 这里如果多行记录，即多个样本的run中有相同的文件，可能会触发多次上传。可能可以通过判断文件是否存在来判断
+            # 需要对file的存在性进行检验
+            # 这里的target是prefix
+            self.logger.info(f"Start upload {value}.")
+            self.ws.files.upload(value,
+                                 target="input_provision/",
+                                 flatten=True)
+            self.logger.info(f"Finish upload {value}.")
+
+            s3_location = self.ws.files.s3_urls(target)[0]
+            update_dict[key] = s3_location
+
+        # update json
+        workflow_input_json.update(update_dict)
+        return workflow_input_json
+
+    def output_provision(self):
+        pass
+
+    def preprocess(
+        self,
+        input_json_file: str,
+        data_model_name: str = "dm",
+        submission_desc: str = "Submit by pybioos",
+        call_caching: bool = True,
+    ):
+        input_json = json.load(open(input_json_file))
+        self.logger.info("Load json input successfully.")
+
+        # 将单例的模式转换成向量形式
+        if isinstance(input_json, list):
+            inputs_list = input_json
+        else:
+            inputs_list = [
+                input_json,
+            ]
+
+        # 处理provision，更新inputs_list
+        inputs_list_update = []
+        for input_dict in inputs_list:
+            input_dict_update = self.input_provision(input_dict)
+            inputs_list_update.append(input_dict_update)
+
+        # 生成datamodel并上传
+        # 这里还需要处理id列的内容
+        df = pd.DataFrame(inputs_list_update)
+        id_col = f"{data_model_name}_id"
+        columns = [
+            id_col,
+        ]
+        columns.extend(df.columns)
+        df[id_col] = [f"tmp_{x}" for x in list(range(len(df)))]
+        df = df.reindex(columns=columns)
+        columns = [key.split(".")[-1] for key in df.columns.to_list()]
+        df.columns = pd.Index(columns)
+
+        # 这里可能要对每次新上传的datamodel进行重命名
+        # 这里经证实只支持全str类型的df
+        self.ws.data_models.write({data_model_name: df.map(str)}, force=True)
+        self.logger.info("Set data model successfully.")
+
+        # 生成veapi需要的输入结构
+        unupdate_dict = inputs_list[0]
+        for key, value in unupdate_dict.items():
+            unupdate_dict[key] = f'this.{key.split(".")[-1]}'
+
+        self.params_submit = {
+            "inputs": json.dumps(unupdate_dict),
+            "outputs": "{}",
+            "data_model_name": data_model_name,
+            "row_ids": df[id_col].to_list(),
+            "submission_desc": submission_desc,
+            "call_caching": call_caching,
+        }
+        self.logger.info("Build submission params successfully.")
+
+        return self.params_submit
+
+    def postprocess(self, download=False):
+        # 假设全部执行完毕
+        #  对运行完成的目录进行下载
+        # 证实bioos包只能对文件的list进行下载，不支持文件夹
+        # ws.files.list方法不能指定起始路径，需要改进
+        # 需要有一个地方执行定时任务，对run的status进行查询，并记录状态，对每次新完成的run进行后处理
+        files = []
+        for file in self.ws.files.list().key:
+            for run in self.runs:
+                if run.submission in file:
+                    print(file)
+                    if "%" in file:
+                        continue
+                    files.append(file)
+
+        if download:
+            try:
+                self.ws.files.download(files, ".", flatten=False)
+            except:  # noqa: E722
+                print('Some file can not download.')
+
+            self.logger.info("Download finish.")
+
+    def submit_workflow_bioosapi(self):
+        self.runs = self.wf.submit(**self.params_submit)
+        self.logger.info("Submit workflow run successfully.")
+        return self.runs
+
+    def monitor_workflow(self):
+        # wf是否有对应的查询方法
+        runs = []
+        for run in self.runs:
+            run.sync()
+            runs.append(run)
+
+        self.runs = runs
+        return self.runs
+
+
+def bioos_workflow():
+
+    # argparse
+    parser = argparse.ArgumentParser(
+        description="Bio-OS instance platform workflow submitter program.")
+    parser.add_argument("--endpoint",
+                        type=str,
+                        help="Bio-OS instance platform endpoint",
+                        default="https://bio-top.miracle.ac.cn")
+    parser.add_argument(
+        "--ak",
+        type=str,
+        help="Access_key for your Bio-OS instance platform account.")
+    parser.add_argument(
+        "--sk",
+        type=str,
+        help="Secret_key for your Bio-OS instance platform account.")
+
+    parser.add_argument("--workspace_name",
+                        type=str,
+                        help="Target workspace name.")
+    parser.add_argument("--workflow_name",
+                        type=str,
+                        help="Target workflow name.")
+    parser.add_argument(
+        "--input_json",
+        type=str,
+        help="The input_json file in Cromwell Womtools format.")
+    parser.add_argument(
+        "--data_model_name",
+        type=str,
+        help=
+        "Intended name for the generated data_model on the Bio-OS instance platform workspace page.",
+        default="dm")
+    parser.add_argument(
+        "--call_caching",
+        action='store_true',
+        help="Call_caching for the submission run.",
+    )
+    parser.add_argument('--submission_desc',
+                        type=str,
+                        help="Description for the submission run.",
+                        default="Submit by pybioos.")
+
+    parser.add_argument(
+        "--monitor",
+        action='store_true',
+        help="Moniter the status of submission run until finishment.")
+    parser.add_argument(
+        "--monitor_interval",
+        type=int,
+        default=600,
+        help="Time interval for query the status for the submission runs.")
+    parser.add_argument(
+        "--download_results",
+        action='store_true',
+        help="Download the submission run result files to local current path.")
+
+    parsed_args = parser.parse_args()
+
+    # login and submit
+    bioos.login(endpoint=parsed_args.endpoint,
+                access_key=parsed_args.ak,
+                secret_key=parsed_args.sk)
+    bw = Bioos_workflow(workspace_name=parsed_args.workspace_name,
+                        workflow_name=parsed_args.workflow_name)
+    bw.preprocess(input_json_file=parsed_args.input_json,
+                  data_model_name=parsed_args.data_model_name,
+                  submission_desc=parsed_args.submission_desc,
+                  call_caching=parsed_args.call_caching)
+    bw.submit_workflow_bioosapi()
+
+    # moniter
+    def all_runs_done() -> bool:
+
+        statuses = []
+        for run in bw.runs:
+            statuses.append(True if run.status in ("Succeeded",
+                                                   "Failed") else False)
+
+        return all(statuses)
+
+    if parsed_args.monitor or parsed_args.download_results:
+        while not all_runs_done():
+            bw.logger.info("Monitoring submission run.")
+            print(bw.runs)
+            time.sleep(parsed_args.monitor_interval)
+            bw.monitor_workflow()
+
+        time.sleep(60)
+        bw.postprocess(download=parsed_args.download_results)

bioos/config.py

@@ -0,0 +1,147 @@
+import os
+
+from typing_extensions import Literal
+from volcengine.const.Const import REGION_CN_NORTH1
+
+from bioos.errors import ConfigurationError
+from bioos.log import PyLogger
+from bioos.service.BioOsService import BioOsService
+
+LOGIN_STATUS = Literal['Already logged in', 'Not logged in']
+
+
+class Config:
+    _service: BioOsService = None
+    _access_key: str = os.environ.get('VOLC_ACCESSKEY')
+    _secret_key: str = os.environ.get('VOLC_SECRETKEY')
+    _endpoint: str = os.environ.get('BIOOS_ENDPOINT')
+    _region: str = REGION_CN_NORTH1
+    Logger = PyLogger
+
+    class LoginInfo:
+        """[Only Read]Record the current login information .
+        """
+
+        @property
+        def access_key(self):
+            """Returns the Login AccessKey .
+            """
+            return Config._access_key
+
+        @property
+        def secret_key(self) -> str:
+            """Returns the Login SecretKey .
+            """
+            return Config._secret_key
+
+        @property
+        def endpoint(self) -> str:
+            """Returns the Login Endpoint .
+            """
+            return Config._endpoint
+
+        @property
+        def region(self) -> str:
+            """Returns the Login Region .
+            """
+            return Config._region
+
+        @property
+        def login_status(self) -> LOGIN_STATUS:
+            """Return the login status .
+
+            :return: Login status: 'Already logged in' or 'Not logged in'
+            :rtype: str
+            """
+            try:
+                Config._ping_func()
+            except ConfigurationError:
+                return "Not logged in"
+            except Exception as e:
+                Config.Logger.error(e)
+                return "Not logged in"
+            return "Already logged in"
+
+        def __repr__(self):
+            return f"{self.login_status}\n" \
+                   f"endpoint: {self.endpoint}\n" \
+                   f"access_key: {self.access_key}\n" \
+                   f"secret_key: {self.secret_key}\n" \
+                   f"region: {self.region}"
+
+    @classmethod
+    def _same_endpoint(cls):
+        return (cls._service.service_info.scheme + "://" +
+                cls._service.service_info.host) == cls._endpoint
+
+    @classmethod
+    def _same_region(cls):
+        return cls._service.service_info.credentials.region == cls._region
+
+    @classmethod
+    def service(cls):
+        if cls._service:
+            return cls._service
+        cls._init_service()
+        return cls._service
+
+    @classmethod
+    def _ping_func(cls):
+        if not cls._service:
+            cls._init_service()
+        cls._service.list_workspaces({})  #通过该函数能否正常执行来判断是否登陆成功。
+
+    @classmethod
+    def login_info(cls):
+        return Config.LoginInfo()
+
+    @classmethod
+    def set_access_key(cls, access_key: str):
+        cls._access_key = access_key
+        if cls._service:
+            cls._service.set_ak(cls._access_key)
+
+    @classmethod
+    def set_secret_key(cls, secret_key: str):
+        cls._secret_key = secret_key
+        if cls._service:
+            cls._service.set_sk(cls._secret_key)
+
+    @classmethod
+    def set_endpoint(cls, endpoint: str):
+        cls._endpoint = endpoint
+        if cls._service and cls._same_endpoint():
+            return
+
+        cls._init_service()
+
+    @classmethod
+    def set_region(cls, region: str):
+        cls._region = region
+        if cls._service and cls._same_region():
+            return
+
+        cls._init_service()
+
+    @classmethod
+    def _init_service(cls):
+        if cls._service and cls._same_region() and cls._same_endpoint():
+            return
+
+        if not cls._endpoint:
+            raise ConfigurationError('ENDPOINT')
+
+        if not cls._region:
+            raise ConfigurationError('REGION')
+
+        if not cls._access_key:
+            raise ConfigurationError('ACCESS_KEY')
+
+        if not cls._secret_key:
+            raise ConfigurationError('SECRET_KEY')
+
+        cls._service = BioOsService(
+            endpoint=cls._endpoint,
+            region=cls._region)  #cls._service 属性保持登陆状态，并做为下游的调用入口
+        cls._service.set_ak(cls._access_key)
+        cls._service.set_sk(cls._secret_key)

bioos/errors.py

@@ -0,0 +1,89 @@
+# coding:utf-8
+class ConfigurationError(Exception):
+    """Exception indicating a required configuration not set .
+    """
+
+    def __init__(self, conf: str):
+        """Initialize the ConfigurationError .
+
+        :param conf: name of the configuration
+        :type conf: str
+        """
+
+        self.conf = conf
+        self.message = "configuration '{}' must be set".format(conf)
+        super().__init__(self.message)
+
+
+class EnvironmentConfigurationError(ConfigurationError):
+    """Exception indicating a required configuration **environment** not set .
+    """
+
+    def __init__(self, env: str):
+        """Initialize the EnvironmentConfigurationError .
+        
+        :param env: environment name of the configuration
+        :type env: str
+        """
+
+        self.env = env
+        self.message = "environment '{}' must be set".format(env)
+        super().__init__(self.message)
+
+
+class NotFoundError(Exception):
+    """Exception indicating an object not found error
+    """
+
+    def __init__(self, typ: str, name: str):
+        """Initialize the NotFoundError .
+
+        :param typ: object type, e.g. Table Workflow
+        :type typ: str
+        :param name: object name
+        :type name: str
+        """
+        self.message = "{} '{}' not found".format(typ, name)
+        super().__init__(self.message)
+
+
+class ParameterError(Exception):
+    """Exception indicating a required parameter not valid
+    """
+
+    def __init__(self, name: str):
+        """Initialize the ParameterError .
+
+        :param name: name of the parameter
+        :type name: str
+        """
+        self.message = "parameter '{}' invalid / not found".format(name)
+        super().__init__(self.message)
+
+
+class ConflictError(Exception):
+    """Exception indicating a conflict
+    """
+
+    def __init__(self, name: str, msg: str = ""):
+        """Initialize the ConflictError .
+
+        :param name: name of the parameter
+        :type name: str
+        :param msg: conflict reason
+        :type msg: str
+        """
+        self.message = "parameter '{}' conflicts".format(
+            name) + f": {msg}" if msg else ""
+        super().__init__(self.message)
+
+
+class NotLoggedInError(Exception):
+    """Exception indicating current user has not logged in yet
+    """
+
+    def __init__(self):
+        """Initialize the NotLoggedInError .
+        """
+        self.message = "not logged in yet, please call bioos.login to login"
+        super().__init__(self.message)

bioos/internal/__init__.py

@@ -0,0 +1 @@
+