pybioos 0.0.3__py3-none-any.whl

bioos/resource/workspaces.py

@@ -0,0 +1,123 @@
+from datetime import datetime
+
+import pandas as pd
+from cachetools import TTLCache, cached
+
+from bioos.config import Config
+from bioos.resource.data_models import DataModelResource
+from bioos.resource.files import FileResource
+from bioos.resource.workflows import Workflow, WorkflowResource
+from bioos.utils.common_tools import SingletonType, dict_str
+
+
+class Workspace(metaclass=SingletonType):
+
+    def __init__(self, id_: str):
+        self._id = id_
+        self._bucket = None
+
+    def __repr__(self) -> str:
+        return f"WorkspaceID: {self._id}\n\n" \
+               f"BasicInfo:\n{dict_str(self.basic_info)}\n\n" \
+               f"EnvInfo:\n{self.env_info}"
+
+    @property
+    @cached(cache=TTLCache(maxsize=10, ttl=1))  #这里为了做缓存，同时定义property，调用时生效
+    def basic_info(self) -> dict:
+        """ Returns basic info of the workspace
+
+        :return: Basic info
+        :rtype: dict
+        """
+        workspace_infos = Config.service().list_workspaces(
+            {  #这里是BioOsService对象的list_workspaces方法，非对外暴露的全局方法
+                "Filter": {
+                    "IDs": [self._id]
+                }
+            }).get("Items")
+        if len(workspace_infos) != 1:
+            return {}
+        workspace_info = workspace_infos[0]
+        s3_bucket = workspace_info.get("S3Bucket")
+        create_time = datetime.fromtimestamp(workspace_info.get("CreateTime"))
+        owner = workspace_info.get("OwnerName")
+        description = workspace_info.get("Description")
+        name = workspace_info.get("Name")
+        return {
+            "name": name,
+            "description": description,
+            "s3_bucket": s3_bucket,
+            "owner": owner,
+            "create_time": create_time
+        }
+
+    @property
+    @cached(cache=TTLCache(maxsize=10, ttl=1))
+    def env_info(self) -> pd.DataFrame:
+        """ Returns cluster info of the workspace
+
+        :return: Cluster info
+        :rtype: pandas.DataFrame
+        """
+        notebook_env_info = Config.service().list_cluster(params={
+            'Type': "notebook",
+            "ID": self._id
+        })
+        workflow_env_info = Config.service().list_cluster(params={
+            'Type': "workflow",
+            "ID": self._id
+        })
+        res = []
+        for cluster in notebook_env_info.get('Items') + workflow_env_info.get(
+                'Items'):
+            info = cluster["ClusterInfo"]
+            if info['Status'] == "Running":
+                res.append({
+                    "cluster_id": info['ID'],
+                    "name": info['Name'],
+                    "description": info['Description'],
+                    "type": cluster["Type"]
+                })
+        return pd.DataFrame(res)
+
+    @property
+    def data_models(self) -> DataModelResource:
+        """Returns DataModelResource object .
+
+        :return: DataModelResource object
+        :rtype: DataModelResource
+        """
+        return DataModelResource(self._id)
+
+    @property
+    def workflows(self) -> WorkflowResource:
+        """Returns WorkflowResource object.
+
+        :return: WorkflowResource object
+        :rtype: WorkflowResource
+        """
+        return WorkflowResource(self._id)
+
+    @property
+    def files(self) -> FileResource:
+        """Returns FileResource object .
+
+        :return: FileResource object
+        :rtype: FileResource
+        """
+        if not self._bucket:
+            self._bucket = self.basic_info.get("s3_bucket")
+
+        return FileResource(self._id, self._bucket)
+
+    def workflow(self, name: str) -> Workflow:  # 通过这里执行的选择workflow生成wf的操作
+        """Returns the workflow for the given name
+
+        :param name: Workflow name
+        :type name: str
+        :return: Specified workflow object
+        :rtype: Workflow
+        """
+        if not self._bucket:
+            self._bucket = self.basic_info.get("s3_bucket")
+        return Workflow(name, self._id, self._bucket)

bioos/service/BioOsService.py

@@ -0,0 +1,191 @@
+# coding:utf-8
+import json
+import threading
+from urllib.parse import urlparse
+
+from volcengine.ApiInfo import ApiInfo
+from volcengine.base.Service import Service
+from volcengine.Credentials import Credentials
+from volcengine.ServiceInfo import ServiceInfo
+
+from bioos.errors import ParameterError
+
+
+class BioOsService(Service):
+    _instance_lock = threading.Lock()
+
+    def __new__(cls, *args, **kwargs):
+        if not hasattr(BioOsService, '_instance'):
+            with BioOsService._instance_lock:
+                if not hasattr(BioOsService, '_instance'):
+                    BioOsService._instance = object.__new__(cls)
+        return BioOsService._instance
+
+    def __init__(self, endpoint, region):
+        self.service_info = BioOsService.get_service_info(endpoint, region)
+        self.api_info = BioOsService.get_api_info()
+        super(BioOsService, self).__init__(self.service_info, self.api_info)
+
+    @staticmethod
+    def get_service_info(endpoint, region):
+        parsed = urlparse(endpoint)
+        scheme, hostname = parsed.scheme, parsed.hostname
+        if not scheme or not hostname:
+            raise ParameterError("ENDPOINT")
+        service_info = ServiceInfo(hostname, {'Accept': 'application/json'},
+                                   Credentials('', '', 'bio', region),
+                                   5,
+                                   5,
+                                   scheme=scheme)
+        return service_info
+
+    @staticmethod
+    def get_api_info():  #新增加的API接口可以配置到这里
+        api_info = {
+            'ListWorkspaces':
+            ApiInfo('POST', '/', {
+                'Action': 'ListWorkspaces',
+                'Version': '2021-03-04'
+            }, {}, {}),
+            'CreateDataModel':
+            ApiInfo('POST', '/', {
+                'Action': 'CreateDataModel',
+                'Version': '2021-03-04'
+            }, {}, {}),
+            'ListDataModels':
+            ApiInfo('POST', '/', {
+                'Action': 'ListDataModels',
+                'Version': '2021-03-04'
+            }, {}, {}),
+            'ListDataModelRows':
+            ApiInfo('POST', '/', {
+                'Action': 'ListDataModelRows',
+                'Version': '2021-03-04'
+            }, {}, {}),
+            'ListAllDataModelRowIDs':
+            ApiInfo('POST', '/', {
+                'Action': 'ListAllDataModelRowIDs',
+                'Version': '2021-03-04'
+            }, {}, {}),
+            'DeleteDataModelRowsAndHeaders':
+            ApiInfo('POST', '/', {
+                'Action': 'DeleteDataModelRowsAndHeaders',
+                'Version': '2021-03-04'
+            }, {}, {}),
+            'ListWorkflows':
+            ApiInfo('POST', '/', {
+                'Action': 'ListWorkflows',
+                'Version': '2021-03-04'
+            }, {}, {}),
+            'CreateSubmission':
+            ApiInfo('POST', '/', {
+                'Action': 'CreateSubmission',
+                'Version': '2021-03-04'
+            }, {}, {}),
+            'ListSubmissions':
+            ApiInfo('POST', '/', {
+                'Action': 'ListSubmissions',
+                'Version': '2021-03-04'
+            }, {}, {}),
+            'DeleteSubmission':
+            ApiInfo('POST', '/', {
+                'Action': 'DeleteSubmission',
+                'Version': '2021-03-04'
+            }, {}, {}),
+            'ListRuns':
+            ApiInfo('POST', '/', {
+                'Action': 'ListRuns',
+                'Version': '2021-03-04'
+            }, {}, {}),
+            'ListTasks':
+            ApiInfo('POST', '/', {
+                'Action': 'ListTasks',
+                'Version': '2021-03-04'
+            }, {}, {}),
+            'GetTOSAccess':
+            ApiInfo('POST', '/', {
+                'Action': 'GetTOSAccess',
+                'Version': '2021-03-04'
+            }, {}, {}),
+            'ListClustersOfWorkspace':
+            ApiInfo('POST', '/', {
+                'Action': 'ListClustersOfWorkspace',
+                'Version': '2021-03-04'
+            }, {}, {}),
+            'CreateWorkflow':
+            ApiInfo('POST', '/', {
+                'Action': 'CreateWorkflow',
+                'Version': '2021-03-04'
+            }, {}, {}),
+            'CheckCreateWorkflow':
+            ApiInfo('POST', '/', {
+                'Action': 'CheckCreateWorkflow',
+                'Version': '2021-03-04'
+            }, {}, {}),
+            'DeleteWorkflow':
+            ApiInfo('POST', '/', {
+                'Action': 'DeleteWorkflow',
+                'Version': '2021-03-04'
+            }, {}, {}),
+        }
+        return api_info
+
+    def list_workspaces(self, params):  # 以下各方法的params需要在外部使用时构建
+        return self.__request('ListWorkspaces', params)
+
+    def create_data_model(self, params):
+        return self.__request('CreateDataModel', params)
+
+    def list_data_models(self, params):
+        return self.__request('ListDataModels', params)
+
+    def list_data_model_rows(self, params):
+        return self.__request('ListDataModelRows', params)
+
+    def list_data_model_row_ids(self, params):
+        return self.__request('ListAllDataModelRowIDs', params)
+
+    def delete_data_model_rows_and_headers(self, params):
+        return self.__request('DeleteDataModelRowsAndHeaders', params)
+
+    def list_workflows(self, params):
+        return self.__request('ListWorkflows', params)
+
+    def create_submission(self, params):
+        return self.__request('CreateSubmission', params)
+
+    def list_submissions(self, params):
+        return self.__request('ListSubmissions', params)
+
+    def delete_submission(self, params):
+        return self.__request('DeleteSubmission', params)
+
+    def list_runs(self, params):
+        return self.__request('ListRuns', params)
+
+    def list_tasks(self, params):
+        return self.__request('ListTasks', params)
+
+    def get_tos_access(self, params):
+        return self.__request('GetTOSAccess', params)
+
+    def list_cluster(self, params):
+        return self.__request('ListClustersOfWorkspace', params)
+
+    def create_workflow(self, params):
+        return self.__request('CreateWorkflow', params)
+
+    def check_workflow(self, params):
+        return self.__request("CheckCreateWorkflow", params)
+
+    def delete_workflow(self, params):
+        return self.__request("DeleteWorkflow", params)
+
+    def __request(self, action, params):
+        res = self.json(
+            action, dict(),
+            json.dumps(params))  # 这里的json来源于其父类，是字节Service中的函数。处理的是和后端交互后的返回值
+        if res == '':
+            raise Exception('empty response')
+        res_json = json.loads(res)
+        return res_json['Result']

bioos/service/__init__.py

@@ -0,0 +1 @@
+

bioos/service/api.py

@@ -0,0 +1,291 @@
+# coding:utf-8
+import csv
+import json
+import os
+import re
+from datetime import datetime
+
+from bioos.errors import NotFoundError, ParameterError
+from bioos.service.config import BioOsServiceConfig as conf
+from bioos.utils import workflows
+
+
+def __set_env():
+    conf.set_env()
+
+
+def set_credential(accesskey, secret):
+    conf.set_access_key(accesskey)
+    conf.set_secret_key(secret)
+    conf.set_env()
+
+
+def upload_entity_table(csvfile, table_name=None):
+    __set_env()
+    if not table_name:
+        table_name = re.sub('\.csv$', '', os.path.basename(csvfile))
+    headers = None
+    rows = []
+    with open(csvfile, 'rt', newline='') as f:
+        reader = csv.reader(f)
+        for row in reader:
+            if headers is None:
+                headers = row
+            else:
+                rows.append(row)
+    params = {
+        'WorkspaceID': conf.workspace_id(),
+        'Name': table_name,
+        'Headers': headers,
+        'Rows': rows
+    }
+    conf.service().create_data_model(params)
+
+
+def list_entity_tables():
+    __set_env()
+    params = {'WorkspaceID': conf.workspace_id()}
+    result = conf.service().list_data_models(params)
+    entities = []
+    for table in result['Items']:
+        if table['Type'] == 'normal':
+            entities.append(table)
+    return entities
+
+
+def get_entity_table(table_name, page_number=1, page_size=10):
+    l = list_entity_tables()
+    res = None
+    for table in l:
+        if table['Name'] == table_name:
+            res = table
+    if not res:
+        return None
+    params = {
+        'WorkspaceID': conf.workspace_id(),
+        'ID': res['ID'],
+        'PageNumber': page_number,
+        'PageSize': page_size
+    }
+    content = conf.service().list_data_model_rows(params)
+    res['Headers'] = content['Headers']
+    res['Rows'] = content['Rows']
+    res['RowTotalCount'] = content['TotalCount']
+    return res
+
+
+def delete_entity_table(table_name):
+    __set_env()
+    table = get_entity_table(table_name, page_size=1)
+    if not table:
+        raise NotFoundError('Table', table_name)
+    params = {'WorkspaceID': conf.workspace_id(), 'ID': table['ID']}
+    ids = conf.service().list_data_model_row_ids(params)
+    params = {
+        'WorkspaceID': conf.workspace_id(),
+        'ID': table['ID'],
+        'RowIDs': ids['RowIDs']
+    }
+    conf.service().delete_data_model_rows_and_headers(params)
+
+
+def delete_entities(table_name, row_ids):
+    if not row_ids:
+        return
+    __set_env()
+    table = get_entity_table(table_name, page_size=1)
+    if not table:
+        raise NotFoundError('Table', table_name)
+    params = {
+        'WorkspaceID': conf.workspace_id(),
+        'ID': table['ID'],
+        'RowIDs': row_ids
+    }
+    conf.service().delete_data_model_rows_and_headers(params)
+
+
+def delete_entity_table_headers(table_name, headers):
+    if not headers:
+        return
+    __set_env()
+    table = get_entity_table(table_name, page_size=1)
+    if not table:
+        raise NotFoundError('Table', table_name)
+    params = {
+        'WorkspaceID': conf.workspace_id(),
+        'ID': table['ID'],
+        'Headers': headers
+    }
+    conf.service().delete_data_model_rows_and_headers(params)
+
+
+def list_workflows(search_keyword=None, page_number=1, page_size=10):
+    __set_env()
+    params = {
+        'WorkspaceID': conf.workspace_id(),
+        'SortBy': 'CreateTime',
+        'PageNumber': page_number,
+        'PageSize': page_size
+    }
+    if search_keyword:
+        params['Filter'] = {'Keyword': search_keyword}
+    return conf.service().list_workflows(params).get('Items')
+
+
+def get_workflow(workflow_name):
+    lis = list_workflows(search_keyword=workflow_name)
+    if not lis:
+        return None
+    lis = [x for x in lis if x['Name'] == workflow_name]
+    if not lis:
+        return None
+    params = {
+        'WorkspaceID': conf.workspace_id(),
+        'Filter': {
+            'IDs': [lis[0].get('ID')]
+        }
+    }
+    workflows = conf.service().list_workflows(params).get('Items')
+    if len(workflows) != 1:
+        return None
+    detail = workflows[0]
+    res = lis[0]
+    for k, v in detail.items():
+        if k != 'Item':
+            res[k] = v
+    return res
+
+
+def add_submission(workflow_name,
+                   table_name,
+                   row_ids,
+                   cluster_id,
+                   inputs={},
+                   outputs={},
+                   submission_name_suffix=None,
+                   submission_desc=None,
+                   call_caching=True):
+    if not row_ids or not isinstance(row_ids, list):
+        raise ParameterError('row_ids')
+    if not inputs and not isinstance(inputs, dict):
+        raise ParameterError('inputs')
+    if not outputs and not isinstance(outputs, dict):
+        raise ParameterError('outputs')
+
+    table = get_entity_table(table_name, page_size=1)
+    if not table:
+        raise NotFoundError('Table', table_name)
+    workflow = get_workflow(workflow_name)
+    if not workflow:
+        raise NotFoundError('Workflow', workflow_name)
+    if not submission_name_suffix:
+        submission_name_suffix = datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
+    params = {
+        "ClusterID": cluster_id,
+        'WorkspaceID': conf.workspace_id(),
+        'WorkflowID': workflow['ID'],
+        'Name': submission_name(workflow_name, submission_name_suffix),
+        'Description': submission_desc,
+        'DataModelID': table['ID'],
+        'DataModelRowIDs': row_ids,
+        'Inputs': json.dumps(inputs),
+        'ExposedOptions': {
+            "ReadFromCache": call_caching
+        },
+        'Outputs': json.dumps(outputs)
+    }
+    conf.service().create_submission(params)
+
+
+def list_submissions(workflow_name=None,
+                     search_keyword=None,
+                     status=None,
+                     cluster_id=None,
+                     page_number=1,
+                     page_size=10):
+    __set_env()
+    params = {
+        'WorkspaceID': conf.workspace_id(),
+        'PageNumber': page_number,
+        'PageSize': page_size,
+        'Filter': {}
+    }
+    if workflow_name:
+        workflow = get_workflow(workflow_name)
+        if not workflow:
+            raise NotFoundError('Workflow', workflow_name)
+        params['Filter']['WorkflowID'] = workflow['ID']
+    if search_keyword:
+        params['Filter']['Keyword'] = search_keyword
+    if status:
+        params['Filter']['Status'] = status
+    if cluster_id:
+        params['Filter']['ClusterID'] = cluster_id
+
+    return conf.service().list_submissions(params).get('Items')
+
+
+def get_submission(submission_name):
+    lis = list_submissions(search_keyword=submission_name)
+    if not lis:
+        raise NotFoundError('Submission', submission_name)
+    lis = [x for x in lis if x['Name'] == submission_name]
+    if not lis or len(lis) < 1:
+        return None
+    submission = lis[0]
+    # list data entity rows by call list runs
+    runs = conf.service().list_runs({
+        'WorkspaceID': submission.get("WorkflowID"),
+        "SubmissionID": submission.get("ID"),
+    }).get("Items")
+    data_entity_row_ids = set()
+    for run in runs:
+        if run.get("DataEntityRowID") != "":
+            data_entity_row_ids.add(run.get("DataEntityRowID"))
+    # get data model name by call list data models
+    models = conf.service().list_data_models({
+        'WorkspaceID':
+        submission.get("WorkflowID"),
+    }).get("Items")
+    data_model = ""
+    for model in models:
+        if model["ID"] == submission.get["DataModelID"]:
+            data_model = model.get("Name")
+            break
+    submission["DataEntity"] = {
+        "ID": submission.get["DataModelID"],
+        "Name": data_model,
+        "RowIDs": list(data_entity_row_ids)
+    }
+
+    return submission
+
+
+def delete_submission(submission_name):
+    submission = get_submission(submission_name)
+    if not submission:
+        raise NotFoundError('Submission', submission_name)
+    params = {'WorkspaceID': conf.workspace_id(), 'ID': submission['ID']}
+    conf.service().delete_submission(params)
+
+
+def list_cluster(cluster_type):
+    if cluster_type not in ("notebook", "workflow"):
+        raise ParameterError("cluster_type")
+    params = {'Type': cluster_type, 'ID': conf.workspace_id()}
+    clusters = conf.service().list_cluster(params).get('Items')
+    res = []
+    for cluster in clusters:
+        info = cluster["ClusterInfo"]
+        if info['Status'] == "Running":
+            res.append({
+                "cluster_id": info['ID'],
+                "name": info['Name'],
+                "description": info['Description'],
+                "type": cluster["Type"]
+            })
+    return res
+
+
+def submission_name(workflow_name, submission_name_suffix):
+    return workflows.submission_name(workflow_name, submission_name_suffix)

bioos/service/config.py

@@ -0,0 +1,37 @@
+import os
+
+from bioos.config import Config
+from bioos.errors import EnvironmentConfigurationError
+
+
+def get_endpoint_env():
+    return os.environ.get('BIOOS_ENDPOINT')
+
+
+def get_workspace_id_env():
+    workspace_id = os.environ.get('BIOOS_WORKSPACE_ID')
+    if not workspace_id:
+        # use in MiracleCloud Notebook editor(jupyterhub)
+        workspace_id = os.environ.get('JUPYTERHUB_SERVER_NAME')
+    return workspace_id
+
+
+class BioOsServiceConfig(Config):
+    _workspace_id: str = None
+
+    @classmethod
+    def set_env(cls):
+        if cls._service is None:
+            endpoint = get_endpoint_env()
+            if not endpoint:
+                raise EnvironmentConfigurationError('BIOOS_ENDPOINT')
+            cls.set_endpoint(endpoint)
+
+        workspace_id = get_workspace_id_env()
+        if not workspace_id:
+            raise EnvironmentConfigurationError('BIOOS_WORKSPACE_ID')
+        cls._workspace_id = workspace_id
+
+    @classmethod
+    def workspace_id(cls):
+        return cls._workspace_id

bioos/tests/base.py

@@ -0,0 +1,21 @@
+import unittest
+from unittest.mock import patch
+
+from volcengine.const.Const import REGION_CN_NORTH1
+
+from bioos import bioos
+from bioos.config import Config
+
+
+class BaseInit(unittest.TestCase):
+    ak = "ak"
+    sk = "sk"
+    endpoint = "http://endpoint"
+    region = REGION_CN_NORTH1
+    workspace_id = "wccxxxxxxxxxxxxxno80"
+    really_login = bioos.login(endpoint, ak, sk, region)
+
+    def __init__(self, *args, **kwargs):
+        with patch.object(Config, "_ping_func"):
+            bioos.login(self.endpoint, self.ak, self.sk, self.region)
+            super(BaseInit, self).__init__(*args, **kwargs)