pybioos 0.0.3__py3-none-any.whl

bioos/tests/bioos.py

@@ -0,0 +1,43 @@
+from unittest import mock
+from unittest.mock import patch
+
+from pandas import DataFrame
+
+from bioos import bioos
+from bioos.config import Config
+from bioos.tests.base import BaseInit
+
+
+class TestBioOs(BaseInit):
+
+    def test_login_status(self):
+        status = bioos.status()
+        with patch.object(Config, "_ping_func") as success_ping:
+            self.assertIsInstance(status, Config.LoginInfo)
+            self.assertEqual(status.access_key, self.ak)
+            self.assertEqual(status.secret_key, self.sk)
+            self.assertEqual(status.endpoint, self.endpoint)
+            self.assertEqual(status.login_status, "Already logged in")
+        success_ping.assert_called_once()
+        with patch.object(Config, "_ping_func", side_effect=Exception(b'foo')) as fail_ping:
+            fake_ak = "aaa"
+            fake_sk = "bbb"
+            fake_endpoint = "http://fake.endpoint.com"
+            fake_region = "region"
+            bioos.login(fake_endpoint, fake_ak, fake_sk, fake_region)
+            status = bioos.status()
+            self.assertIsInstance(status, Config.LoginInfo)
+            self.assertEqual(status.access_key, fake_ak)
+            self.assertEqual(status.secret_key, fake_sk)
+            self.assertEqual(status.endpoint, fake_endpoint)
+            self.assertEqual(status.region, fake_region)
+            self.assertEqual(status.login_status, "Not logged in")
+        fail_ping.assert_has_calls([mock.call(), mock.call()])
+
+    def test_list_workspaces(self):
+        with patch.object(Config.service(), "list_workspaces",
+                          return_value={"Items": [{}]}) as success_list:
+            workspaces = bioos.list_workspaces()
+            self.assertIsInstance(workspaces, DataFrame)
+        success_list.assert_called_once()
+        success_list.assert_called_with({"PageSize": 0})

bioos/tests/data_models.py

@@ -0,0 +1,259 @@
+import copy
+from unittest import mock
+from unittest.mock import patch
+
+import pandas as pd
+import pandas.testing
+from pandas import DataFrame
+
+from bioos import bioos
+from bioos.errors import ConflictError, NotFoundError
+from bioos.service.BioOsService import BioOsService
+from bioos.tests.base import BaseInit
+
+
+class TestDataModel(BaseInit):
+    list_data_models_val = {'TotalCount': 9, 'Items': [
+        {'ID': 'dcblq1tteig44bng68od0', 'Name': 'jxc', 'RowCount': 162, 'Type': 'normal'},
+        {'ID': 'dccc0ne5eig41ascop420', 'Name': 'run', 'RowCount': 499, 'Type': 'normal'},
+        {'ID': 'dccc0nkleig41ascop42g', 'Name': 'sample', 'RowCount': 3, 'Type': 'normal'},
+        {'ID': 'dccc0nmteig41ascop430', 'Name': 'sample6', 'RowCount': 1922, 'Type': 'normal'},
+        {'ID': 'dccdaq2deig42s7rgs7j0', 'Name': 'sample_set', 'RowCount': 1, 'Type': 'set'},
+        {'ID': 'dccc0o0teig41ascop43g', 'Name': 'test', 'RowCount': 3000, 'Type': 'normal'},
+        {'ID': 'dccdaq6leig42s7rgs7jg', 'Name': 'test_set', 'RowCount': 1, 'Type': 'set'},
+        {'ID': 'dccc0o2teig41ascop440', 'Name': 'testaa', 'RowCount': 3000, 'Type': 'normal'},
+        {'ID': 'dcc6tbkleig4c9lddjt2g', 'Name': 'workspace_data', 'RowCount': 257788, 'Type': 'workspace'}]}
+    list_data_model_rows_val = \
+        {
+            'TotalCount': 3,
+            'Headers':
+                ['sample_id', 'column-1-file-CRAM', 'date',
+                 '123456789s_123456789s_123456789s_123456789s_', 'gg', 'hh', 'jj'],
+            'Rows': [
+                [
+                    'your-sample-1-id', 'OK', '01/01/2022', 'we啊', 'abc',
+                    's3://bioos-transfertest-testfake_workspace/analysis/scc9oqc5eig42lnksf7qg/test/b6d2bf67-760c-4c56-b5b0-30a2e08dc180/call-step1/execution/resp.txt',
+                    'abc'
+                ],
+                [
+                    'your-sample-2-id', 'OK', '01/01/2022', "s'd", 'abc',
+                    's3://bioos-transfertest-testfake_workspace/analysis/scc9oqc5eig42lnksf7qg/test/cccd6f44-a53b-4f77-8b0a-b72768aba55a/call-step1/execution/resp.txt',
+                    'abc'
+                ],
+                [
+                    'your-sample-3-id', 'OK', '01/01/2022', "s'd", 'abc',
+                    's3://bioos-transfertest-testfake_workspace/analysis/scc9oqc5eig42lnksf7qg/test/c86ce783-6dd7-47e1-88ba-e282e5b0d2c4/call-step1/execution/resp.txt',
+                    'abc'
+                ]
+            ]
+        }
+    list_data_model_rows_id = {
+        "RowIDs": [
+            "your-sample-1-id",
+            "your-sample-2-id",
+            "your-sample-changed"
+        ]
+    }
+
+    sample_data = pd.DataFrame(
+        {'sample_id': {0: 'your-sample-1-id', 1: 'your-sample-2-id', 2: 'your-sample-3-id'},
+         'column-1-file-CRAM': {0: 'OK', 1: 'OK', 2: 'OK'},
+         'date': {0: '01/01/2022', 1: '01/01/2022', 2: '01/01/2022'},
+         '123456789s_123456789s_123456789s_123456789s_': {0: 'we啊', 1: "s'd", 2: "s'd"},
+         'gg': {0: 'abc', 1: 'abc', 2: 'abc'}, 'hh': {
+            0: 's3://bioos-transfertest-testfake_workspace/analysis/scc9oqc5eig42lnksf7qg/test/b6d2bf67-760c-4c56-b5b0-30a2e08dc180/call-step1/execution/resp.txt',
+            1: 's3://bioos-transfertest-testfake_workspace/analysis/scc9oqc5eig42lnksf7qg/test/cccd6f44-a53b-4f77-8b0a-b72768aba55a/call-step1/execution/resp.txt',
+            2: 's3://bioos-transfertest-testfake_workspace/analysis/scc9oqc5eig42lnksf7qg/test/c86ce783-6dd7-47e1-88ba-e282e5b0d2c4/call-step1/execution/resp.txt'},
+         'jj': {0: 'abc', 1: 'abc', 2: 'abc'}})
+    sample_data_to_write = pd.DataFrame(
+        {'sample_id': {0: 'your-sample-1-id', 1: 'your-sample-2-id', 2: 'your-sample-changed'},
+         'column-1-file-CRAM': {0: 'OK', 1: 'OK', 2: 'OK'},
+         'date': {0: '01/01/2022', 1: '01/01/2022', 2: '01/01/2022'},
+         '123456789s_123456789s_123456789s_123456789s_': {0: 'we啊', 1: "s'd", 2: "s'd"},
+         'gg': {0: 'abc', 1: 'abc', 2: 'abc'}, 'hh': {
+            0: 's3://bioos-transfertest-testfake_workspace/analysis/scc9oqc5eig42lnksf7qg/test/b6d2bf67-760c-4c56-b5b0-30a2e08dc180/call-step1/execution/resp.txt',
+            1: 's3://bioos-transfertest-testfake_workspace/analysis/scc9oqc5eig42lnksf7qg/test/cccd6f44-a53b-4f77-8b0a-b72768aba55a/call-step1/execution/resp.txt',
+            2: 's3://bioos-transfertest-testfake_workspace/analysis/scc9oqc5eig42lnksf7qg/test/c86ce783-6dd7-47e1-88ba-e282e5b0d2c4/call-step1/execution/resp.txt'},
+         'jj': {0: 'abc', 1: 'abc', 2: 'abc'}})
+
+    def __init__(self, *args, **kwargs):
+        super(TestDataModel, self).__init__(*args, **kwargs)
+        self.data_models = bioos.workspace(self.workspace_id).data_models
+
+    def test_repr(self):
+        with patch.object(BioOsService, "list_data_models",
+                          return_value=self.list_data_models_val) as success_list:
+            # makes no call
+            self.data_models
+            # call list_data_models once
+            repr(self.data_models)
+            # makes another list_data_models call
+            pandas.testing.assert_frame_equal(self.data_models.list(), DataFrame.from_records([
+                {"ID": "dcblq1tteig44bng68od0", "Name": "jxc",
+                 "RowCount": 162, "Type": "normal"},
+                {"ID": "dccc0ne5eig41ascop420", "Name": "run",
+                 "RowCount": 499, "Type": "normal"},
+                {"ID": "dccc0nkleig41ascop42g", "Name": "sample",
+                 "RowCount": 3, "Type": "normal"},
+                {"ID": "dccc0nmteig41ascop430", "Name": "sample6",
+                 "RowCount": 1922, "Type": "normal"},
+                {"ID": "dccc0o0teig41ascop43g", "Name": "test",
+                 "RowCount": 3000, "Type": "normal"},
+                {"ID": "dccc0o2teig41ascop440", "Name": "testaa",
+                 "RowCount": 3000, "Type": "normal"},
+            ]))
+        success_list.assert_has_calls([
+            mock.call({
+                'WorkspaceID': self.workspace_id
+            }), mock.call({
+                'WorkspaceID': self.workspace_id
+            })
+        ])
+
+    def test_read_all(self):
+        with patch.object(BioOsService, "list_data_models",
+                          return_value=self.list_data_models_val) as success_list:
+            with patch.object(BioOsService, "list_data_model_rows",
+                              return_value={}) as success_rows_list:
+                self.data_models.read()
+        success_list.assert_called_once_with({
+            'WorkspaceID': self.workspace_id,
+        })
+        calls = []
+        for model in self.list_data_models_val.get("Items"):
+            if model.get("Type") == "normal":
+                calls.append(mock.call({
+                    'WorkspaceID': self.workspace_id,
+                    'ID': model["ID"],
+                    'PageSize': 0,
+                }))
+        success_rows_list.assert_has_calls(
+            calls
+        )
+
+    def test_crud(self):
+        new_sample = "new_sample"
+        new_sample_id = "acbq1tteig44bng68oa0"
+        data_to_write = {"sample": self.sample_data_to_write,
+                         new_sample: self.sample_data_to_write}
+        with patch.object(BioOsService, "delete_data_model_rows_and_headers") as success_delete:
+            with patch.object(BioOsService, "create_data_model",
+                              return_value={
+                                  "ID": new_sample_id
+                              }) as success_create:
+
+                with patch.object(BioOsService, "list_data_models",
+                                  return_value=self.list_data_models_val) as success_list:
+                    with patch.object(BioOsService, "list_data_model_rows",
+                                      return_value=self.list_data_model_rows_val) as success_rows_list:
+                        read_res = self.data_models.read("sample")
+                        self.assertEqual(len(read_res), 1)
+                        self.assertIsInstance(read_res, dict)
+                        pandas.testing.assert_frame_equal(read_res["sample"], self.sample_data)
+
+                        try:
+                            self.data_models.write(data_to_write, force=False)
+                        except ConflictError as e:
+                            self.assertEqual(e.message,
+                                             "parameter 'sources' conflicts: {'sample'} "
+                                             "already exists, pls use force=True to overwrite")
+                        success_create.assert_not_called()
+
+                        self.data_models.write(data_to_write, force=True)
+                        success_create.assert_has_calls([
+                            mock.call({
+                                'WorkspaceID': self.workspace_id,
+                                'Name': "sample",
+                                'Headers': list(self.sample_data_to_write.head()),
+                                'Rows': self.sample_data_to_write.values.tolist(),
+                            }),
+                            mock.call({
+                                'WorkspaceID': self.workspace_id,
+                                'Name': new_sample,
+                                'Headers': list(self.sample_data_to_write.head()),
+                                'Rows': self.sample_data_to_write.values.tolist(),
+                            })
+                        ])
+
+                success_list.assert_has_calls([
+                    mock.call({
+                        'WorkspaceID': self.workspace_id,
+                    }),
+                    mock.call({
+                        'WorkspaceID': self.workspace_id,
+                    }),
+                ])
+                success_rows_list.assert_called_once_with({
+                    'WorkspaceID': self.workspace_id,
+                    'ID': "dccc0nkleig41ascop42g",
+                    'PageSize': 0,
+                })
+
+                new_list_data_models_val = copy.deepcopy(self.list_data_models_val)
+                new_list_data_models_val["TotalCount"] = 10
+                new_list_data_models_val['Items'].append(
+                    {'ID': new_sample_id, 'Name': new_sample, 'RowCount': 3, 'Type': 'normal'})
+
+                new_list_data_model_rows_val = copy.deepcopy(self.list_data_model_rows_val)
+                new_list_data_model_rows_val['Rows'][2][0] = "your-sample-changed"
+                with patch.object(BioOsService, "list_data_models",
+                                  return_value=new_list_data_models_val) as success_list:
+                    with patch.object(BioOsService, "list_data_model_rows",
+                                      return_value=new_list_data_model_rows_val) as success_rows_list:
+                        with patch.object(BioOsService, "list_data_model_row_ids",
+                                          return_value=self.list_data_model_rows_id) as success_rows_id_list:
+                            sample_res = self.data_models.read("sample")
+                            self.assertEqual(len(sample_res), 1)
+                            self.assertIsInstance(sample_res, dict)
+                            pandas.testing.assert_frame_equal(sample_res["sample"],
+                                                              self.sample_data_to_write)
+
+                            new_sample_res = self.data_models.read(new_sample)
+                            self.assertEqual(len(new_sample_res), 1)
+                            self.assertIsInstance(new_sample_res, dict)
+                            pandas.testing.assert_frame_equal(sample_res["sample"],
+                                                              new_sample_res[new_sample])
+
+                            self.data_models.delete(new_sample)
+
+                success_list.assert_has_calls([
+                    mock.call({
+                        'WorkspaceID': self.workspace_id,
+                    }),
+                    mock.call({
+                        'WorkspaceID': self.workspace_id,
+                    }),
+                    mock.call({
+                        'WorkspaceID': self.workspace_id,
+                    })
+                ])
+                success_rows_list.assert_has_calls([
+                    mock.call({
+                        'WorkspaceID': self.workspace_id,
+                        'ID': "dccc0nkleig41ascop42g",
+                        'PageSize': 0,
+                    }),
+                    mock.call({
+                        'WorkspaceID': self.workspace_id,
+                        'ID': new_sample_id,
+                        'PageSize': 0,
+                    }),
+                ])
+                success_rows_id_list.assert_called_once_with({
+                    'WorkspaceID': self.workspace_id,
+                    'ID': new_sample_id,
+                })
+                success_delete.assert_called_once_with({
+                    'WorkspaceID': self.workspace_id,
+                    'ID': new_sample_id,
+                    'RowIDs': ['your-sample-1-id', 'your-sample-2-id', 'your-sample-changed']
+                })
+
+                new_list_data_models_val["TotalCount"] = 9
+                new_list_data_models_val["Items"] = new_list_data_models_val["Items"][:-1]
+                with patch.object(BioOsService, "list_data_models",
+                                  return_value=new_list_data_models_val) as success_list:
+                    try:
+                        self.data_models.delete(new_sample)
+                    except NotFoundError as e:
+                        self.assertEqual(e.message, f"target '{new_sample}' not found")

bioos/tests/files.py

@@ -0,0 +1,174 @@
+import copy
+import hashlib
+import os
+import random
+import shutil
+import time
+import unittest
+
+from pandas import DataFrame
+
+from bioos.bioos import workspace
+from bioos.tests.base import BaseInit
+
+
+class TestFiles(BaseInit):
+    tos_target_dir = f"test_upload_files-{time.time()}"
+    local_small_files_dir = "small_files/"
+    local_downloads_files_dir = "downloads/"
+    small_files = {}
+    big_file = {}
+    small_num = 5
+    big_num = 1
+
+    @classmethod
+    def get_file_md5(cls, file_path):
+        with open(file_path, 'rb') as f:
+            data = f.read()
+        return hashlib.md5(data).hexdigest()
+
+    @classmethod
+    def generate_random_str(cls, length=16):
+        """
+            生成一个指定长度的随机字符串
+            """
+        random_str = ''
+        base_str = 'abcdefghigklmnopqrstuvwxyz'
+        seed_len = len(base_str) - 1
+        for i in range(length):
+            random_str += base_str[random.randint(0, seed_len)]
+        return random_str
+
+    @classmethod
+    def generate_random_file(cls, size, cur_dir="./"):
+        """
+            生成临时文件。
+        """
+        if not os.path.exists(cur_dir):
+            os.mkdir(cur_dir)
+        tmp_file = os.path.join(cur_dir, cls.generate_random_str(8))
+        with open(tmp_file, 'wb') as f:
+            while size > 0:
+                sz = int(min(size, 5 * 1024 * 1024))
+                data = os.urandom(sz)
+                f.write(data)
+                size -= sz
+            f.close()
+        return tmp_file
+
+    def setUp(self):
+        self.files = workspace(self.workspace_id).files
+        if not self.really_login:
+            return
+        # create 5 small files and 1 big file
+        for i in range(self.small_num):
+            small_file = os.path.normpath(
+                self.generate_random_file(5 * 1024 * 1024, self.local_small_files_dir))
+            self.small_files[small_file] = self.get_file_md5(small_file)
+        big_file = os.path.normpath(self.generate_random_file(200 * 1024 * 1024))
+        self.big_file[big_file] = self.get_file_md5(big_file)
+
+    def tearDown(self):
+        for f in self.big_file:
+            os.remove(f)
+        if os.path.exists(self.local_small_files_dir):
+            shutil.rmtree(self.local_small_files_dir)
+        if os.path.exists(self.local_downloads_files_dir):
+            shutil.rmtree(self.local_downloads_files_dir)
+        TestFiles.small_files = {}
+        TestFiles.big_file = {}
+
+    @unittest.skipUnless(BaseInit.really_login, "need real ak,sk,endpoint,"
+                                                "workspace_id")
+    def test_1_crud_case(self):
+        files = copy.deepcopy(self.small_files)
+        files.update(self.big_file)
+        self.files.upload(files, self.tos_target_dir, flatten=True)
+
+        list_df = self.files.list()
+        self.assertIsInstance(list_df, DataFrame)
+        uploaded_files_in_tos = \
+            list(list_df.query(f"key.str.startswith('{self.tos_target_dir}')")["key"])
+        self.assertEqual(len(uploaded_files_in_tos), self.small_num + self.big_num)
+
+        for f in uploaded_files_in_tos:
+            # local_dir will not be mapped to tos
+            self.assertNotIn(f"{self.tos_target_dir}/{self.local_small_files_dir}", f)
+
+        self.files.download([os.path.join(self.tos_target_dir, os.path.basename(f)) for f in files],
+                            self.local_downloads_files_dir,
+                            flatten=True)
+        md5_set = set()
+        for root, dirs, files in os.walk(self.local_downloads_files_dir):
+            self.assertEqual(len(files), self.big_num + self.small_num)
+            self.assertEqual(len(dirs), 0)
+            for name in files:
+                md5_set.add(self.get_file_md5(os.path.join(root, name)))
+        self.assertSetEqual(md5_set,
+                            set(self.big_file.values()).union(set(self.small_files.values())))
+
+        # clean tos files
+        for f in uploaded_files_in_tos:
+            self.files.delete(f)
+
+        list_df = self.files.list()
+        files_in_tos_after_delete = list_df.query(f"key.str.startswith('{self.tos_target_dir}')")[
+            "key"]
+        self.assertEqual(len(files_in_tos_after_delete), 0)
+
+        # clean downloads files
+        shutil.rmtree(self.local_downloads_files_dir)
+
+    @unittest.skipUnless(BaseInit.really_login, "need real ak,sk,endpoint,"
+                                                "workspace_id")
+    def test_2_crud_case(self):
+        files = copy.deepcopy(self.small_files)
+        files.update(self.big_file)
+        self.files.upload(files, self.tos_target_dir, flatten=False)
+
+        list_df = self.files.list()
+        self.assertIsInstance(list_df, DataFrame)
+        uploaded_files_in_tos = \
+            list(list_df.query(f"key.str.startswith('{self.tos_target_dir}')")["key"])
+        self.assertEqual(len(uploaded_files_in_tos), self.big_num + self.small_num)
+
+        for f in uploaded_files_in_tos:
+            # make sure that all small files are in the directory
+            if os.path.basename(f) not in files:
+                self.assertIn(f"{self.tos_target_dir}/{self.local_small_files_dir}", f)
+            # make sure that the only one big file matches
+            else:
+                self.assertEqual(f"{self.tos_target_dir}/{tuple(self.big_file.keys())[0]}", f)
+
+        self.files.download([os.path.join(self.tos_target_dir, f) for f in files],
+                            self.local_downloads_files_dir,
+                            flatten=False)
+        md5_set = set()
+        for root, dirs, files in os.walk(self.local_downloads_files_dir):
+            # dir for big_file
+            if os.path.samefile(root,
+                                os.path.join(self.local_downloads_files_dir, self.tos_target_dir)):
+                self.assertEqual(len(files), self.big_num)
+            # dir for small_files
+            elif os.path.samefile(root, os.path.join(self.local_downloads_files_dir,
+                                                     os.path.join(self.tos_target_dir,
+                                                                  self.local_small_files_dir))):
+                self.assertEqual(len(files), self.small_num)
+            else:
+                self.assertEqual(len(files), 0)
+            for name in files:
+                md5_set.add(self.get_file_md5(os.path.join(root, name)))
+        self.assertSetEqual(md5_set,
+                            set(self.big_file.values()).union(set(self.small_files.values())))
+
+        # clean tos files
+        for f in uploaded_files_in_tos:
+            self.files.delete(f)
+
+        list_df = self.files.list()
+        files_in_tos_after_delete = list_df.query(f"key.str.startswith('{self.tos_target_dir}')")[
+            "key"]
+        self.assertEqual(len(files_in_tos_after_delete), 0)
+
+        # clean downloads files
+        shutil.rmtree(self.local_downloads_files_dir)

bioos/tests/utils.py

@@ -0,0 +1,68 @@
+import threading
+import time
+import unittest
+
+from bioos.utils.common_tools import SingletonType
+
+
+class Foo(metaclass=SingletonType):
+    def __init__(self, name):
+        time.sleep(1)
+        self.name = name
+
+
+class MyThread(threading.Thread):
+    def __init__(self, func, args, name=''):
+        threading.Thread.__init__(self)
+        self.func = func
+        self.name = name
+        self.args = args
+        self.res = None
+
+    def run(self):
+        self.res = self.func(*self.args)
+
+    def get_res(self):
+        return self.res
+
+
+class TestUtils(unittest.TestCase):
+    @classmethod
+    def init_foo(cls, arg):
+        obj = Foo(str(arg))
+        return obj
+
+    def test_singletonType_diff_obj(self):
+
+        threads = []
+        for i in range(10):
+            threads.append(MyThread(func=self.init_foo, args=(i,), name=f'diff_obj {i}'))
+
+        for t in threads:
+            t.start()
+
+        for t in threads:
+            t.join()
+
+        res = set()
+        for t in threads:
+            res.add(t.get_res())
+        self.assertEqual(len(res), 10)
+
+    def test_singletonType_same_obj(self):
+
+        threads = []
+        for i in range(100):
+            threads.append(MyThread(func=self.init_foo, args=(0,), name=f'same_obj {i}'))
+
+        for t in threads:
+            t.start()
+
+        for t in threads:
+            t.join()
+
+        res = set()
+        for t in threads:
+            res.add(t.get_res())
+
+        self.assertEqual(len(res), 1)