cloudos-cli 2.17.0__tar.gz → 2.18.0__tar.gz

{cloudos_cli-2.17.0 → cloudos_cli-2.18.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: cloudos_cli
-Version: 2.17.0
+Version: 2.18.0
 Summary: Python package for interacting with CloudOS
 Home-page: https://github.com/lifebit-ai/cloudos-cli
 Author: David Piñeyro
@@ -231,7 +231,7 @@ CLOUDOS="https://cloudos.lifebit.ai"
 WORKSPACE_ID="xxxxx"
 PROJECT_NAME="API jobs"
 WORKFLOW_NAME="rnatoy"
-JOB_PARAMS="cloudos/examples/rnatoy.config"
+JOB_PARAMS="cloudos_cli/examples/rnatoy.config"
 ```
 
 As you can see, a file with the job parameters is used to configure the

{cloudos_cli-2.17.0 → cloudos_cli-2.18.0}/README.md

@@ -198,7 +198,7 @@ CLOUDOS="https://cloudos.lifebit.ai"
 WORKSPACE_ID="xxxxx"
 PROJECT_NAME="API jobs"
 WORKFLOW_NAME="rnatoy"
-JOB_PARAMS="cloudos/examples/rnatoy.config"
+JOB_PARAMS="cloudos_cli/examples/rnatoy.config"
 ```
 
 As you can see, a file with the job parameters is used to configure the

{cloudos_cli-2.17.0 → cloudos_cli-2.18.0}/cloudos_cli/__main__.py

@@ -807,6 +807,10 @@ def list_jobs(apikey,
         print('\t' + str(cl) + '\n')
         print('\tSearching for jobs in the following workspace: ' +
               f'{workspace_id}')
+    # Check if the user provided the --page option
+    ctx = click.get_current_context()
+    if not isinstance(page, int) or page < 1:
+        raise ValueError('Please, use a positive integer (>= 1) for the --page parameter')
     if last_n_jobs != 'all':
         try:
             last_n_jobs = int(last_n_jobs)
@@ -814,7 +818,12 @@ def list_jobs(apikey,
             print("[ERROR] last-n-jobs value was not valid. Please use a positive int or 'all'")
             raise
     my_jobs_r = cl.get_job_list(workspace_id, last_n_jobs, page, archived, verify_ssl)
-    if output_format == 'csv':
+    if len(my_jobs_r) == 0:
+        if ctx.get_parameter_source('page') == click.core.ParameterSource.DEFAULT:
+            print('\t[Message] A total of 0 jobs collected. This is likely because your workspace has no jobs created yet.')
+        else:
+            print('\t[Message] A total of 0 jobs collected. This is likely because the --page you requested does not exist. Please, try a smaller number for --page or collect all the jobs by not using --page parameter.')
+    elif output_format == 'csv':
         my_jobs = cl.process_job_list(my_jobs_r, all_fields)
         my_jobs.to_csv(outfile, index=False)
         print(f'\tJob list collected with a total of {my_jobs.shape[0]} jobs.')
@@ -997,6 +1006,10 @@ def import_workflows(apikey,
                     'just the preconfigured selected fields. Only applicable ' +
                     'when --output-format=csv'),
               is_flag=True)
+@click.option('--page',
+              help=('Response page to retrieve. Default=1.'),
+              type=int,
+              default=1)
 @click.option('--verbose',
               help='Whether to print information messages or not.',
               is_flag=True)
@@ -1012,6 +1025,7 @@ def list_projects(apikey,
                   output_basename,
                   output_format,
                   all_fields,
+                  page,
                   verbose,
                   disable_ssl_verification,
                   ssl_cert):
@@ -1028,21 +1042,28 @@ def list_projects(apikey,
         print('\t' + str(cl) + '\n')
         print('\tSearching for projects in the following workspace: ' +
               f'{workspace_id}')
-    my_projects_r = cl.get_project_list(workspace_id, verify_ssl)
-    if output_format == 'csv':
+    # Check if the user provided the --page option
+    ctx = click.get_current_context()
+    if ctx.get_parameter_source('page') == click.core.ParameterSource.DEFAULT:
+        get_all = True
+    else:
+        get_all = False
+        if not isinstance(page, int) or page < 1:
+            raise ValueError('Please, use a positive integer (>= 1) for the --page parameter')
+    my_projects_r = cl.get_project_list(workspace_id, verify_ssl, page=page, get_all=get_all)
+    if len(my_projects_r) == 0:
+        if ctx.get_parameter_source('page') == click.core.ParameterSource.DEFAULT:
+            print('\t[Message] A total of 0 projects collected. This is likely because your workspace has no projects created yet.')
+        else:
+            print('\t[Message] A total of 0 projects collected. This is likely because the --page you requested does not exist. Please, try a smaller number for --page or collect all the projects by not using --page parameter.')
+    elif output_format == 'csv':
         my_projects = cl.process_project_list(my_projects_r, all_fields)
         my_projects.to_csv(outfile, index=False)
         print(f'\tProject list collected with a total of {my_projects.shape[0]} projects.')
     elif output_format == 'json':
         with open(outfile, 'w') as o:
-            o.write(my_projects_r.text)
-        content = json.loads(my_projects_r.content)
-        if 'projects' in content:
-            content_l = len(content['projects'])
-        else:
-            content_l = len(content)
-        print('\tProject list collected with a total of ' +
-              f'{content_l} projects.')
+            o.write(json.dumps(my_projects_r))
+        print(f'\tProject list collected with a total of {len(my_projects_r)} projects.')
     else:
         raise ValueError('Unrecognised output format. Please use one of [csv|json]')
     print(f'\tProject list saved to {outfile}')

cloudos_cli-2.18.0/cloudos_cli/_version.py

@@ -0,0 +1 @@
+__version__ = '2.18.0'

{cloudos_cli-2.17.0 → cloudos_cli-2.18.0}/cloudos_cli/clos.py

@@ -313,6 +313,8 @@ class Cloudos:
                    'project.updatedAt'
                    ]
         df_full = pd.json_normalize(r)
+        if df_full.empty:
+            return df_full
         if all_fields:
             df = df_full
         else:
@@ -548,7 +550,8 @@ class Cloudos:
         else:
             return False
 
-    def get_project_list(self, workspace_id, verify=True):
+    def get_project_list(self, workspace_id, verify=True, get_all=True,
+                        page=1, page_size=10, max_page_size=1000):
         """Get all the project from a CloudOS workspace.
 
         Parameters
@@ -559,6 +562,15 @@ class Cloudos:
             Whether to use SSL verification or not. Alternatively, if
             a string is passed, it will be interpreted as the path to
             the SSL certificate file.
+        get_all : bool
+            Whether to get all available curated workflows or just the
+            indicated page.
+        page : int
+            The page number to retrieve, from the paginated response.
+        page_size : int
+            The number of workflows by page. From 1 to 1000.
+        max_page_size : int
+            Max page size defined by the API server. It is currently 1000.
 
         Returns
         -------
@@ -569,11 +581,36 @@ class Cloudos:
             "Content-type": "application/json",
             "apikey": self.apikey
         }
-        r = retry_requests_get("{}/api/v1/projects?teamId={}".format(self.cloudos_url, workspace_id),
+        r = retry_requests_get("{}/api/v2/projects?teamId={}&pageSize={}&page={}".format(self.cloudos_url, workspace_id, page_size, page),
                                headers=headers, verify=verify)
         if r.status_code >= 400:
             raise BadRequestException(r)
-        return r
+        content = json.loads(r.content)
+        if get_all:
+            total_projects = content['total']
+            if total_projects <= max_page_size:
+                r = retry_requests_get("{}/api/v2/projects?teamId={}&pageSize={}&page={}".format(self.cloudos_url, workspace_id, total_projects, 1),
+                               headers=headers, verify=verify)
+                if r.status_code >= 400:
+                    raise BadRequestException(r)
+                return json.loads(r.content)['projects']
+            else:
+                n_pages = (total_projects // max_page_size) + int((total_projects % max_page_size) > 0)
+                for p in range(n_pages):
+                    p += 1
+                    r = retry_requests_get(
+                        "{}/api/v2/projects?teamId={}&pageSize={}&page={}".format(
+                            self.cloudos_url, workspace_id, max_page_size, p),
+                        headers=headers, verify=verify)
+                    if r.status_code >= 400:
+                        raise BadRequestException(r)
+                    if p == 1:
+                        all_content_p = json.loads(r.content)['projects']
+                    else:
+                        all_content_p += json.loads(r.content)['projects']
+                return all_content_p
+        else:
+            return content['projects']
 
     @staticmethod
     def process_project_list(r, all_fields=False):
@@ -582,10 +619,7 @@ class Cloudos:
         Parameters
         ----------
         r : requests.models.Response
-        The server response. There are two types of responses:
-            - A list with 2 elements: 'total' and 'projects', being 'projects' a list of dicts,
-              one for each project.
-            - A list of dicts, one for each project.
+            A list of dicts, each corresponding to a project.
         all_fields : bool. Default=False
             Whether to return a reduced version of the DataFrame containing
             only the selected columns or the full DataFrame.
@@ -607,10 +641,9 @@ class Cloudos:
                    'jobCount',
                    'notebookSessionCount'
                    ]
-        my_projects = json.loads(r.content)
-        if 'projects' in my_projects:
-            my_projects = my_projects['projects']
-        df_full = pd.json_normalize(my_projects)
+        df_full = pd.json_normalize(r)
+        if df_full.empty:
+            return df_full
         if all_fields:
             df = df_full
         else:

{cloudos_cli-2.17.0 → cloudos_cli-2.18.0}/cloudos_cli/jobs/job.py

@@ -157,18 +157,11 @@ class Job(Cloudos):
                         elif "importsFile" in element.keys() and element["importsFile"] == importsfile:
                             return element["_id"]
         elif resource == 'projects':
-            r = self.get_project_list(workspace_id, verify=verify)
-            content = json.loads(r.content)
+            content = self.get_project_list(workspace_id, verify=verify)
             # New API projects endpoint spec
-            if type(content) is dict:
-                for element in content["projects"]:
-                    if element["name"] == name:
-                        return element["_id"]
-            # Old API projects endpoint spec added for backwards compatibility
-            elif type(content) is list:
-                for element in content:
-                    if element["name"] == name:
-                        return element["_id"]
+            for element in content:
+                if element["name"] == name:
+                    return element["_id"]
         if mainfile is not None:
             raise ValueError(f'[ERROR] A workflow named \'{name}\' with a mainFile \'{mainfile}\'' +
                              f' and an importsFile \'{importsfile}\' was not found')

{cloudos_cli-2.17.0 → cloudos_cli-2.18.0}/cloudos_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: cloudos_cli
-Version: 2.17.0
+Version: 2.18.0
 Summary: Python package for interacting with CloudOS
 Home-page: https://github.com/lifebit-ai/cloudos-cli
 Author: David Piñeyro
@@ -231,7 +231,7 @@ CLOUDOS="https://cloudos.lifebit.ai"
 WORKSPACE_ID="xxxxx"
 PROJECT_NAME="API jobs"
 WORKFLOW_NAME="rnatoy"
-JOB_PARAMS="cloudos/examples/rnatoy.config"
+JOB_PARAMS="cloudos_cli/examples/rnatoy.config"
 ```
 
 As you can see, a file with the job parameters is used to configure the

{cloudos_cli-2.17.0 → cloudos_cli-2.18.0}/tests/test_clos/test_get_project_list.py

@@ -6,11 +6,14 @@ import responses
 from responses import matchers
 from cloudos_cli.clos import Cloudos
 from cloudos_cli.utils.errors import BadRequestException
+from tests.functions_for_pytest import load_json_file
 
+INPUT = "tests/test_data/projects.json"
 APIKEY = 'vnoiweur89u2ongs'
 CLOUDOS_URL = 'http://cloudos.lifebit.ai'
 WORKSPACE_ID = 'lv89ufc838sdig'
-
+PAGE_SIZE = 10
+PAGE = 1
 
 @mock.patch('cloudos_cli.clos', mock.MagicMock())
 @responses.activate
@@ -18,27 +21,29 @@ def test_get_project_list_correct_response():
     """
     Test 'get_project_list' to work as intended
     """
-    params = {"teamId": WORKSPACE_ID}
+    create_json = load_json_file(INPUT)
+    params = {"teamId": WORKSPACE_ID, "pageSize": PAGE_SIZE, "page": PAGE}
     header = {
         "Accept": "application/json, text/plain, */*",
         "Content-Type": "application/json;charset=UTF-8",
         "apikey": APIKEY
     }
-    search_str = f"teamId={WORKSPACE_ID}"
+    search_str = f"teamId={WORKSPACE_ID}&pageSize={PAGE_SIZE}&page={PAGE}"
     # mock GET method with the .json
     responses.add(
             responses.GET,
-            url=f"{CLOUDOS_URL}/api/v1/projects?{search_str}",
+            url=f"{CLOUDOS_URL}/api/v2/projects?{search_str}",
+            body=create_json,
             headers=header,
             match=[matchers.query_param_matcher(params)],
             status=200)
     # start cloudOS service
     clos = Cloudos(apikey=APIKEY, cromwell_token=None, cloudos_url=CLOUDOS_URL)
     # get mock response
-    response = clos.get_project_list(WORKSPACE_ID)
+    response = clos.get_project_list(WORKSPACE_ID, page_size=PAGE_SIZE, page=PAGE)
     # check the response
-    assert response.status_code == 200
-    assert isinstance(response, requests.models.Response)
+    assert isinstance(response, list)
+    assert len(response) == 1
 
 
 @mock.patch('cloudos_cli.clos', mock.MagicMock())
@@ -51,17 +56,17 @@ def test_get_project_list_incorrect_response():
     error_message = {"statusCode": 400, "code": "BadRequest",
                      "message": "Bad Request.", "time": "2022-11-23_17:31:07"}
     error_json = json.dumps(error_message)
-    params = {"teamId": WORKSPACE_ID}
+    params = {"teamId": WORKSPACE_ID, "pageSize": PAGE_SIZE, "page": PAGE}
     header = {
         "Accept": "application/json, text/plain, */*",
         "Content-Type": "application/json;charset=UTF-8",
         "apikey": APIKEY
     }
-    search_str = f"teamId={WORKSPACE_ID}"
+    search_str = f"teamId={WORKSPACE_ID}&pageSize={PAGE_SIZE}&page={PAGE}"
     # mock GET method with the .json
     responses.add(
             responses.GET,
-            url=f"{CLOUDOS_URL}/api/v1/projects?{search_str}",
+            url=f"{CLOUDOS_URL}/api/v2/projects?{search_str}",
             body=error_json,
             headers=header,
             match=[matchers.query_param_matcher(params)],

{cloudos_cli-2.17.0 → cloudos_cli-2.18.0}/tests/test_clos/test_process_project_list.py

@@ -18,10 +18,10 @@ def fixture_mocked_requests_get():
     with open(INPUT_JSON, encoding="utf-8") as json_data:
         data_d = json.load(json_data)
     with requests_mock.Mocker() as mock:
-        mock.get(f"http://test_cloud_os/api/v1/projects?teamId={test_workspace_id}",
+        mock.get(f"http://test_cloud_os/api/v2/projects?teamId={test_workspace_id}",
                  json=data_d)
-        r_get = requests.get(f"http://test_cloud_os/api/v1/projects?teamId={test_workspace_id}")
-    return r_get
+        r_get = requests.get(f"http://test_cloud_os/api/v2/projects?teamId={test_workspace_id}")
+    return json.loads(r_get.content)['projects']
 
 
 def test_process_project_list_all_fields_false(mocked_requests_get):

{cloudos_cli-2.17.0 → cloudos_cli-2.18.0}/tests/test_jobs/test_project_id.py

@@ -26,7 +26,7 @@ def test_project_id():
     """
     create_json_project = load_json_file(INPUT_PROJECT)
     create_json_workflow = load_json_file(INPUT_WORKFLOW)
-    params_projects = {"teamId": WORKSPACE_ID}
+    params_projects = {"teamId": WORKSPACE_ID, "pageSize": PAGE_SIZE, "page": PAGE}
     params_workflows = {
         "teamId": WORKSPACE_ID,
         "pageSize": PAGE_SIZE,
@@ -37,12 +37,12 @@ def test_project_id():
         "Content-Type": "application/json;charset=UTF-8",
         "apikey": APIKEY
     }
-    search_str_projects = f"teamId={WORKSPACE_ID}"
+    search_str_projects = f"teamId={WORKSPACE_ID}&pageSize={PAGE_SIZE}&page={PAGE}"
     search_str_workflows = f"teamId={WORKSPACE_ID}&pageSize={PAGE_SIZE}&page={PAGE}&archived.status={ARCHIVED_STATUS}"
     # mock GET method with the .json
     responses.add(
             responses.GET,
-            url=f"{CLOUDOS_URL}/api/v1/projects?{search_str_projects}",
+            url=f"{CLOUDOS_URL}/api/v2/projects?{search_str_projects}",
             body=create_json_project,
             headers=header,
             match=[matchers.query_param_matcher(params_projects)],

{cloudos_cli-2.17.0 → cloudos_cli-2.18.0}/tests/test_jobs/test_send_job.py

@@ -35,6 +35,7 @@ def test_send_job():
     create_json_workflow = load_json_file(INPUT_WORKFLOW)
     create_json = load_json_file(INPUT)
     params_job = {"teamId": WORKSPACE_ID}
+    params_projects = {"teamId": WORKSPACE_ID, "pageSize": PAGE_SIZE, "page": PAGE}
     params_workflows = {
         "teamId": WORKSPACE_ID,
         "pageSize": PAGE_SIZE,
@@ -45,6 +46,7 @@ def test_send_job():
             "apikey": APIKEY
         }
     search_str = f"teamId={WORKSPACE_ID}"
+    search_str_projects = f"teamId={WORKSPACE_ID}&pageSize={PAGE_SIZE}&page={PAGE}"
     search_str_workflows = f"teamId={WORKSPACE_ID}&pageSize={PAGE_SIZE}&page={PAGE}&archived.status={ARCHIVED_STATUS}"
     # mock GET method with the .json
     responses.add(
@@ -56,10 +58,10 @@ def test_send_job():
             status=200)
     responses.add(
             responses.GET,
-            url=f"{CLOUDOS_URL}/api/v1/projects?{search_str}",
+            url=f"{CLOUDOS_URL}/api/v2/projects?{search_str_projects}",
             body=create_json_project,
             headers=header,
-            match=[matchers.query_param_matcher(params_job)],
+            match=[matchers.query_param_matcher(params_projects)],
             status=200)
     responses.add(
             responses.GET,

{cloudos_cli-2.17.0 → cloudos_cli-2.18.0}/tests/test_jobs/test_workflow_id.py

@@ -26,7 +26,7 @@ def test_workflow_id():
     """
     create_json_project = load_json_file(INPUT_PROJECT)
     create_json_workflow = load_json_file(INPUT_WORKFLOW)
-    params = {"teamId": WORKSPACE_ID}
+    params = {"teamId": WORKSPACE_ID, "pageSize": PAGE_SIZE, "page": PAGE}
     params_workflows = {
         "teamId": WORKSPACE_ID,
         "pageSize": PAGE_SIZE,
@@ -37,12 +37,12 @@ def test_workflow_id():
         "Content-Type": "application/json;charset=UTF-8",
         "apikey": APIKEY
     }
-    search_str = f"teamId={WORKSPACE_ID}"
+    search_str = f"teamId={WORKSPACE_ID}&pageSize={PAGE_SIZE}&page={PAGE}"
     search_str_workflows = f"teamId={WORKSPACE_ID}&pageSize={PAGE_SIZE}&page={PAGE}&archived.status={ARCHIVED_STATUS}"
     # mock GET method with the .json
     responses.add(
             responses.GET,
-            url=f"{CLOUDOS_URL}/api/v1/projects?{search_str}",
+            url=f"{CLOUDOS_URL}/api/v2/projects?{search_str}",
             body=create_json_project,
             headers=header,
             match=[matchers.query_param_matcher(params)],

cloudos_cli-2.17.0/cloudos_cli/_version.py

@@ -1 +0,0 @@
-__version__ = '2.17.0'