oscar-python 1.2.0__tar.gz → 1.3.0__tar.gz

{oscar_python-1.2.0 → oscar_python-1.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: oscar_python
-Version: 1.2.0
+Version: 1.3.0
 Summary: OSCAR API for python
 Home-page: https://github.com/grycap/oscar_python
 Author: GRyCAP - Universitat Politecnica de Valencia
@@ -10,6 +10,22 @@ Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: Apache Software License
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: webdavclient3==3.14.6
+Requires-Dist: requests
+Requires-Dist: boto3
+Requires-Dist: setuptools>=40.8.0
+Requires-Dist: pyyaml
+Requires-Dist: aiohttp
+Requires-Dist: liboidcagent
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license
+Dynamic: requires-dist
+Dynamic: summary
 
 ## Python OSCAR client
 
@@ -170,6 +186,9 @@ response = client.remove_service("service_name") # returns an http response
 ``` python
 # make a synchronous execution 
 response = client.run_service("service_name", input="input", output="out.png", timeout=100) # returns an http response
+
+# make an asynchronous execution
+response = client.run_service("service_name", input="input", async_call=True) # returns an http response
 ```
 
 #### Logs methods

{oscar_python-1.2.0 → oscar_python-1.3.0}/README.md

@@ -157,6 +157,9 @@ response = client.remove_service("service_name") # returns an http response
 ``` python
 # make a synchronous execution 
 response = client.run_service("service_name", input="input", output="out.png", timeout=100) # returns an http response
+
+# make an asynchronous execution
+response = client.run_service("service_name", input="input", async_call=True) # returns an http response
 ```
 
 #### Logs methods

{oscar_python-1.2.0 → oscar_python-1.3.0}/oscar_python/_providers/_minio.py

@@ -10,15 +10,16 @@
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
-# limitations under the License. 
+# limitations under the License.
 
 import boto3
 from oscar_python._providers._s3 import S3
 
 _DEFAULT_MINIO_ENDPOINT = 'http://minio-service.minio:9000'
 
+
 class Minio(S3):
-   
+
     def _get_client(self, c):
         """Return Minio client with user configuration."""
         if c["endpoint"] == '':
@@ -30,4 +31,4 @@ class Minio(S3):
                             region_name=c["region"],
                             verify=c["verify"],
                             aws_access_key_id=c["access_key"],
-                            aws_secret_access_key=c["secret_key"])
+                            aws_secret_access_key=c["secret_key"])

{oscar_python-1.2.0 → oscar_python-1.3.0}/oscar_python/_providers/_onedata.py

@@ -10,29 +10,30 @@
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
-# limitations under the License. 
+# limitations under the License.
 
 from oscar_python._providers._providers_base import StorageProvider
 import requests
 import json
 
-#TODO fix error returns
+
+# TODO fix error returns
 class Onedata(StorageProvider):
-    
+
     _CDMI_PATH = '/cdmi'
     _CDMI_VERSION_HEADER = {'X-CDMI-Specification-Version': '1.1.1'}
 
     def __init__(self, credentials) -> None:
         super().__init__()
         self._get_client(credentials)
-    
+
     def _get_client(self, c):
         self.oneprovider_space = c["space"]
         self.oneprovider_host = c["oneprovider_host"]
         self.headers = {'X-Auth-Token': c["token"]}
         self.url = (f'https://{self.oneprovider_host}{self._CDMI_PATH}/'
                     f'{self.oneprovider_space}/')
-    
+
     def upload_file(self, local_path, remote_path):
         file_name = local_path.split('/')[-1]
         if not self._folder_exists(remote_path):
@@ -57,12 +58,13 @@ class Onedata(StorageProvider):
         file_name = remote_path.split('/')[-1]
         url = self.url+remote_path
         res = requests.get(url=url, headers=self.headers)
-        if res.status_code == 200: print("Saving file to {0}/{1}".format(local_path, file_name))
+        if res.status_code == 200:
+            print("Saving file to {0}/{1}".format(local_path, file_name))
         self._save_file(local_path+"/"+file_name, res.content, mode='wb')
-    
+
     def list_files_from_path(self, path):
         headers = {**self._CDMI_VERSION_HEADER, **self.headers}
-        return requests.get(url = self.url+path+"/", headers=headers)
+        return requests.get(url=self.url+path+"/", headers=headers)
 
     def _save_file(self, path, content, mode='w'):
         with open(path, mode) as fwc:
@@ -72,12 +74,13 @@ class Onedata(StorageProvider):
 
     def _folder_exists(self, folder_name):
         headers = {**self._CDMI_VERSION_HEADER, **self.headers}
-        response = requests.get(url = self.url+folder_name+"/", headers=headers)
+        response = requests.get(url=self.url+folder_name+"/", headers=headers)
         if response.status_code == 200:
             return True
         return False
-    
+
     def _create_folder(self, folder_name):
-        response = requests.put(url = self.url+folder_name+"/", headers=self.headers)
-        if response.status_code != 201: 
-            return False
+        response = requests.put(url=self.url+folder_name+"/", headers=self.headers)
+        if response.status_code == 201:
+            return True
+        return False

{oscar_python-1.2.0 → oscar_python-1.3.0}/oscar_python/_providers/_providers_base.py

@@ -10,10 +10,11 @@
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
-# limitations under the License. 
+# limitations under the License.
 
 import abc
 
+
 class StorageProvider(abc.ABC):
     @abc.abstractmethod
     def download_file(self, local_path, remote_path):
@@ -22,11 +23,11 @@ class StorageProvider(abc.ABC):
     @abc.abstractmethod
     def upload_file(self, local_path, remote_path):
         """Generic method to be implemented by all the storage providers."""
-    
+
     @abc.abstractmethod
     def list_files_from_path(self, path):
         """Generic method to be implemented by all the storage providers."""
 
     @abc.abstractmethod
     def _get_client(self):
-        """Generic method to be implemented by all the storage providers."""
+        """Generic method to be implemented by all the storage providers."""

{oscar_python-1.2.0 → oscar_python-1.3.0}/oscar_python/_providers/_s3.py

@@ -10,19 +10,19 @@
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
-# limitations under the License. 
+# limitations under the License.
 
 from aiohttp import ClientError
-import logging
 import boto3
 from oscar_python._providers._providers_base import StorageProvider
 
+
 class S3(StorageProvider):
     def __init__(self, credentials) -> None:
         super().__init__()
         self.client = self._get_client(credentials)
         self.resource = self._get_resource(credentials)
-    
+
     def _get_client(self, c):
         """Returns S3 client with default configuration."""
         if c is None:
@@ -42,39 +42,41 @@ class S3(StorageProvider):
         if region == '':
             region = None
         return boto3.resource('s3',
-                            region_name=region,
-                            aws_access_key_id=c["access_key"],
-                            aws_secret_access_key=c["secret_key"])
+                              region_name=region,
+                              aws_access_key_id=c["access_key"],
+                              aws_secret_access_key=c["secret_key"])
 
     def list_files_from_path(self, path):
         prefix = ""
         path_split = path.split('/')
         bucket = path_split[0]
         if len(path_split) > 1:
-            prefix = path.split('/',2)[1]
-        print("Reading content from path:",path)
+            prefix = path.split('/', 2)[1]
+        print("Reading content from path:", path)
         return self.client.list_objects(Bucket=bucket, Prefix=prefix)
-    
+
     def upload_file(self, local_path, remote_path):
         bucket_name = remote_path.split('/')[0]
-        file_key = remote_path.split('/',1)[1]
+        file_key = remote_path.split('/', 1)[1]
         file_name = local_path.split('/')[-1]
         print("Uploading to bucket '{0}' with key '{1}'".format(bucket_name,file_key))
         with open(local_path, 'rb') as data:
             try:
-                self.client.upload_fileobj(data, bucket_name, file_key+"/"+file_name)
+                self.client.upload_fileobj(data, bucket_name, file_key + "/" + file_name)
+                return True
             except ClientError as err:
                 print("Error uploading file: ", err)
                 return False
 
     def download_file(self, local_path, remote_path):
         bucket_name = remote_path.split('/')[0]
-        file_key = remote_path.split('/',1)[1]
+        file_key = remote_path.split('/', 1)[1]
         file_path = local_path+"/"+remote_path.split('/')[-1]
         print("Downloading from bucket '{0}' to path '{1}' with key '{2}'".format(bucket_name, file_path, file_key))
         with open(file_path, 'wb') as data:
             try:
                 self.client.download_fileobj(bucket_name, file_key, data)
+                return True
             except ClientError as err:
                 print("Error downloading file: ", err)
-                return False
+                return False

{oscar_python-1.2.0 → oscar_python-1.3.0}/oscar_python/_providers/_webdav.py

@@ -10,12 +10,13 @@
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
-# limitations under the License. 
+# limitations under the License.
 
 from oscar_python._providers._providers_base import StorageProvider
 from webdav3.client import Client
 from webdav3.exceptions import WebDavException
 
+
 class WebDav(StorageProvider):
     def __init__(self, credentials) -> None:
         self.client = self._get_client(credentials)
@@ -26,9 +27,9 @@ class WebDav(StorageProvider):
         if 'https://' not in hostname:
             hostname = 'https://'+hostname
         options = {
-        'webdav_hostname': hostname,
-        'webdav_login':    c["login"],
-        'webdav_password': c["password"]
+            'webdav_hostname': hostname,
+            'webdav_login':    c["login"],
+            'webdav_password': c["password"]
         }
         return Client(options=options)
 
@@ -45,13 +46,12 @@ class WebDav(StorageProvider):
         except WebDavException as exception:
             print("error uploading file to path '{0}': {1}".format(remote_path+"/"+file_name, exception))
 
-    
     def download_file(self, local_path, remote_path):
         file_name = remote_path.split('/')[-1]
         try:
             self.client.download_sync(remote_path, local_path+"/"+file_name)
         except WebDavException as exception:
             print("error downloading file from path '{0}': {1}".format(remote_path, exception))
-    
+
     def list_files_from_path(self, path):
-        return self.client.list(path)
+        return self.client.list(path)

{oscar_python-1.2.0 → oscar_python-1.3.0}/oscar_python/_utils.py

@@ -10,7 +10,7 @@
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
-# limitations under the License. 
+# limitations under the License.
 
 import base64
 import os
@@ -18,18 +18,19 @@ import requests
 import liboidcagent as agent
 _DEFAULT_TIMEOUT = 60
 
-""" Generic http request """
-def make_request(c , path, method, **kwargs):
 
-    headers = get_headers(c)  
+def make_request(c, path, method, **kwargs):
+    """ Generic http request """
 
-    if "timeout" in kwargs.keys() and kwargs["timeout"]: 
+    headers = get_headers(c)
+
+    if "timeout" in kwargs.keys() and kwargs["timeout"]:
         timeout = kwargs["timeout"]
-    else: 
+    else:
         timeout = _DEFAULT_TIMEOUT
 
     url = c.endpoint+path
-    
+
     if method in ["post", "put"]:
         if "token" in kwargs.keys() and kwargs["token"]: 
             headers = get_headers_with_token(kwargs["token"])
@@ -40,37 +41,42 @@ def make_request(c , path, method, **kwargs):
 
     if "handle" in kwargs.keys() and kwargs["handle"] == False:
         return result
-    
+
     result.raise_for_status()
     return result
 
-""" Function to generate headers with basic authentication or OIDC """
+
 def get_headers(c):
+    """ Function to generate headers with basic authentication or OIDC """
     if c._AUTH_TYPE == "basicauth":
-        usr_pass_as_bytes = bytes(c.user+":"+c.password,"utf-8")
+        usr_pass_as_bytes = bytes(c.user + ":" + c.password, "utf-8")
         usr_pass_base_64 = base64.b64encode(usr_pass_as_bytes).decode("utf-8")
-        return {"Authorization": "Basic "+ usr_pass_base_64}
+        return {"Authorization": "Basic " + usr_pass_base_64}
     if c._AUTH_TYPE == "oidc-agent":
         token = agent.get_access_token(c.shortname)
         return get_headers_with_token(token)
     if c._AUTH_TYPE == "oidc":
         return get_headers_with_token(c.oidc_token)
 
-""" Function to generate headers with token auth """
+
 def get_headers_with_token(token):
-    return {"Authorization": "Bearer "+ str(token)}
+    """ Function to generate headers with token auth """
+    return {"Authorization": "Bearer " + str(token)}
+
 
 def write_text_file(content, file_path):
     with open(file_path, 'w') as f:
         f.write(content)
 
+
 def isBase64(st):
     try:
         base64.b64decode(st)
         return True
-    except:
+    except Exception:
         return False
 
+
 def decode_b64(b64_str, file_out):
     file_extension = os.path.splitext(file_out)[1]
     try:
@@ -80,7 +86,7 @@ def decode_b64(b64_str, file_out):
             decode = 'w'
             decoded_data = decoded_data.decode("utf-8")
         else:
-            decode = 'wb' 
+            decode = 'wb'
 
         with open(file_out, decode) as f:
             f.write(decoded_data)
@@ -90,6 +96,7 @@ def decode_b64(b64_str, file_out):
     except OSError:
         print('Error decoding output: Failed to write decoded data to file.')   
 
+
 def encode_input(data):
     if os.path.isfile(data):
         try:
@@ -103,12 +110,11 @@ def encode_input(data):
         message_bytes = data.encode('ascii')
         return base64.b64encode(message_bytes)
 
+
 def decode_output(output, file_path):
-    if(isBase64(output)):
+    if isBase64(output):
         decode_b64(output, file_path)
         return
-    if(isinstance(output,str)):
-        write_text_file(output,file_path)
+    if isinstance(output, str):
+        write_text_file(output, file_path)
         return
-
- 

{oscar_python-1.2.0 → oscar_python-1.3.0}/oscar_python/client.py

@@ -11,14 +11,14 @@
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
-# limitations under the License. 
+# limitations under the License.
 
 import json
 import os
 import yaml
 import liboidcagent as agent
 import oscar_python._utils as utils
-from default_client import DefaultClient
+from oscar_python.default_client import DefaultClient
 from oscar_python.storage import Storage
 
 _INFO_PATH = "/system/info"
@@ -26,7 +26,10 @@ _CONFIG_PATH = "/system/config"
 _SVC_PATH = "/system/services"
 _LOGS_PATH = "/system/logs"
 _RUN_PATH = "/run"
-#_JOB_PATH = "/job"
+_STATUS_PATH="/system/status"
+
+
+# _JOB_PATH = "/job"
 
 
 _GET = "get"
@@ -34,8 +37,9 @@ _POST = "post"
 _PUT = "put"
 _DELETE = "delete"
 
+
 class Client(DefaultClient):
-    #Cluster info 
+    # Cluster info
     def __init__(self, options) -> None:
         self.set_auth_type(options)
         if self._AUTH_TYPE == 'basicauth':
@@ -57,7 +61,7 @@ class Client(DefaultClient):
         self.endpoint = options['endpoint']
         self.shortname = options['shortname']
         self.ssl = bool(options['ssl'])
-    
+
     def oidc_client(self, options):
         self.id = options['cluster_id']
         self.endpoint = options['endpoint']
@@ -78,7 +82,7 @@ class Client(DefaultClient):
         else:
             raise ValueError("Unrecognized authentication credentials in options")
 
-    """ Creates a generic storage client to interact with the storage providers 
+    """ Creates a generic storage client to interact with the storage providers
     defined on a specific service of the refered OSCAR cluster """
     def create_storage_client(self, svc):
         return Storage(
@@ -96,7 +100,7 @@ class Client(DefaultClient):
     """ List all services from the current cluster """
     def list_services(self):
         return utils.make_request(self, _SVC_PATH, _GET)
-    
+
     """ Retreive a specific service """
     def get_service(self, name):
         return utils.make_request(self, _SVC_PATH+"/"+name, _GET)
@@ -111,15 +115,15 @@ class Client(DefaultClient):
                     try:
                         svc = element[self.id]
                     except KeyError as err:
-                        raise("FDL clusterID does not match current clusterID: {0}".format(err))
+                        raise Exception("FDL clusterID does not match current clusterID: {0}".format(err))
                     try:
                         with open(svc["script"]) as s:
                             svc["script"] = s.read()
-                    except IOError as err:
-                        raise("Couldn't read script")
-                    
+                    except IOError:
+                        raise Exception("Couldn't read script")
+
                     # cpu parameter has to be string on the request
-                    if type(svc["cpu"]) is int or type(svc["cpu"]) is float: svc["cpu"]= str(svc["cpu"])
+                    if type(svc["cpu"]) is int or type(svc["cpu"]) is float: svc["cpu"] = str(svc["cpu"])
 
             except ValueError as err:
                 print(err)
@@ -127,7 +131,11 @@ class Client(DefaultClient):
         else:
             raise ValueError("Bad yaml format: {0}".format(fdl))
         return svc
-        
+
+    """ Get status of a cluster (CPU and Memory) """
+    def get_cluster_status(self):
+        return utils.make_request(self, _STATUS_PATH, _GET)
+
     """ Make the request to create a new service """
     def _apply_service(self, svc, method):
         # Check if service already exists when the function is called from create_service
@@ -136,18 +144,16 @@ class Client(DefaultClient):
             if svc_exists.status_code == 200:
                 raise ValueError("A service with name '{0}' is already present on the cluster".format(svc["name"]))
         return utils.make_request(self, _SVC_PATH, method, data=json.dumps(svc))
-    
+
     """ Create a service on the current cluster from a FDL file or a JSON definition """
     def create_service(self, service_definition):
         if type(service_definition) is dict:
             return self._apply_service(service_definition, _POST)
         if os.path.isfile(service_definition):
-            try:
-               service = self._check_fdl_definition(service_definition)
-            except Exception:
-                raise
+            service = self._check_fdl_definition(service_definition)
             return self._apply_service(service, _POST)
-        
+        raise ValueError("Service definition must be a dictionary or a file path")
+
     """ Update a specific service from a FDL file or a JSON definition """
     def update_service(self, name, new_service):
         # Check if service exists before update
@@ -158,7 +164,7 @@ class Client(DefaultClient):
             return self._apply_service(new_service, _PUT)
         if os.path.isfile(new_service):
             try:
-               service = self._check_fdl_definition(new_service)
+                service = self._check_fdl_definition(new_service)
             except Exception:
                 raise
             return self._apply_service(service, _PUT)
@@ -178,11 +184,11 @@ class Client(DefaultClient):
         except ValueError as err:
             return err
         return fdl_yaml
-    
+
     """ Get logs of a service job """
     def get_job_logs(self, svc, job):
         return utils.make_request(self, _LOGS_PATH+"/"+svc+"/"+job, _GET)
-    
+
     """ List a service jobs """
     def list_jobs(self, svc):
         return utils.make_request(self, _LOGS_PATH+"/"+svc, _GET)

{oscar_python-1.2.0 → oscar_python-1.3.0}/oscar_python/client_anon.py

@@ -1,9 +1,10 @@
-from default_client import DefaultClient
+from oscar_python.default_client import DefaultClient
+
 
 class AnonymousClient(DefaultClient):
-    #Cluster info 
+    # Cluster info
     def __init__(self, options) -> None:
         self.id = options['cluster_id']
         self.endpoint = options["endpoint"]
         self.ssl = bool(options['ssl'])
-        self._AUTH_TYPE = "anon"
+        self._AUTH_TYPE = "anon"

oscar_python-1.3.0/oscar_python/default_client.py

@@ -0,0 +1,39 @@
+import abc
+import oscar_python._utils as utils
+
+_RUN_PATH = "/run"
+_JOB_PATH = "/job"
+_POST = "post"
+
+
+class DefaultClient(metaclass=abc.ABCMeta):
+
+    _AUTH_TYPE = ''
+
+    """ Run an execution.
+    If async is set to True the execution is run asynchronously.
+    If an output is provided the result is decoded onto the file.
+    In both cases the function returns the HTTP response."""
+    def run_service(self, name, **kwargs):
+        if kwargs.get("token"):
+            token = kwargs["token"]
+        else:
+            token = self._get_token(name)
+
+        send_data = None
+        if kwargs.get("input"):
+            send_data = kwargs["input"]
+            if not isinstance(send_data, dict):
+                send_data = utils.encode_input(send_data)
+
+        path = _RUN_PATH + "/"+name
+        if kwargs.get("async_call"):
+            path = _JOB_PATH + "/" + name
+
+        response = utils.make_request(self, path, _POST, data=send_data,
+                                      token=token, timeout=kwargs.get("timeout"))
+
+        if kwargs.get("output"):
+            utils.decode_output(response.text, kwargs["output"])
+
+        return response

{oscar_python-1.2.0 → oscar_python-1.3.0}/oscar_python/local_test.py

@@ -7,4 +7,4 @@ if res:
     print(res.status_code)
     print(res.content)
 else:
-    print("empty")
+    print("empty")