accli 0.1.1__py3-none-any.whl

accli/AcceleratorProjectService.py

@@ -0,0 +1,377 @@
+import json
+import base64
+import urllib3
+
+class AcceleratorProjectService:
+    def __init__(
+            self,
+            user_token, 
+            cli_base_url='http://accelerator-api/v1/acli',
+            verify_cert=True
+        ):
+        self.user_token = user_token
+
+        if verify_cert:
+            self.http_client = urllib3.poolmanager.PoolManager(num_pools=1)
+        else:
+            self.http_client = urllib3.poolmanager.PoolManager(cert_reqs="CERT_NONE", num_pools=1)
+
+        self.cli_base_url = cli_base_url
+        self.common_request_headers = {
+            'x-authorization': user_token
+        }
+
+    def get_file_stat(self, bucket_object_id):
+        res = self.http_client.request(
+            "GET", 
+            f"{self.cli_base_url}/file-stat/{bucket_object_id}",
+            headers=self.common_request_headers
+        )
+        return res.json()
+    
+    def get_dataset_type(self, *args, **kwargs):
+        return self.get_bucket_object_validation_type(*args, **kwargs)
+
+    def get_bucket_object_validation_type(self, bucket_object_id):
+        res = self.http_client.request(
+            "GET", 
+            f"{self.cli_base_url}/dataset-type/{bucket_object_id}",
+            headers=self.common_request_headers
+        )
+        if res.data:
+            return res.data.decode()
+
+    
+    def get_file_url(self, bucket_object_id):
+        res = self.http_client.request(
+            "GET", 
+            f"{self.cli_base_url}/file-url/{bucket_object_id}",
+            headers=self.common_request_headers
+        )
+
+        if res.data:
+            return res.data.decode()
+
+
+    def get_file_stream(self, bucket_object_id):
+        url = self.get_file_url(bucket_object_id)
+        if url:
+            url = res.data.decode()
+            resp = http_client.request("GET", url, preload_content=False)
+            return resp
+
+    def get_multipart_put_create_signed_url(
+        self,
+        app_bucket_id,
+        object_name,
+        upload_id,
+        part_number
+    ):
+        res = self.http_client.request(
+            "GET", 
+            f"{self.cli_base_url}/put-create-signed-url",
+            fields=dict(
+                app_bucket_id=app_bucket_id,
+                object_name=object_name,
+                upload_id=upload_id,
+                part_number=part_number
+            ),
+            headers=self.common_request_headers
+        )
+
+        return res.data.decode()
+
+    def get_multipart_put_update_signed_url(
+        self,
+        bucket_object_id,
+        upload_id,
+        part_number,
+    ):
+        res = self.http_client.request(
+            "GET", 
+            f"{self.cli_base_url}/put-update-signed-url",
+            fields=dict(
+                bucket_object_id=bucket_object_id,
+                upload_id=upload_id,
+                part_number=part_number
+            ),
+            headers=self.common_request_headers
+        )
+
+        return res.data.decode()
+
+    def get_put_create_multipart_upload_id(self, filename):
+        res = self.http_client.request(
+            "GET", 
+            f"{self.cli_base_url}/create-multipart-upload-id/{filename}",
+            headers=self.common_request_headers
+        )
+
+        return res.data.decode()
+
+    def get_put_update_multipart_upload_id(self, bucket_object_id):
+        res = self.http_client.request(
+            "GET", 
+            f"{self.cli_base_url}/create-multipart-upload-id/{bucket_object_id}",
+            headers=self.common_request_headers
+        )
+
+        return res.data.decode()
+
+    def complete_job_multipart_upload(
+        self,
+        app_bucket_id,
+        filename,
+        upload_id,
+        parts: list[tuple[str, str]]
+    ):
+        headers = {"Content-Type": "application/json"}
+
+        headers.update(self.common_request_headers)
+
+        res = self.http_client.request(
+            "PUT", 
+            f"{self.cli_base_url}/complete-create-multipart-upload",
+            json=dict(
+                app_bucket_id=app_bucket_id,
+                filename=filename,
+                upload_id=upload_id,
+                parts=base64.b64encode(json.dumps(parts))
+            ),
+            headers=headers
+        )
+
+    def complete_update_multipart_upload(
+        self, bucket_object_id, upload_id, parts: list[tuple[str, str]]
+    ):
+
+        headers = {"Content-Type": "application/json"}
+
+        headers.update(self.common_request_headers)
+
+        res = self.http_client.request(
+            "PUT", 
+            f"{self.cli_base_url}/complete-update-multipart-upload",
+            json=dict(
+                bucket_object_id=bucket_object_id,
+                upload_id=upload_id,
+                parts=base64.b64encode(json.dumps(parts))
+            ),
+            headers=headers
+        )
+
+    def abort_create_multipart_upload(self, app_bucket_id, filename, upload_id):
+        
+        headers = {"Content-Type": "application/json"}
+
+        headers.update(self.common_request_headers)
+
+        res = self.http_client.request(
+            "PUT", 
+            f"{self.cli_base_url}/abort-create-multipart-upload",
+            json=dict(
+                app_bucket_id=app_bucket_id,
+                filename=filename,
+                upload_id=upload_id
+            ),
+            headers=headers
+        )
+
+    def abort_update_multipart_upload(self, bucket_object_id, upload_id):
+        
+        headers = {"Content-Type": "application/json"}
+
+        headers.update(self.common_request_headers)
+
+        res = self.http_client.request(
+            "PUT", 
+            f"{self.cli_base_url}/abort-update-multipart-upload",
+            json=dict(
+                bucket_object_id=bucket_object_id,
+                upload_id=upload_id
+            ),
+            headers=headers
+        )
+    # TODO @wrufesh ensure it requires special token
+    def register_iamc_validation(
+        self, validated_bucket_object_id, indexdb_bucket_object_id
+    ):
+
+        headers = {"Content-Type": "application/json"}
+
+        headers.update(self.common_request_headers)
+
+        res = self.http_client.request(
+            "PUT", 
+            f"{self.cli_base_url}/register-iamc-validation",
+            json=dict(
+                validated_bucket_object_id=validated_bucket_object_id,
+                indexdb_bucket_object_id=indexdb_bucket_object_id
+            ),
+            headers=headers
+        )
+
+      
+
+    def read_part_data(self, stream, size, part_data=b"", progress=None):
+        """Read part data of given size from stream."""
+        size -= len(part_data)
+        while size:
+            data = stream.read(size)
+            if not data:
+                break  # EOF reached
+            if not isinstance(data, bytes):
+                raise ValueError("read() must return 'bytes' object")
+            part_data += data
+            size -= len(data)
+            if progress:
+                progress.update(len(data))
+        return part_data
+
+    def add_filestream_as_job_output(self, filename, file_stream, user_id, job_id):
+        headers = dict()
+        headers["Content-Type"] = "application/octet-stream"
+
+        part_size, part_count = 5 * 1024**2, -1
+
+        upload_id = None
+        app_bucket_id = None
+        uniqified_filename = None
+
+        one_byte = b""
+        stop = False
+        part_number = 0
+        parts = []
+        uploaded_size = 0
+        put_presigned_url = None
+
+        try:
+            while not stop:
+                part_number += 1
+                part_data = self.read_part_data(
+                    file_stream,
+                    part_size + 1,
+                    one_byte,
+                    progress=None,
+                )
+
+                # If part_data_size is less or equal to part_size,
+                # then we have reached last part.
+                if len(part_data) <= part_size:
+                    part_count = part_number
+                    stop = True
+                else:
+                    one_byte = part_data[-1:]
+                    part_data = part_data[:-1]
+
+                uploaded_size += len(part_data)
+
+                if not upload_id:
+                    (
+                        upload_id,
+                        app_bucket_id,
+                        uniqified_filename,
+                    ) = self.get_put_create_multipart_upload_id(
+                        filename, headers=headers
+                    )
+
+                put_presigned_url = self.get_multipart_put_create_signed_url(
+                    app_bucket_id, uniqified_filename, upload_id, part_number
+                )
+
+                part_upload_response = requests.put(
+                    put_presigned_url,
+                    data=part_data,
+                    # headers=headers,
+                    verify=False,
+                )
+
+                etag = part_upload_response.headers.get("etag").replace('"', "")
+                parts.append((part_number, etag))
+
+            result, created_bucket_object_id = self.complete_job_multipart_upload(
+                app_bucket_id, uniqified_filename, upload_id, parts, job_id, user_id
+            )
+            return created_bucket_object_id
+
+        except Exception as err:
+            # Cancel if any error
+            if upload_id:
+                self.abort_create_multipart_upload(
+                    app_bucket_id,
+                    uniqified_filename,
+                    upload_id,
+                )
+            raise err
+
+    def replace_bucket_object_id_content(self, bucket_object_id, file_stream):
+        headers = dict()
+        headers["Content-Type"] = "application/octet-stream"
+
+        part_size, part_count = 5 * 1024**2, -1
+
+        upload_id = None
+        app_bucket_id = None
+        uniqified_filename = None
+
+        one_byte = b""
+        stop = False
+        part_number = 0
+        parts = []
+        uploaded_size = 0
+        put_presigned_url = None
+
+        try:
+            while not stop:
+                part_number += 1
+                part_data = self.read_part_data(
+                    file_stream,
+                    part_size + 1,
+                    one_byte,
+                    progress=None,
+                )
+
+                # If part_data_size is less or equal to part_size,
+                # then we have reached last part.
+                if len(part_data) <= part_size:
+                    part_count = part_number
+                    stop = True
+                else:
+                    one_byte = part_data[-1:]
+                    part_data = part_data[:-1]
+
+                uploaded_size += len(part_data)
+
+                if not upload_id:
+                    upload_id = self.get_put_update_multipart_upload_id(
+                        bucket_object_id, headers=headers
+                    )
+
+                put_presigned_url = self.get_multipart_put_update_signed_url(
+                    bucket_object_id, upload_id, part_number
+                )
+
+                part_upload_response = requests.put(
+                    put_presigned_url,
+                    data=part_data,
+                    # headers=headers,
+                    verify=False,
+                )
+
+                etag = part_upload_response.headers.get("etag").replace('"', "")
+                parts.append((part_number, etag))
+
+            result = self.complete_update_multipart_upload(
+                bucket_object_id,
+                upload_id,
+                parts,
+            )
+        except Exception as err:
+            # Cancel if any error
+            if upload_id:
+                self.abort_update_multipart_upload(
+                    bucket_object_id,
+                    upload_id,
+                )
+
+            raise err

accli/__init__.py

@@ -0,0 +1,3 @@
+from .AcceleratorProjectService import AcceleratorProjectService
+
+ACliService = AcceleratorProjectService

accli/enums.py

@@ -0,0 +1,3 @@
+class ValidationType(enum.Enum):
+    IAMC = 1
+    SPATIALX = 2

accli-0.1.1.dist-info/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2018 The Python Packaging Authority
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

accli-0.1.1.dist-info/METADATA

@@ -0,0 +1,35 @@
+Metadata-Version: 2.1
+Name: accli
+Version: 0.1.1
+Summary: IIASA Accelerator Client
+Author-email: Wrufesh S <wrufesh@gmail.com>
+License: Copyright (c) 2018 The Python Packaging Authority
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+Project-URL: Homepage, https://github.com/iiasa/accli
+Project-URL: Bug Tracker, https://github.com/iiasa/accli/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: urllib3 >=2.0.4
+
+## Accelerator python client

accli-0.1.1.dist-info/RECORD

@@ -0,0 +1,8 @@
+accli/AcceleratorProjectService.py,sha256=sVoQQqmNgtXahNWxiag27MAWwnyj6H7iFg9H1ADp0sk,12031
+accli/__init__.py,sha256=GT9LGFh1C0q2V7aKWSN1x52ESO7p0cybOK12N4B0ySU,105
+accli/enums.py,sha256=hyvf0FyMsqrPXWVjE8_pUdtn6_LRFEO9M3ACIHmX6YE,62
+accli-0.1.1.dist-info/LICENSE,sha256=2bm9uFabQZ3Ykb_SaSU_uUbAj2-htc6WJQmS_65qD00,1073
+accli-0.1.1.dist-info/METADATA,sha256=1Nz22CIat2TYaJShVrxIJiVSK7ps3q-vNPQNgwCJpdA,1764
+accli-0.1.1.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
+accli-0.1.1.dist-info/top_level.txt,sha256=XI5YW6x7TyhYQ0YF-dJwgPrAiRKlmBeQEY70-LyLJoo,6
+accli-0.1.1.dist-info/RECORD,,

accli-0.1.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.41.2)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

accli-0.1.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+accli