accli 0.1.2__tar.gz → 0.1.3.1__tar.gz

{accli-0.1.2 → accli-0.1.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: accli
-Version: 0.1.2
+Version: 0.1.3.1
 Summary: IIASA Accelerator Client
 Author-email: Wrufesh S <wrufesh@gmail.com>
 License: The MIT License (MIT)
@@ -33,6 +33,9 @@ Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: urllib3
+Requires-Dist: typer[all]
+Requires-Dist: tinydb
+Requires-Dist: requests
 
 ## Accelerator python client
 

accli-0.1.2/accli/AcceleratorProjectService.py → accli-0.1.3.1/accli/AcceleratorJobProjectService.py

@@ -11,11 +11,11 @@ class AccAPIError(Exception):
     #     self.response_data = kwargs.pop('response_data')
     #     super().__init__(*args, **kwargs)
 
-class AcceleratorProjectService:
+class AcceleratorJobProjectService:
     def __init__(
             self,
             user_token, 
-            cli_base_url='http://accelerator-api/v1/acli',
+            job_cli_base_url='http://accelerator-api/v1/ajob-cli',
             verify_cert=True
         ):
         
@@ -26,7 +26,7 @@ class AcceleratorProjectService:
         else:
             self.http_client = urllib3.poolmanager.PoolManager(cert_reqs="CERT_NONE", num_pools=1)
 
-        self.cli_base_url = cli_base_url
+        self.cli_base_url = job_cli_base_url
         self.common_request_headers = {
             'x-authorization': user_token
         }
@@ -63,6 +63,15 @@ class AcceleratorProjectService:
         if res.data:
             return res.json()
 
+    def get_bucket_object_validation_details(self, bucket_object_id):
+        res = self.http_client_request(
+            "GET", 
+            f"{self.cli_base_url}/validation-detail/{bucket_object_id}",
+            headers=self.common_request_headers
+        )
+        if res.data:
+            return res.json()
+
     
     def get_file_url(self, bucket_object_id):
         res = self.http_client_request(
@@ -299,7 +308,7 @@ class AcceleratorProjectService:
         headers = dict()
         headers["Content-Type"] = "application/octet-stream"
 
-        part_size, part_count = 5 * 1024**2, -1
+        part_size, part_count = 50 * 1024**2, -1
 
         upload_id = None
         app_bucket_id = None
@@ -376,7 +385,7 @@ class AcceleratorProjectService:
         headers = dict()
         headers["Content-Type"] = "application/octet-stream"
 
-        part_size, part_count = 5 * 1024**2, -1
+        part_size, part_count = 50 * 1024**2, -1
 
         upload_id = None
         app_bucket_id = None

accli-0.1.3.1/accli/AcceleratorTerminalCliProjectService.py

@@ -0,0 +1,272 @@
+import os
+import requests
+import urllib3
+import json
+import base64
+import concurrent.futures
+from rich.progress import Progress
+
+
+class AccAPIError(Exception):
+
+    def __init__(self, message, response, status_code):            
+            
+            super().__init__(message)
+                
+            self.response = response
+            self.status_code = status_code
+
+class AcceleratorTerminalCliProjectService:
+    def __init__(
+            self,
+            user_token, 
+            cli_base_url='http://accelerator-api/v1/aterm-cli',
+            verify_cert=True
+        ):
+        
+        self.user_token = user_token
+
+        if verify_cert:
+            self.http_client = urllib3.poolmanager.PoolManager(num_pools=1)
+        else:
+            self.http_client = urllib3.poolmanager.PoolManager(cert_reqs="CERT_NONE", num_pools=1)
+
+        self.cli_base_url = cli_base_url
+        self.common_request_headers = {
+            'x-authorization': user_token
+        }
+
+    def http_client_request(self, *args, **kwargs):
+        res = self.http_client.request(*args, **kwargs)
+
+        if str(res.status)[0] in ['4', '5']:
+            raise AccAPIError(
+                f"Accelerator api error:: status_code={res.status} :: response_data={res.data}", 
+                status_code=res.status, 
+                response=res
+            )
+        
+        return res
+
+    def get_file_stat(self, project_slug, filename):
+        
+        try:
+            res = self.http_client_request(
+                "POST", 
+                f"{self.cli_base_url}/{project_slug}/file-stat/",
+                json=dict(filename=filename),
+                headers=self.common_request_headers
+            )
+        except AccAPIError as err:
+            if err.status_code == 404:
+                return None
+            else:
+                raise err
+
+        return res.json()
+
+    
+    def get_multipart_put_create_signed_url(
+        self,
+        project_slug,
+        app_bucket_id,
+        object_name,
+        upload_id,
+        part_number
+    ):
+        res = self.http_client_request(
+            "GET", 
+            f"{self.cli_base_url}/{project_slug}/put-multipart-signed-url",
+            fields=dict(
+                app_bucket_id=app_bucket_id,
+                object_name=object_name,
+                upload_id=upload_id,
+                part_number=part_number
+            ),
+            headers=self.common_request_headers
+        )
+
+        return res.json()
+
+    
+
+    def get_put_create_multipart_upload_id(self, project_slug, filename):
+        
+        b64_filename = base64.b64encode(filename.encode()).decode()
+
+        res = self.http_client_request(
+            "GET", 
+            f"{self.cli_base_url}/{project_slug}/create-multipart-upload-id/{b64_filename}",
+            headers=self.common_request_headers
+        )
+
+        data = res.json()
+
+        return data['upload_id'], data['app_bucket_id'], data['uniqified_filename']
+
+
+    def complete_create_multipart_upload(
+        self,
+        project_slug,
+        app_bucket_id,
+        filename,
+        upload_id,
+        parts: list[tuple[str, str]]
+    ):
+        headers = {"Content-Type": "application/json"}
+
+        headers.update(self.common_request_headers)
+
+        res = self.http_client_request(
+            "PUT", 
+            f"{self.cli_base_url}/{project_slug}/complete-create-multipart-upload",
+            json=dict(
+                app_bucket_id=app_bucket_id,
+                filename=filename,
+                upload_id=upload_id,
+                parts=base64.b64encode(json.dumps(parts).encode()).decode()
+            ),
+            headers=headers
+        )
+
+        return res.json()
+
+    
+    def abort_create_multipart_upload(self, project_slug, app_bucket_id, filename, upload_id):
+        
+        headers = {"Content-Type": "application/json"}
+
+        headers.update(self.common_request_headers)
+
+        res = self.http_client_request(
+            "PUT", 
+            f"{self.cli_base_url}/{project_slug}/abort-create-multipart-upload",
+            json=dict(
+                app_bucket_id=app_bucket_id,
+                filename=filename,
+                upload_id=upload_id
+            ),
+            headers=headers
+        )
+
+    
+
+    def read_part_data(self, stream, size, part_data=b"", progress=None):
+        """Read part data of given size from stream."""
+        size -= len(part_data)
+        while size:
+            data = stream.read(size)
+            if not data:
+                break  # EOF reached
+            if not isinstance(data, bytes):
+                raise ValueError("read() must return 'bytes' object")
+            part_data += data
+            size -= len(data)
+            if progress:
+                progress.update(len(data))
+        return part_data
+    
+
+    def put_part(self, project_slug, app_bucket_id, uniqified_filename, upload_id, part_number, part_data, progress, task):
+
+        put_presigned_url = self.get_multipart_put_create_signed_url(
+                        project_slug, app_bucket_id, uniqified_filename, upload_id, part_number
+                    )
+        
+        part_upload_response = requests.put(
+            put_presigned_url,
+            data=part_data,
+            # headers=headers,
+            verify=False,
+        )
+
+        progress.update(task, advance=len(part_data))
+
+        etag = part_upload_response.headers.get("etag").replace('"', "")
+        return part_number, etag
+
+
+    def upload_filestream_to_accelerator(self, project_slug, filename, file_stream, progress, task, max_workers=os.cpu_count()):
+        headers = dict()
+        headers["Content-Type"] = "application/octet-stream"
+
+        part_size, part_count = 200 * 1024**2, -1
+
+        upload_id = None
+        app_bucket_id = None
+        uniqified_filename = None
+
+        one_byte = b""
+        stop = False
+        part_number = 0
+        parts = []
+        uploaded_size = 0
+        put_presigned_url = None
+
+        futures = []
+
+        try:
+
+            with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+
+                while not stop:
+                    part_number += 1
+                    part_data = self.read_part_data(
+                        file_stream,
+                        part_size + 1,
+                        one_byte,
+                        progress=None,
+                    )
+
+                    # If part_data_size is less or equal to part_size,
+                    # then we have reached last part.
+                    if len(part_data) <= part_size:
+                        part_count = part_number
+                        stop = True
+                    else:
+                        one_byte = part_data[-1:]
+                        part_data = part_data[:-1]
+
+                    uploaded_size += len(part_data)
+
+                    if not upload_id:
+                        (
+                            upload_id,
+                            app_bucket_id,
+                            uniqified_filename,
+                        ) = self.get_put_create_multipart_upload_id(
+                            project_slug,
+                            filename, 
+                            # headers=headers
+                        )
+
+                        
+                    future = executor.submit(self.put_part, project_slug, app_bucket_id, uniqified_filename, upload_id, part_number, part_data, progress, task)
+
+                    futures.append(future)
+            
+            futures_results = concurrent.futures.wait(futures, return_when=concurrent.futures.ALL_COMPLETED)
+
+            parts = [f.result() for f in futures_results.done]
+
+            parts.sort(key=lambda x: x[0])
+
+
+            created_bucket_object_id = self.complete_create_multipart_upload(
+                project_slug, app_bucket_id, uniqified_filename, upload_id, parts
+            )
+            return created_bucket_object_id
+
+        except Exception as err:
+            # Cancel if any error
+            if upload_id:
+                self.abort_create_multipart_upload(
+                    project_slug,
+                    app_bucket_id,
+                    uniqified_filename,
+                    upload_id,
+                )
+
+            raise err
+
+    

accli-0.1.3.1/accli/__init__.py

@@ -0,0 +1,6 @@
+from .AcceleratorJobProjectService import AcceleratorJobProjectService
+
+from .cli import app 
+
+AjobCliService = AcceleratorJobProjectService
+

accli-0.1.3.1/accli/__main__.py

@@ -0,0 +1,4 @@
+from .cli import app as cli_app
+
+if __name__ == "__main__":
+    cli_app()

accli-0.1.3.1/accli/cli.py

@@ -0,0 +1,179 @@
+import re
+import glob
+import os
+import typer
+import requests
+from rich import print
+from typing_extensions import Annotated
+from tinydb import TinyDB, Query
+
+from .AcceleratorTerminalCliProjectService import AcceleratorTerminalCliProjectService
+
+from rich.progress import Progress, SpinnerColumn, TextColumn, ProgressColumn, BarColumn, TaskProgressColumn, TimeElapsedColumn, TimeRemainingColumn
+
+
+app = typer.Typer()
+
+def get_db_path():
+
+    home = os.path.expanduser("~")
+    token_directory = f"{home}/.accli"
+
+    if not os.path.exists(token_directory):
+        os.makedirs(token_directory)
+    
+    return f"{token_directory}/data.json"
+
+
+def save_token_details(token, server_url, webcli_url):
+
+    db_path = get_db_path()
+
+    try:
+        os.remove(db_path)
+    except OSError:
+        pass
+
+    db = TinyDB(db_path)
+    db.insert({
+        'token': token,
+        'server_url': server_url,
+        'webcli_url': webcli_url
+    })
+
+def get_token():
+    db_path = get_db_path()
+
+    db = TinyDB(db_path)
+
+    for item in db:
+        token = item.get('token')
+        if token:
+            break
+
+    if not token:
+        print("Token does not exists. Please login.")
+    return token
+
+
+def get_server_url():
+    db_path = get_db_path()
+
+    db = TinyDB(db_path)
+
+    for item in db:
+        server_url = item.get('server_url')
+        if server_url:
+            break
+
+    if not server_url:
+        print("Server url does not exists. Please login.")
+    return server_url
+
+
+def get_size(path):
+    size = 0
+
+    for file in glob.iglob(f"{path}/**/*.*", recursive=True):
+        
+        size += os.path.getsize(file)
+
+    return size
+
+
+
+@app.command()
+def login(
+    server: Annotated[str, typer.Option(help="Accelerator server url.")] = "https://acceleratoapi.iiasa.ac.at", 
+    webcli: Annotated[str, typer.Option(help="Accelerator web client for authorization.")] = "https://accelerator.iiasa.ac.at"
+):
+    print(
+        f"Welcome to Accelerator Terminal Client.\n"
+        f"Powered by IIASA\n"
+    )
+
+    print(f"Get authorization code [link={webcli}/acli-auth-code]here[/link].\n")
+
+    device_authorization_code = typer.prompt("Enter the authorization code?")
+
+    token_response = requests.post(
+        f"{server}/v1/oauth/device/token/", 
+        json={"device_authorization_code": device_authorization_code}
+    )
+
+    print("")
+
+    if token_response.status_code == 400:
+        print(f"ERROR: {token_response.json().get('detail')}")
+        raise typer.Exit(1)
+
+    save_token_details(token_response.json(), server, webcli)
+
+    print("Successfully logged in.:rocket: :rocket:")
+
+def upload_file(project_slug, accelerator_filename, local_filepath, progress, task, folder_name, max_workers=os.cpu_count()):
+
+    access_token = get_token()
+
+    server_url = get_server_url()
+
+    term_cli_project_service = AcceleratorTerminalCliProjectService(
+        user_token=access_token,
+        cli_base_url=f"{server_url}/v1/aterm-cli",
+        verify_cert=False
+    )
+
+    stat = term_cli_project_service.get_file_stat(project_slug, f"{folder_name}/{accelerator_filename}")
+
+    if stat:
+        progress.update(task, advance=stat.get('size'))
+    else:
+
+        with open(local_filepath, 'rb') as file_stream:
+            term_cli_project_service.upload_filestream_to_accelerator(project_slug, f"{folder_name}/{accelerator_filename}", file_stream, progress, task, max_workers=max_workers)
+
+@app.command()
+def upload(
+    project_slug: Annotated[str, typer.Option(help="Project slug")],
+    path: Annotated[str, typer.Option(help="File or folder path.")],
+    folder_name: Annotated[str, typer.Option(help="New folder name.")],
+    max_workers: Annotated[int, typer.Option(help="Maximum worker pool for multipart upload")] = os.cpu_count()
+):
+    
+    if not re.fullmatch(r'\w+', folder_name):
+        raise ValueError("Folder name is invalid.")
+
+
+    with Progress(
+        TextColumn("[progress.description]"),
+        BarColumn(),
+        TaskProgressColumn(),
+        TimeElapsedColumn(),
+        TextColumn("{task.description}"),
+        transient=True
+    ) as progress:
+    
+        if (os.path.isdir(path)):
+
+            if not path.endswith("/"):
+                path = path + ("/")
+
+            folder_size = get_size(path)
+            print('Folder size', folder_size)
+            upload_task = progress.add_task("[cyan]Uploading.", total=folder_size)
+
+            for local_file_path in glob.iglob(f"{path}/**/*.*", recursive=True):
+                accelerator_filename = os.path.relpath(local_file_path, path)
+
+                progress.update(
+                    upload_task, 
+                    description=f"[cyan]Uploading {local_file_path} \t"
+                )
+
+                upload_file(project_slug, accelerator_filename, local_file_path,  progress, upload_task, folder_name, max_workers=max_workers)
+        elif (os.path.isfile(path)):
+            raise NotImplementedError('Only folder can be uploaded. File upload is not implemented.')
+        else:
+            print("ERROR: No such file or directory.")
+            typer.Exit(1)
+

{accli-0.1.2 → accli-0.1.3.1}/accli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: accli
-Version: 0.1.2
+Version: 0.1.3.1
 Summary: IIASA Accelerator Client
 Author-email: Wrufesh S <wrufesh@gmail.com>
 License: The MIT License (MIT)
@@ -33,6 +33,9 @@ Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: urllib3
+Requires-Dist: typer[all]
+Requires-Dist: tinydb
+Requires-Dist: requests
 
 ## Accelerator python client
 

accli-0.1.3.1/accli.egg-info/SOURCES.txt

@@ -0,0 +1,14 @@
+LICENSE
+README.md
+pyproject.toml
+accli/AcceleratorJobProjectService.py
+accli/AcceleratorTerminalCliProjectService.py
+accli/__init__.py
+accli/__main__.py
+accli/cli.py
+accli.egg-info/PKG-INFO
+accli.egg-info/SOURCES.txt
+accli.egg-info/dependency_links.txt
+accli.egg-info/entry_points.txt
+accli.egg-info/requires.txt
+accli.egg-info/top_level.txt

accli-0.1.3.1/accli.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+accli = accli.cli.app

accli-0.1.3.1/accli.egg-info/requires.txt

@@ -0,0 +1,4 @@
+urllib3
+typer[all]
+tinydb
+requests

{accli-0.1.2 → accli-0.1.3.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "accli"
-version = "0.1.2"
+version = "0.1.3.1"
 authors = [
   { name="Wrufesh S", email="wrufesh@gmail.com" },
 ]
@@ -12,7 +12,7 @@ description = "IIASA Accelerator Client"
 readme = "README.md"
 license = {file = "LICENSE"}
 requires-python = ">=3.7"
-dependencies = ['urllib3']
+dependencies = ['urllib3', 'typer[all]', 'tinydb', 'requests']
 classifiers = [
     "Programming Language :: Python :: 3",
     "License :: OSI Approved :: MIT License",
@@ -24,4 +24,7 @@ packages = ['accli']
 
 [project.urls]
 "Homepage" = "https://github.com/iiasa/accli"
-"Bug Tracker" = "https://github.com/iiasa/accli/issues"
+"Bug Tracker" = "https://github.com/iiasa/accli/issues"
+
+[project.scripts]
+accli = "accli.cli.app"

accli-0.1.2/accli/__init__.py

@@ -1,3 +0,0 @@
-from .AcceleratorProjectService import AcceleratorProjectService
-
-ACliService = AcceleratorProjectService

accli-0.1.2/accli.egg-info/SOURCES.txt

@@ -1,10 +0,0 @@
-LICENSE
-README.md
-pyproject.toml
-accli/AcceleratorProjectService.py
-accli/__init__.py
-accli.egg-info/PKG-INFO
-accli.egg-info/SOURCES.txt
-accli.egg-info/dependency_links.txt
-accli.egg-info/requires.txt
-accli.egg-info/top_level.txt

accli-0.1.2/accli.egg-info/requires.txt

@@ -1 +0,0 @@
-urllib3