accli 0.4__tar.gz → 0.5__tar.gz

{accli-0.4/accli.egg-info → accli-0.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: accli
-Version: 0.4
+Version: 0.5
 Summary: IIASA Accelerator Client
 Author-email: Wrufesh S <wrufesh@gmail.com>
 License: The MIT License (MIT)

{accli-0.4 → accli-0.5}/accli/AcceleratorJobProjectService.py

@@ -16,6 +16,16 @@ class AccAPIError(Exception):
     #     self.response_data = kwargs.pop('response_data')
     #     super().__init__(*args, **kwargs)
 
+retries = urllib3.util.Retry(total=10, backoff_factor=1)
+
+http_client = urllib3.poolmanager.PoolManager(
+    num_pools=20, retries=retries
+)
+
+http_client_wo_cert_verification = urllib3.poolmanager.PoolManager(
+    cert_reqs="CERT_NONE", num_pools=20, retries=retries
+)
+
 class AcceleratorJobProjectService:
     def __init__(
             self,
@@ -26,12 +36,10 @@ class AcceleratorJobProjectService:
         
         self.user_token = user_token
 
-        retries = urllib3.util.Retry(total=10, backoff_factor=1)
-
         if verify_cert:
-            self.http_client = urllib3.poolmanager.PoolManager(num_pools=1, retries=retries)
+            self.http_client = http_client
         else:
-            self.http_client = urllib3.poolmanager.PoolManager(cert_reqs="CERT_NONE", num_pools=1, retries=retries)
+            self.http_client = http_client_wo_cert_verification
 
         self.cli_base_url = f"{server_url}/v1/ajob-cli"
         self.common_request_headers = {
@@ -58,6 +66,16 @@ class AcceleratorJobProjectService:
         )
         return res.json()
     
+    def get_file_url_from_repo(self, filename):
+        project_slug = filename.split('/')[0]
+        res = self.http_client_request(
+            "GET", 
+            f"{self.cli_base_url}/{project_slug}/get-file-download-url/?filename={filename}",
+            headers=self.common_request_headers
+        )
+        if res.data:
+            return res.json()
+    
     def get_dataset_type(self, *args, **kwargs):
         return self.get_bucket_object_validation_type(*args, **kwargs)
 
@@ -100,13 +118,14 @@ class AcceleratorJobProjectService:
     def add_log_file(self, data: bytes, filename):
         res = self.http_client_request(
             "GET",
-            f"{self.cli_base_url}/presigned-upload-url/?filename={filename}",
+            f"{self.cli_base_url}/presigned-log-upload-url/?filename={filename}",
             headers=self.common_request_headers
         ).json()
 
         upload_url = res['upload_url']
         app_bucket_id = res['app_bucket_id']
         res_filename = res['filename']
+        is_healthy = res['is_healthy']
 
         requests.put(
             upload_url,
@@ -125,6 +144,9 @@ class AcceleratorJobProjectService:
             headers=self.common_request_headers
         )
 
+        if not is_healthy:
+            os.Exit(1)
+
         
 
     def get_multipart_put_create_signed_url(
@@ -532,10 +554,29 @@ class Fs:
             file.write(filestream.getvalue())
 
     @staticmethod
-    def write_file(source: typing.Union[str, io.BytesIO], dest_filepath):
+    def get_file_url(remote_filepath):
         user_token = os.environ.get("ACC_JOB_JOB_TOKEN", None)
         server_url = os.environ.get("ACC_JOB_GATEWAY_SERVER", None)
 
+        if not (user_token and server_url):
+            raise ValueError("Remote data repository credentials not found.")
+        
+        accelerator_job_service = AcceleratorJobProjectService(
+            user_token, 
+            server_url=server_url,
+            verify_cert=True
+        )
+
+        return accelerator_job_service.get_file_url_from_repo(
+            remote_filepath
+        )
+        
+
+    @staticmethod
+    def write_file(source: typing.Union[str, io.BytesIO], dest_filepath):
+        user_token = os.environ.get("ACC_JOB_TOKEN", None)
+        server_url = os.environ.get("ACC_JOB_GATEWAY_SERVER", None)
+
         if isinstance(source, io.BytesIO):
             if (user_token and server_url):
                 Fs.write_stream_remote(source, dest_filepath, user_token, server_url)

{accli-0.4 → accli-0.5}/accli/AcceleratorTaskDispatcher.py

@@ -4,7 +4,7 @@ import hashlib
 import shutil
 import tempfile
 from typing import Optional, List, Dict
-from pydantic import BaseModel, root_validator
+from pydantic.v1 import BaseModel, root_validator
 
 from accli.token import get_github_app_token
 
@@ -139,6 +139,8 @@ class WKubeTaskKwargs(BaseModel):
 
     conf: Dict[str,str] = {}
 
+    build_timeout: Optional[int]
+
     def dict(self, *args, **kwargs):
         result = super().dict(*args, **kwargs)
         if 'job_folder' in result:

{accli-0.4 → accli-0.5}/accli/AcceleratorTerminalCliProjectService.py

@@ -18,6 +18,16 @@ class AccAPIError(Exception):
             self.response = response
             self.status_code = status_code
 
+retries = urllib3.util.Retry(total=10, backoff_factor=1)
+
+http_client = urllib3.poolmanager.PoolManager(
+    num_pools=20, retries=retries
+)
+
+http_client_wo_cert_verification = urllib3.poolmanager.PoolManager(
+    cert_reqs="CERT_NONE", num_pools=20, retries=retries
+)
+
 class AcceleratorTerminalCliProjectService:
     def __init__(
             self,
@@ -28,12 +38,10 @@ class AcceleratorTerminalCliProjectService:
         
         self.user_token = user_token
 
-        retries = urllib3.util.Retry(total=10, backoff_factor=1)
-
         if verify_cert:
-            self.http_client = urllib3.poolmanager.PoolManager(num_pools=1, retries=retries)
+            self.http_client = http_client
         else:
-            self.http_client = urllib3.poolmanager.PoolManager(cert_reqs="CERT_NONE", num_pools=1, retries=retries)
+            self.http_client = http_client_wo_cert_verification
 
         self.server_url = server_url
         self.cli_base_url = f"{self.server_url}/v1/aterm-cli"
@@ -70,6 +78,16 @@ class AcceleratorTerminalCliProjectService:
 
         return res.json()
     
+    def get_file_url_from_repo(self, filename):
+        project_slug = filename.split('/')[0]
+        res = self.http_client_request(
+            "GET", 
+            f"{self.cli_base_url}/{project_slug}/get-file-download-url/?filename={filename}",
+            headers=self.common_request_headers
+        )
+        if res.data:
+            return res.json()
+    
     def get_github_app_token(self, project_slug):
         
         res = self.http_client_request(

{accli-0.4 → accli-0.5}/accli/_version.py

@@ -1,3 +1,3 @@
 # Please strictly put double quote to use this info for git tag 
 # Read developer guide for git tag command with regex
-VERSION = "v0.4" 
+VERSION = "v0.5" 

{accli-0.4 → accli-0.5/accli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: accli
-Version: 0.4
+Version: 0.5
 Summary: IIASA Accelerator Client
 Author-email: Wrufesh S <wrufesh@gmail.com>
 License: The MIT License (MIT)

{accli-0.4 → accli-0.5}/accli.egg-info/SOURCES.txt

@@ -1,9 +1,6 @@
-.gitignore
-Dockerfile.dev
 LICENSE
 README.md
 pyproject.toml
-requirements.txt
 accli/AcceleratorJobProjectService.py
 accli/AcceleratorTaskDispatcher.py
 accli/AcceleratorTerminalCliProjectService.py

{accli-0.4 → accli-0.5}/accli.egg-info/requires.txt

@@ -3,4 +3,5 @@ typer[all]
 tinydb
 requests
 jsonschema
-pydantic==1.10.13
+pydantic
+gitpython

{accli-0.4 → accli-0.5}/pyproject.toml

@@ -12,7 +12,7 @@ description = "IIASA Accelerator Client"
 readme = "README.md"
 license = {file = "LICENSE"}
 requires-python = ">=3.7.17"
-dependencies = ['urllib3', 'typer[all]', 'tinydb', 'requests', 'jsonschema', 'pydantic==1.10.13']
+dependencies = ['urllib3', 'typer[all]', 'tinydb', 'requests', 'jsonschema', 'pydantic', 'gitpython']
 classifiers = [
     "Programming Language :: Python :: 3",
     "License :: OSI Approved :: MIT License",

accli-0.4/.gitignore

@@ -1,6 +0,0 @@
-build
-dist
-.pypirc
-*.egg-info/
-__pycache__
-.venv

accli-0.4/Dockerfile.dev

@@ -1,2 +0,0 @@
-
-FROM python:3.7.17

accli-0.4/requirements.txt

@@ -1,8 +0,0 @@
-urllib3
-requests
-typer[all]
-tinydb
-twine
-jsonschema
-pydantic==1.10.13
-gitpython==3.1.43