accli 0.4.1__tar.gz → 0.5.1__tar.gz

{accli-0.4.1/accli.egg-info → accli-0.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: accli
-Version: 0.4.1
+Version: 0.5.1
 Summary: IIASA Accelerator Client
 Author-email: Wrufesh S <wrufesh@gmail.com>
 License: The MIT License (MIT)

{accli-0.4.1 → accli-0.5.1}/accli/AcceleratorJobProjectService.py

@@ -1,5 +1,6 @@
 import os
 import io
+import sys
 import typing
 import requests
 import json
@@ -16,6 +17,19 @@ class AccAPIError(Exception):
     #     self.response_data = kwargs.pop('response_data')
     #     super().__init__(*args, **kwargs)
 
+class UnhealthyControlServerError(Exception):
+    pass
+
+retries = urllib3.util.Retry(total=10, backoff_factor=1)
+
+http_client = urllib3.poolmanager.PoolManager(
+    num_pools=20, retries=retries
+)
+
+http_client_wo_cert_verification = urllib3.poolmanager.PoolManager(
+    cert_reqs="CERT_NONE", num_pools=20, retries=retries
+)
+
 class AcceleratorJobProjectService:
     def __init__(
             self,
@@ -26,12 +40,10 @@ class AcceleratorJobProjectService:
         
         self.user_token = user_token
 
-        retries = urllib3.util.Retry(total=10, backoff_factor=1)
-
         if verify_cert:
-            self.http_client = urllib3.poolmanager.PoolManager(num_pools=1, retries=retries)
+            self.http_client = http_client
         else:
-            self.http_client = urllib3.poolmanager.PoolManager(cert_reqs="CERT_NONE", num_pools=1, retries=retries)
+            self.http_client = http_client_wo_cert_verification
 
         self.cli_base_url = f"{server_url}/v1/ajob-cli"
         self.common_request_headers = {
@@ -39,15 +51,13 @@ class AcceleratorJobProjectService:
         }
 
     def http_client_request(self, *args, **kwargs):
+        
         res = self.http_client.request(*args, **kwargs)
 
         if str(res.status)[0] in ['4', '5']:
             raise AccAPIError(
                 f"Accelerator api error:: status_code={res.status} :: response_data={res.data}", 
-                # status_code=res.status, 
-                # response_data=res.data
             )
-        
         return res
 
     def get_file_stat(self, bucket_object_id):
@@ -58,6 +68,16 @@ class AcceleratorJobProjectService:
         )
         return res.json()
     
+    def get_file_url_from_repo(self, filename):
+        project_slug = filename.split('/')[0]
+        res = self.http_client_request(
+            "GET", 
+            f"{self.cli_base_url}/{project_slug}/get-file-download-url/?filename={filename}",
+            headers=self.common_request_headers
+        )
+        if res.data:
+            return res.json()
+    
     def get_dataset_type(self, *args, **kwargs):
         return self.get_bucket_object_validation_type(*args, **kwargs)
 
@@ -96,17 +116,34 @@ class AcceleratorJobProjectService:
         if url:
             resp = self.http_client_request("GET", url, preload_content=False)
             return resp
+
+    def check_job_health(self):
+        try:
+            res = self.http_client_request(
+                "GET",
+                f"{self.cli_base_url}/is-healthy/",
+                headers=self.common_request_headers
+            ).json()
+        except Exception as err:
+            raise UnhealthyControlServerError(str(err))
+        
+        is_healthy = res['is_healthy']
+        return is_healthy
         
     def add_log_file(self, data: bytes, filename):
-        res = self.http_client_request(
-            "GET",
-            f"{self.cli_base_url}/presigned-upload-url/?filename={filename}",
-            headers=self.common_request_headers
-        ).json()
+        try:
+            res = self.http_client_request(
+                "GET",
+                f"{self.cli_base_url}/presigned-log-upload-url/?filename={filename}",
+                headers=self.common_request_headers
+            ).json()
+        except Exception as err:
+            raise UnhealthyControlServerError(str(err))
 
         upload_url = res['upload_url']
         app_bucket_id = res['app_bucket_id']
         res_filename = res['filename']
+        is_healthy = res['is_healthy']
 
         requests.put(
             upload_url,
@@ -115,15 +152,21 @@ class AcceleratorJobProjectService:
             verify=False,
         )
 
-        self.http_client_request(
-            "POST",
-            f"{self.cli_base_url}/register-log-file/",
-            json=dict(
-                filename=res_filename,
-                app_bucket_id=app_bucket_id
-            ),
-            headers=self.common_request_headers
-        )
+        try:
+            self.http_client_request(
+                "POST",
+                f"{self.cli_base_url}/register-log-file/",
+                json=dict(
+                    filename=res_filename,
+                    app_bucket_id=app_bucket_id
+                ),
+                headers=self.common_request_headers
+            )
+        except Exception as err:
+            raise UnhealthyControlServerError(str(err))
+
+        if not is_healthy:
+            raise UnhealthyControlServerError('Unhealthy control server.')
 
         
 
@@ -531,9 +574,28 @@ class Fs:
         with open(dest_filepath, 'wb') as file:
             file.write(filestream.getvalue())
 
+    @staticmethod
+    def get_file_url(remote_filepath):
+        user_token = os.environ.get("ACC_JOB_TOKEN", None)
+        server_url = os.environ.get("ACC_JOB_GATEWAY_SERVER", None)
+
+        if not (user_token and server_url):
+            raise ValueError("Remote data repository credentials not found.")
+        
+        accelerator_job_service = AcceleratorJobProjectService(
+            user_token, 
+            server_url=server_url,
+            verify_cert=True
+        )
+
+        return accelerator_job_service.get_file_url_from_repo(
+            remote_filepath
+        )
+        
+
     @staticmethod
     def write_file(source: typing.Union[str, io.BytesIO], dest_filepath):
-        user_token = os.environ.get("ACC_JOB_JOB_TOKEN", None)
+        user_token = os.environ.get("ACC_JOB_TOKEN", None)
         server_url = os.environ.get("ACC_JOB_GATEWAY_SERVER", None)
 
         if isinstance(source, io.BytesIO):

{accli-0.4.1 → accli-0.5.1}/accli/AcceleratorTaskDispatcher.py

@@ -1,37 +1,46 @@
 import git
 import os
+import zipfile
 import hashlib
 import shutil
 import tempfile
+import requests
 from typing import Optional, List, Dict
-from pydantic import BaseModel, root_validator
-
-from accli.token import get_github_app_token
+from pydantic.v1 import BaseModel, root_validator
+from accli.AcceleratorTerminalCliProjectService import AcceleratorTerminalCliProjectService
+from accli.token import get_github_app_token, get_token, get_server_url, get_project_slug
 
 FOLDER_JOB_REPO_URL = 'https://github.com/IIASA-Accelerator/wkube-job.git'
 
-def hash_folder(folder_path):
-    md5_hash = hashlib.md5()
-    sha256_hash = hashlib.sha256()
 
-    for root, _, files in os.walk(folder_path):
-        for file_name in files:
-            file_path = os.path.join(root, file_name)
-            try:
+def compress_folder(folder_path, output_path):
+    # Fixed timestamp for normalization
+    fixed_time = (1980, 1, 1, 0, 0, 0)
+
+    with zipfile.ZipFile(output_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
+        for root, dirs, files in os.walk(folder_path):
+            dirs.sort()
+            files.sort()
+            for file in files:
+                file_path = os.path.join(root, file)
+                arcname = os.path.relpath(file_path, start=folder_path)
+                # Add file to zip with fixed timestamp
+                info = zipfile.ZipInfo(arcname)
+                info.date_time = fixed_time
+                info.external_attr = 0o600 << 16  # Set file permissions
                 with open(file_path, 'rb') as f:
-                    # Read and update MD5 and SHA256 hash object with file content
-                    for chunk in iter(lambda: f.read(4096), b""):
-                        md5_hash.update(chunk)
-                        sha256_hash.update(chunk)
-            except IOError:
-                # Handle read error (if any)
-                print(f"Error reading file: {file_path}")
+                    
+                    zipf.writestr(info, f.read(), zipfile.ZIP_DEFLATED)
 
-    # Get hexadecimal digest of hashes
-    md5_digest = md5_hash.hexdigest()
-    # sha256_digest = sha256_hash.hexdigest()
+def get_file_sha1(file_path):
+    sha1_hash = hashlib.sha1()
+    with open(file_path, "rb") as f:
+        chunk = f.read(8192)
+        while chunk:
+            sha1_hash.update(chunk)
+            chunk = f.read(8192)
+    return sha1_hash.hexdigest()
 
-    return md5_digest
 
 def copy_tree(src, dst):
     """Recursively copy from src to dst, excluding .git folders."""
@@ -49,35 +58,50 @@ def copy_tree(src, dst):
 
 def push_folder_job(dir):
 
+    access_token = get_token()
+
+    server_url = get_server_url()
+
+    project_slug = get_project_slug()
+
+    term_cli_project_service = AcceleratorTerminalCliProjectService(
+        user_token=access_token,
+        server_url=server_url,
+        verify_cert=False
+    )
+
     repo_dir = tempfile.mkdtemp()
 
     copy_tree(dir, repo_dir)
-    
-    branch_name = hash_folder(repo_dir)
-    
-    repo = git.Repo.init(repo_dir)
 
-    git_token = get_github_app_token()
+    temp_zip_path = f"{repo_dir}/temp.zip"
 
-    remote_url = f'https://x-access-token:{git_token}@{FOLDER_JOB_REPO_URL.split("https://")[1]}'
-    remote = repo.create_remote("accelerator_job_repo", remote_url)
+    compress_folder(repo_dir, temp_zip_path)
+    
+    sha256_hash = get_file_sha1(temp_zip_path)
 
-    remote.pull('master')
+    final_zip_path = f"{repo_dir}/{sha256_hash}.zip"
 
-    try:
-        repo.git.checkout(branch_name)
-    except git.exc.GitCommandError:
-        repo.git.checkout('-b', branch_name)
+    
+    os.rename(temp_zip_path, final_zip_path)
 
-    repo.git.add('.')
+    presigned_push_url = term_cli_project_service.get_jobstore_push_url(
+        project_slug, f"{sha256_hash}.zip"
+    )
 
-    repo.index.commit(branch_name)
 
-    # TODO see what happens with the same checksum
-    remote.push(branch_name)
+    if presigned_push_url:
+        with  open(final_zip_path, 'rb') as f:
+            requests.put(
+                presigned_push_url,
+                data=f,
+                verify=False,
+            )
+    
+    
     shutil.rmtree(repo_dir)
 
-    return FOLDER_JOB_REPO_URL, branch_name
+    return f"s3accjobstore://{sha256_hash}.zip", sha256_hash
 
 
 class JobDispatchModel(BaseModel):
@@ -139,6 +163,8 @@ class WKubeTaskKwargs(BaseModel):
 
     conf: Dict[str,str] = {}
 
+    build_timeout: Optional[int]
+
     def dict(self, *args, **kwargs):
         result = super().dict(*args, **kwargs)
         if 'job_folder' in result:
@@ -202,7 +228,7 @@ class WKubeTask(GenericTask):
             
             WKubeTaskPydantic(*t_args, **t_kwargs)
             wkube_task_kwargs = WKubeTaskKwargs(*t_args, **t_kwargs)
-            wkube_task_meta.update(WKubeTaskMeta(*t_args, **t_kwargs).dict())
+            wkube_task_meta.update(WKubeTaskMeta(*t_args, **t_kwargs).dict(exclude_unset=True))
             
 
         self.dispatch_model_task = JobDispatchModel(
@@ -210,7 +236,7 @@ class WKubeTask(GenericTask):
             execute_cluster='WKUBE',
             job_location='acc_native_jobs.dispatch_wkube_task',
             job_args=[],
-            job_kwargs= wkube_task_kwargs.dict() if wkube_task_kwargs else dict(),
+            job_kwargs= wkube_task_kwargs.dict(exclude_unset=True) if wkube_task_kwargs else dict(),
             **wkube_task_meta
         )
 

{accli-0.4.1 → accli-0.5.1}/accli/AcceleratorTerminalCliProjectService.py

@@ -18,6 +18,16 @@ class AccAPIError(Exception):
             self.response = response
             self.status_code = status_code
 
+retries = urllib3.util.Retry(total=10, backoff_factor=1)
+
+http_client = urllib3.poolmanager.PoolManager(
+    num_pools=20, retries=retries
+)
+
+http_client_wo_cert_verification = urllib3.poolmanager.PoolManager(
+    cert_reqs="CERT_NONE", num_pools=20, retries=retries
+)
+
 class AcceleratorTerminalCliProjectService:
     def __init__(
             self,
@@ -28,12 +38,10 @@ class AcceleratorTerminalCliProjectService:
         
         self.user_token = user_token
 
-        retries = urllib3.util.Retry(total=10, backoff_factor=1)
-
         if verify_cert:
-            self.http_client = urllib3.poolmanager.PoolManager(num_pools=1, retries=retries)
+            self.http_client = http_client
         else:
-            self.http_client = urllib3.poolmanager.PoolManager(cert_reqs="CERT_NONE", num_pools=1, retries=retries)
+            self.http_client = http_client_wo_cert_verification
 
         self.server_url = server_url
         self.cli_base_url = f"{self.server_url}/v1/aterm-cli"
@@ -70,6 +78,16 @@ class AcceleratorTerminalCliProjectService:
 
         return res.json()
     
+    def get_file_url_from_repo(self, filename):
+        project_slug = filename.split('/')[0]
+        res = self.http_client_request(
+            "GET", 
+            f"{self.cli_base_url}/{project_slug}/get-file-download-url/?filename={filename}",
+            headers=self.common_request_headers
+        )
+        if res.data:
+            return res.json()
+    
     def get_github_app_token(self, project_slug):
         
         res = self.http_client_request(
@@ -80,6 +98,19 @@ class AcceleratorTerminalCliProjectService:
 
         return res.json()
     
+    def get_jobstore_push_url(self, project_slug, filename):
+        
+        res = self.http_client_request(
+            "GET", 
+            f"{self.cli_base_url}/{project_slug}/jobstore-push-url/?filename={filename}",
+            headers=self.common_request_headers
+        )
+
+        if res.status == 409 or res.status == '409':
+            return None
+
+        return res.json()
+    
     def dispatch(self, project_slug, job_description):
         
         try:

{accli-0.4.1 → accli-0.5.1}/accli/_version.py

@@ -1,3 +1,3 @@
 # Please strictly put double quote to use this info for git tag 
 # Read developer guide for git tag command with regex
-VERSION = "v0.4.1" 
+VERSION = "v0.5.1" 

{accli-0.4.1 → accli-0.5.1}/accli/cli.py

@@ -9,7 +9,7 @@ import importlib.util
 from rich import print
 from typing_extensions import Annotated
 
-from accli.token import save_token_details, get_token, get_server_url, set_github_app_token
+from accli.token import save_token_details, get_token, get_server_url, set_github_app_token, set_project_slug
 
 from accli.CsvRegionalTimeseriesValidator import CsvRegionalTimeseriesValidator
 from ._version import VERSION
@@ -181,7 +181,7 @@ def dispatch(
     root_task_variable: Annotated[str, typer.Argument(help="Root task variable in workflow_file.")],
     server: Annotated[str, typer.Option(help="Accelerator server url.")] = "https://accelerator-api.iiasa.ac.at",
 ):
-    
+    set_project_slug(project_slug)
     access_token = get_token()
 
     server_url = get_server_url()

{accli-0.4.1 → accli-0.5.1}/accli/token.py

@@ -61,6 +61,25 @@ def set_github_app_token(github_app_token):
     db = TinyDB(db_path)
     db.update({'github_app_token': github_app_token}, doc_ids=[1])
 
+def set_project_slug(project_slug):
+    db_path = get_db_path()
+    db = TinyDB(db_path)
+    db.update({'project_slug': project_slug}, doc_ids=[1])
+
+def get_project_slug():
+    db_path = get_db_path()
+
+    db = TinyDB(db_path)
+
+    for item in db:
+        project_slug = item.get('project_slug')
+        if project_slug:
+            break
+
+    if not project_slug:
+        print("project slug was not set.")
+    return project_slug
+
 
 def get_server_url():
     db_path = get_db_path()

{accli-0.4.1 → accli-0.5.1/accli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: accli
-Version: 0.4.1
+Version: 0.5.1
 Summary: IIASA Accelerator Client
 Author-email: Wrufesh S <wrufesh@gmail.com>
 License: The MIT License (MIT)

{accli-0.4.1 → accli-0.5.1}/accli.egg-info/SOURCES.txt

@@ -1,9 +1,6 @@
-.gitignore
-Dockerfile.dev
 LICENSE
 README.md
 pyproject.toml
-requirements.txt
 accli/AcceleratorJobProjectService.py
 accli/AcceleratorTaskDispatcher.py
 accli/AcceleratorTerminalCliProjectService.py

{accli-0.4.1 → accli-0.5.1}/accli.egg-info/requires.txt

@@ -3,5 +3,5 @@ typer[all]
 tinydb
 requests
 jsonschema
-pydantic==1.10.13
+pydantic
 gitpython

{accli-0.4.1 → accli-0.5.1}/pyproject.toml

@@ -12,7 +12,7 @@ description = "IIASA Accelerator Client"
 readme = "README.md"
 license = {file = "LICENSE"}
 requires-python = ">=3.7.17"
-dependencies = ['urllib3', 'typer[all]', 'tinydb', 'requests', 'jsonschema', 'pydantic==1.10.13', 'gitpython']
+dependencies = ['urllib3', 'typer[all]', 'tinydb', 'requests', 'jsonschema', 'pydantic', 'gitpython']
 classifiers = [
     "Programming Language :: Python :: 3",
     "License :: OSI Approved :: MIT License",

accli-0.4.1/.gitignore

@@ -1,6 +0,0 @@
-build
-dist
-.pypirc
-*.egg-info/
-__pycache__
-.venv

accli-0.4.1/Dockerfile.dev

@@ -1,2 +0,0 @@
-
-FROM python:3.7.17

accli-0.4.1/requirements.txt

@@ -1,8 +0,0 @@
-urllib3
-requests
-typer[all]
-tinydb
-twine
-jsonschema
-pydantic==1.10.13
-gitpython==3.1.43