accli 0.5__tar.gz → 0.5.1__tar.gz

{accli-0.5 → accli-0.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: accli
-Version: 0.5
+Version: 0.5.1
 Summary: IIASA Accelerator Client
 Author-email: Wrufesh S <wrufesh@gmail.com>
 License: The MIT License (MIT)

{accli-0.5 → accli-0.5.1}/accli/AcceleratorJobProjectService.py

@@ -1,5 +1,6 @@
 import os
 import io
+import sys
 import typing
 import requests
 import json
@@ -16,6 +17,9 @@ class AccAPIError(Exception):
     #     self.response_data = kwargs.pop('response_data')
     #     super().__init__(*args, **kwargs)
 
+class UnhealthyControlServerError(Exception):
+    pass
+
 retries = urllib3.util.Retry(total=10, backoff_factor=1)
 
 http_client = urllib3.poolmanager.PoolManager(
@@ -47,15 +51,13 @@ class AcceleratorJobProjectService:
         }
 
     def http_client_request(self, *args, **kwargs):
+        
         res = self.http_client.request(*args, **kwargs)
 
         if str(res.status)[0] in ['4', '5']:
             raise AccAPIError(
                 f"Accelerator api error:: status_code={res.status} :: response_data={res.data}", 
-                # status_code=res.status, 
-                # response_data=res.data
             )
-        
         return res
 
     def get_file_stat(self, bucket_object_id):
@@ -114,13 +116,29 @@ class AcceleratorJobProjectService:
         if url:
             resp = self.http_client_request("GET", url, preload_content=False)
             return resp
+
+    def check_job_health(self):
+        try:
+            res = self.http_client_request(
+                "GET",
+                f"{self.cli_base_url}/is-healthy/",
+                headers=self.common_request_headers
+            ).json()
+        except Exception as err:
+            raise UnhealthyControlServerError(str(err))
+        
+        is_healthy = res['is_healthy']
+        return is_healthy
         
     def add_log_file(self, data: bytes, filename):
-        res = self.http_client_request(
-            "GET",
-            f"{self.cli_base_url}/presigned-log-upload-url/?filename={filename}",
-            headers=self.common_request_headers
-        ).json()
+        try:
+            res = self.http_client_request(
+                "GET",
+                f"{self.cli_base_url}/presigned-log-upload-url/?filename={filename}",
+                headers=self.common_request_headers
+            ).json()
+        except Exception as err:
+            raise UnhealthyControlServerError(str(err))
 
         upload_url = res['upload_url']
         app_bucket_id = res['app_bucket_id']
@@ -134,18 +152,21 @@ class AcceleratorJobProjectService:
             verify=False,
         )
 
-        self.http_client_request(
-            "POST",
-            f"{self.cli_base_url}/register-log-file/",
-            json=dict(
-                filename=res_filename,
-                app_bucket_id=app_bucket_id
-            ),
-            headers=self.common_request_headers
-        )
+        try:
+            self.http_client_request(
+                "POST",
+                f"{self.cli_base_url}/register-log-file/",
+                json=dict(
+                    filename=res_filename,
+                    app_bucket_id=app_bucket_id
+                ),
+                headers=self.common_request_headers
+            )
+        except Exception as err:
+            raise UnhealthyControlServerError(str(err))
 
         if not is_healthy:
-            os.Exit(1)
+            raise UnhealthyControlServerError('Unhealthy control server.')
 
         
 
@@ -555,7 +576,7 @@ class Fs:
 
     @staticmethod
     def get_file_url(remote_filepath):
-        user_token = os.environ.get("ACC_JOB_JOB_TOKEN", None)
+        user_token = os.environ.get("ACC_JOB_TOKEN", None)
         server_url = os.environ.get("ACC_JOB_GATEWAY_SERVER", None)
 
         if not (user_token and server_url):

{accli-0.5 → accli-0.5.1}/accli/AcceleratorTaskDispatcher.py

@@ -1,37 +1,46 @@
 import git
 import os
+import zipfile
 import hashlib
 import shutil
 import tempfile
+import requests
 from typing import Optional, List, Dict
 from pydantic.v1 import BaseModel, root_validator
-
-from accli.token import get_github_app_token
+from accli.AcceleratorTerminalCliProjectService import AcceleratorTerminalCliProjectService
+from accli.token import get_github_app_token, get_token, get_server_url, get_project_slug
 
 FOLDER_JOB_REPO_URL = 'https://github.com/IIASA-Accelerator/wkube-job.git'
 
-def hash_folder(folder_path):
-    md5_hash = hashlib.md5()
-    sha256_hash = hashlib.sha256()
 
-    for root, _, files in os.walk(folder_path):
-        for file_name in files:
-            file_path = os.path.join(root, file_name)
-            try:
+def compress_folder(folder_path, output_path):
+    # Fixed timestamp for normalization
+    fixed_time = (1980, 1, 1, 0, 0, 0)
+
+    with zipfile.ZipFile(output_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
+        for root, dirs, files in os.walk(folder_path):
+            dirs.sort()
+            files.sort()
+            for file in files:
+                file_path = os.path.join(root, file)
+                arcname = os.path.relpath(file_path, start=folder_path)
+                # Add file to zip with fixed timestamp
+                info = zipfile.ZipInfo(arcname)
+                info.date_time = fixed_time
+                info.external_attr = 0o600 << 16  # Set file permissions
                 with open(file_path, 'rb') as f:
-                    # Read and update MD5 and SHA256 hash object with file content
-                    for chunk in iter(lambda: f.read(4096), b""):
-                        md5_hash.update(chunk)
-                        sha256_hash.update(chunk)
-            except IOError:
-                # Handle read error (if any)
-                print(f"Error reading file: {file_path}")
+                    
+                    zipf.writestr(info, f.read(), zipfile.ZIP_DEFLATED)
 
-    # Get hexadecimal digest of hashes
-    md5_digest = md5_hash.hexdigest()
-    # sha256_digest = sha256_hash.hexdigest()
+def get_file_sha1(file_path):
+    sha1_hash = hashlib.sha1()
+    with open(file_path, "rb") as f:
+        chunk = f.read(8192)
+        while chunk:
+            sha1_hash.update(chunk)
+            chunk = f.read(8192)
+    return sha1_hash.hexdigest()
 
-    return md5_digest
 
 def copy_tree(src, dst):
     """Recursively copy from src to dst, excluding .git folders."""
@@ -49,35 +58,50 @@ def copy_tree(src, dst):
 
 def push_folder_job(dir):
 
+    access_token = get_token()
+
+    server_url = get_server_url()
+
+    project_slug = get_project_slug()
+
+    term_cli_project_service = AcceleratorTerminalCliProjectService(
+        user_token=access_token,
+        server_url=server_url,
+        verify_cert=False
+    )
+
     repo_dir = tempfile.mkdtemp()
 
     copy_tree(dir, repo_dir)
-    
-    branch_name = hash_folder(repo_dir)
-    
-    repo = git.Repo.init(repo_dir)
 
-    git_token = get_github_app_token()
+    temp_zip_path = f"{repo_dir}/temp.zip"
 
-    remote_url = f'https://x-access-token:{git_token}@{FOLDER_JOB_REPO_URL.split("https://")[1]}'
-    remote = repo.create_remote("accelerator_job_repo", remote_url)
+    compress_folder(repo_dir, temp_zip_path)
+    
+    sha256_hash = get_file_sha1(temp_zip_path)
 
-    remote.pull('master')
+    final_zip_path = f"{repo_dir}/{sha256_hash}.zip"
 
-    try:
-        repo.git.checkout(branch_name)
-    except git.exc.GitCommandError:
-        repo.git.checkout('-b', branch_name)
+    
+    os.rename(temp_zip_path, final_zip_path)
 
-    repo.git.add('.')
+    presigned_push_url = term_cli_project_service.get_jobstore_push_url(
+        project_slug, f"{sha256_hash}.zip"
+    )
 
-    repo.index.commit(branch_name)
 
-    # TODO see what happens with the same checksum
-    remote.push(branch_name)
+    if presigned_push_url:
+        with  open(final_zip_path, 'rb') as f:
+            requests.put(
+                presigned_push_url,
+                data=f,
+                verify=False,
+            )
+    
+    
     shutil.rmtree(repo_dir)
 
-    return FOLDER_JOB_REPO_URL, branch_name
+    return f"s3accjobstore://{sha256_hash}.zip", sha256_hash
 
 
 class JobDispatchModel(BaseModel):
@@ -204,7 +228,7 @@ class WKubeTask(GenericTask):
             
             WKubeTaskPydantic(*t_args, **t_kwargs)
             wkube_task_kwargs = WKubeTaskKwargs(*t_args, **t_kwargs)
-            wkube_task_meta.update(WKubeTaskMeta(*t_args, **t_kwargs).dict())
+            wkube_task_meta.update(WKubeTaskMeta(*t_args, **t_kwargs).dict(exclude_unset=True))
             
 
         self.dispatch_model_task = JobDispatchModel(
@@ -212,7 +236,7 @@ class WKubeTask(GenericTask):
             execute_cluster='WKUBE',
             job_location='acc_native_jobs.dispatch_wkube_task',
             job_args=[],
-            job_kwargs= wkube_task_kwargs.dict() if wkube_task_kwargs else dict(),
+            job_kwargs= wkube_task_kwargs.dict(exclude_unset=True) if wkube_task_kwargs else dict(),
             **wkube_task_meta
         )
 

{accli-0.5 → accli-0.5.1}/accli/AcceleratorTerminalCliProjectService.py

@@ -98,6 +98,19 @@ class AcceleratorTerminalCliProjectService:
 
         return res.json()
     
+    def get_jobstore_push_url(self, project_slug, filename):
+        
+        res = self.http_client_request(
+            "GET", 
+            f"{self.cli_base_url}/{project_slug}/jobstore-push-url/?filename={filename}",
+            headers=self.common_request_headers
+        )
+
+        if res.status == 409 or res.status == '409':
+            return None
+
+        return res.json()
+    
     def dispatch(self, project_slug, job_description):
         
         try:

{accli-0.5 → accli-0.5.1}/accli/_version.py

@@ -1,3 +1,3 @@
 # Please strictly put double quote to use this info for git tag 
 # Read developer guide for git tag command with regex
-VERSION = "v0.5" 
+VERSION = "v0.5.1" 

{accli-0.5 → accli-0.5.1}/accli/cli.py

@@ -9,7 +9,7 @@ import importlib.util
 from rich import print
 from typing_extensions import Annotated
 
-from accli.token import save_token_details, get_token, get_server_url, set_github_app_token
+from accli.token import save_token_details, get_token, get_server_url, set_github_app_token, set_project_slug
 
 from accli.CsvRegionalTimeseriesValidator import CsvRegionalTimeseriesValidator
 from ._version import VERSION
@@ -181,7 +181,7 @@ def dispatch(
     root_task_variable: Annotated[str, typer.Argument(help="Root task variable in workflow_file.")],
     server: Annotated[str, typer.Option(help="Accelerator server url.")] = "https://accelerator-api.iiasa.ac.at",
 ):
-    
+    set_project_slug(project_slug)
     access_token = get_token()
 
     server_url = get_server_url()

{accli-0.5 → accli-0.5.1}/accli/token.py

@@ -61,6 +61,25 @@ def set_github_app_token(github_app_token):
     db = TinyDB(db_path)
     db.update({'github_app_token': github_app_token}, doc_ids=[1])
 
+def set_project_slug(project_slug):
+    db_path = get_db_path()
+    db = TinyDB(db_path)
+    db.update({'project_slug': project_slug}, doc_ids=[1])
+
+def get_project_slug():
+    db_path = get_db_path()
+
+    db = TinyDB(db_path)
+
+    for item in db:
+        project_slug = item.get('project_slug')
+        if project_slug:
+            break
+
+    if not project_slug:
+        print("project slug was not set.")
+    return project_slug
+
 
 def get_server_url():
     db_path = get_db_path()

{accli-0.5 → accli-0.5.1}/accli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: accli
-Version: 0.5
+Version: 0.5.1
 Summary: IIASA Accelerator Client
 Author-email: Wrufesh S <wrufesh@gmail.com>
 License: The MIT License (MIT)