accli 0.5__tar.gz → 0.5.2__tar.gz

{accli-0.5 → accli-0.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: accli
-Version: 0.5
+Version: 0.5.2
 Summary: IIASA Accelerator Client
 Author-email: Wrufesh S <wrufesh@gmail.com>
 License: The MIT License (MIT)
@@ -83,5 +83,3 @@ Please follow [this link.](https://packaging.python.org/en/latest/tutorials/pack
 2. Run 'python scripts/tag.py'
 3. `python -m build`
 4. `twine upload -r pypi -u __token__ -p <password-or-token> ./dist/*`
-
-

{accli-0.5 → accli-0.5.2}/README.md

@@ -48,5 +48,3 @@ Please follow [this link.](https://packaging.python.org/en/latest/tutorials/pack
 2. Run 'python scripts/tag.py'
 3. `python -m build`
 4. `twine upload -r pypi -u __token__ -p <password-or-token> ./dist/*`
-
-

{accli-0.5 → accli-0.5.2}/accli/AcceleratorJobProjectService.py

@@ -1,5 +1,6 @@
 import os
 import io
+import sys
 import typing
 import requests
 import json
@@ -8,6 +9,8 @@ import urllib3
 from pathlib import Path
 from typing import List, Tuple
 
+from accli.common import todict
+
 class AccAPIError(Exception):
     pass
 
@@ -16,6 +19,9 @@ class AccAPIError(Exception):
     #     self.response_data = kwargs.pop('response_data')
     #     super().__init__(*args, **kwargs)
 
+class UnhealthyControlServerError(Exception):
+    pass
+
 retries = urllib3.util.Retry(total=10, backoff_factor=1)
 
 http_client = urllib3.poolmanager.PoolManager(
@@ -47,15 +53,13 @@ class AcceleratorJobProjectService:
         }
 
     def http_client_request(self, *args, **kwargs):
+        
         res = self.http_client.request(*args, **kwargs)
 
         if str(res.status)[0] in ['4', '5']:
             raise AccAPIError(
                 f"Accelerator api error:: status_code={res.status} :: response_data={res.data}", 
-                # status_code=res.status, 
-                # response_data=res.data
             )
-        
         return res
 
     def get_file_stat(self, bucket_object_id):
@@ -64,7 +68,7 @@ class AcceleratorJobProjectService:
             f"{self.cli_base_url}/file-stat/{bucket_object_id}",
             headers=self.common_request_headers
         )
-        return res.json()
+        return todict(res.data)
     
     def get_file_url_from_repo(self, filename):
         project_slug = filename.split('/')[0]
@@ -74,7 +78,7 @@ class AcceleratorJobProjectService:
             headers=self.common_request_headers
         )
         if res.data:
-            return res.json()
+            return todict(res.data)
     
     def get_dataset_type(self, *args, **kwargs):
         return self.get_bucket_object_validation_type(*args, **kwargs)
@@ -86,7 +90,7 @@ class AcceleratorJobProjectService:
             headers=self.common_request_headers
         )
         if res.data:
-            return res.json()
+            return todict(res.data)
 
     def get_bucket_object_validation_details(self, bucket_object_id):
         res = self.http_client_request(
@@ -95,7 +99,7 @@ class AcceleratorJobProjectService:
             headers=self.common_request_headers
         )
         if res.data:
-            return res.json()
+            return todict(res.data)
 
     
     def get_file_url(self, bucket_object_id):
@@ -106,7 +110,7 @@ class AcceleratorJobProjectService:
         )
 
         if res.data:
-            return res.json()
+            return todict(res.data)
 
 
     def get_file_stream(self, bucket_object_id):
@@ -114,13 +118,33 @@ class AcceleratorJobProjectService:
         if url:
             resp = self.http_client_request("GET", url, preload_content=False)
             return resp
+
+    def check_job_health(self):
+        try:
+            res = self.http_client_request(
+                "GET",
+                f"{self.cli_base_url}/is-healthy/",
+                headers=self.common_request_headers
+            )
+            if res.data:
+                res = todict(res.data)
+        except Exception as err:
+            raise UnhealthyControlServerError(str(err))
+        
+        is_healthy = res['is_healthy']
+        return is_healthy
         
     def add_log_file(self, data: bytes, filename):
-        res = self.http_client_request(
-            "GET",
-            f"{self.cli_base_url}/presigned-log-upload-url/?filename={filename}",
-            headers=self.common_request_headers
-        ).json()
+        try:
+            res = self.http_client_request(
+                "GET",
+                f"{self.cli_base_url}/presigned-log-upload-url/?filename={filename}",
+                headers=self.common_request_headers
+            )
+            if res.data:
+                res = todict(res.data)
+        except Exception as err:
+            raise UnhealthyControlServerError(str(err))
 
         upload_url = res['upload_url']
         app_bucket_id = res['app_bucket_id']
@@ -134,18 +158,21 @@ class AcceleratorJobProjectService:
             verify=False,
         )
 
-        self.http_client_request(
-            "POST",
-            f"{self.cli_base_url}/register-log-file/",
-            json=dict(
-                filename=res_filename,
-                app_bucket_id=app_bucket_id
-            ),
-            headers=self.common_request_headers
-        )
+        try:
+            self.http_client_request(
+                "POST",
+                f"{self.cli_base_url}/register-log-file/",
+                json=dict(
+                    filename=res_filename,
+                    app_bucket_id=app_bucket_id
+                ),
+                headers=self.common_request_headers
+            )
+        except Exception as err:
+            raise UnhealthyControlServerError(str(err))
 
         if not is_healthy:
-            os.Exit(1)
+            raise UnhealthyControlServerError('Unhealthy control server.')
 
         
 
@@ -168,7 +195,7 @@ class AcceleratorJobProjectService:
             headers=self.common_request_headers
         )
 
-        return res.json()
+        return todict(res.data)
 
     def get_multipart_put_update_signed_url(
         self,
@@ -187,7 +214,7 @@ class AcceleratorJobProjectService:
             headers=self.common_request_headers
         )
 
-        return res.json()
+        return todict(res.data)
 
     def get_put_create_multipart_upload_id(self, filename):
         res = self.http_client_request(
@@ -196,7 +223,7 @@ class AcceleratorJobProjectService:
             headers=self.common_request_headers
         )
 
-        data = res.json()
+        data = todict(res.data)
 
         return data['upload_id'], data['app_bucket_id'], data['uniqified_filename']
 
@@ -207,7 +234,7 @@ class AcceleratorJobProjectService:
             headers=self.common_request_headers
         )
 
-        return res.json()
+        return todict(res.data)
 
     def complete_job_multipart_upload(
         self,
@@ -234,7 +261,7 @@ class AcceleratorJobProjectService:
             headers=headers
         )
 
-        return res.json()
+        return todict(res.data)
 
     def complete_update_multipart_upload(
         self, bucket_object_id, upload_id, parts: List[Tuple[str, str]]
@@ -344,7 +371,7 @@ class AcceleratorJobProjectService:
             headers=headers
         )
 
-        return res.json()
+        return todict(res.data)
 
       
 
@@ -555,7 +582,7 @@ class Fs:
 
     @staticmethod
     def get_file_url(remote_filepath):
-        user_token = os.environ.get("ACC_JOB_JOB_TOKEN", None)
+        user_token = os.environ.get("ACC_JOB_TOKEN", None)
         server_url = os.environ.get("ACC_JOB_GATEWAY_SERVER", None)
 
         if not (user_token and server_url):

{accli-0.5 → accli-0.5.2}/accli/AcceleratorTaskDispatcher.py

@@ -1,37 +1,46 @@
 import git
 import os
+import zipfile
 import hashlib
 import shutil
 import tempfile
+import requests
 from typing import Optional, List, Dict
 from pydantic.v1 import BaseModel, root_validator
-
-from accli.token import get_github_app_token
+from accli.AcceleratorTerminalCliProjectService import AcceleratorTerminalCliProjectService
+from accli.token import get_github_app_token, get_token, get_server_url, get_project_slug
 
 FOLDER_JOB_REPO_URL = 'https://github.com/IIASA-Accelerator/wkube-job.git'
 
-def hash_folder(folder_path):
-    md5_hash = hashlib.md5()
-    sha256_hash = hashlib.sha256()
 
-    for root, _, files in os.walk(folder_path):
-        for file_name in files:
-            file_path = os.path.join(root, file_name)
-            try:
+def compress_folder(folder_path, output_path):
+    # Fixed timestamp for normalization
+    fixed_time = (1980, 1, 1, 0, 0, 0)
+
+    with zipfile.ZipFile(output_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
+        for root, dirs, files in os.walk(folder_path):
+            dirs.sort()
+            files.sort()
+            for file in files:
+                file_path = os.path.join(root, file)
+                arcname = os.path.relpath(file_path, start=folder_path)
+                # Add file to zip with fixed timestamp
+                info = zipfile.ZipInfo(arcname)
+                info.date_time = fixed_time
+                info.external_attr = 0o600 << 16  # Set file permissions
                 with open(file_path, 'rb') as f:
-                    # Read and update MD5 and SHA256 hash object with file content
-                    for chunk in iter(lambda: f.read(4096), b""):
-                        md5_hash.update(chunk)
-                        sha256_hash.update(chunk)
-            except IOError:
-                # Handle read error (if any)
-                print(f"Error reading file: {file_path}")
+                    
+                    zipf.writestr(info, f.read(), zipfile.ZIP_DEFLATED)
 
-    # Get hexadecimal digest of hashes
-    md5_digest = md5_hash.hexdigest()
-    # sha256_digest = sha256_hash.hexdigest()
+def get_file_sha1(file_path):
+    sha1_hash = hashlib.sha1()
+    with open(file_path, "rb") as f:
+        chunk = f.read(8192)
+        while chunk:
+            sha1_hash.update(chunk)
+            chunk = f.read(8192)
+    return sha1_hash.hexdigest()
 
-    return md5_digest
 
 def copy_tree(src, dst):
     """Recursively copy from src to dst, excluding .git folders."""
@@ -49,35 +58,50 @@ def copy_tree(src, dst):
 
 def push_folder_job(dir):
 
+    access_token = get_token()
+
+    server_url = get_server_url()
+
+    project_slug = get_project_slug()
+
+    term_cli_project_service = AcceleratorTerminalCliProjectService(
+        user_token=access_token,
+        server_url=server_url,
+        verify_cert=False
+    )
+
     repo_dir = tempfile.mkdtemp()
 
     copy_tree(dir, repo_dir)
-    
-    branch_name = hash_folder(repo_dir)
-    
-    repo = git.Repo.init(repo_dir)
 
-    git_token = get_github_app_token()
+    temp_zip_path = f"{repo_dir}/temp.zip"
 
-    remote_url = f'https://x-access-token:{git_token}@{FOLDER_JOB_REPO_URL.split("https://")[1]}'
-    remote = repo.create_remote("accelerator_job_repo", remote_url)
+    compress_folder(repo_dir, temp_zip_path)
+    
+    sha256_hash = get_file_sha1(temp_zip_path)
 
-    remote.pull('master')
+    final_zip_path = f"{repo_dir}/{sha256_hash}.zip"
 
-    try:
-        repo.git.checkout(branch_name)
-    except git.exc.GitCommandError:
-        repo.git.checkout('-b', branch_name)
+    
+    os.rename(temp_zip_path, final_zip_path)
 
-    repo.git.add('.')
+    presigned_push_url = term_cli_project_service.get_jobstore_push_url(
+        project_slug, f"{sha256_hash}.zip"
+    )
 
-    repo.index.commit(branch_name)
 
-    # TODO see what happens with the same checksum
-    remote.push(branch_name)
+    if presigned_push_url:
+        with  open(final_zip_path, 'rb') as f:
+            requests.put(
+                presigned_push_url,
+                data=f,
+                verify=False,
+            )
+    
+    
     shutil.rmtree(repo_dir)
 
-    return FOLDER_JOB_REPO_URL, branch_name
+    return f"s3accjobstore://{sha256_hash}.zip", sha256_hash
 
 
 class JobDispatchModel(BaseModel):
@@ -204,7 +228,7 @@ class WKubeTask(GenericTask):
             
             WKubeTaskPydantic(*t_args, **t_kwargs)
             wkube_task_kwargs = WKubeTaskKwargs(*t_args, **t_kwargs)
-            wkube_task_meta.update(WKubeTaskMeta(*t_args, **t_kwargs).dict())
+            wkube_task_meta.update(WKubeTaskMeta(*t_args, **t_kwargs).dict(exclude_unset=True))
             
 
         self.dispatch_model_task = JobDispatchModel(
@@ -212,7 +236,7 @@ class WKubeTask(GenericTask):
             execute_cluster='WKUBE',
             job_location='acc_native_jobs.dispatch_wkube_task',
             job_args=[],
-            job_kwargs= wkube_task_kwargs.dict() if wkube_task_kwargs else dict(),
+            job_kwargs= wkube_task_kwargs.dict(exclude_unset=True) if wkube_task_kwargs else dict(),
             **wkube_task_meta
         )
 

{accli-0.5 → accli-0.5.2}/accli/AcceleratorTerminalCliProjectService.py

@@ -7,7 +7,7 @@ import concurrent.futures
 from typing import List, Tuple
 from rich.progress import Progress
 
-
+from accli.common import todict
 
 class AccAPIError(Exception):
 
@@ -76,7 +76,7 @@ class AcceleratorTerminalCliProjectService:
             else:
                 raise err
 
-        return res.json()
+        return todict(res.data)
     
     def get_file_url_from_repo(self, filename):
         project_slug = filename.split('/')[0]
@@ -86,7 +86,7 @@ class AcceleratorTerminalCliProjectService:
             headers=self.common_request_headers
         )
         if res.data:
-            return res.json()
+            return todict(res.data)
     
     def get_github_app_token(self, project_slug):
         
@@ -96,7 +96,20 @@ class AcceleratorTerminalCliProjectService:
             headers=self.common_request_headers
         )
 
-        return res.json()
+        return todict(res.data)
+    
+    def get_jobstore_push_url(self, project_slug, filename):
+        
+        res = self.http_client_request(
+            "GET", 
+            f"{self.cli_base_url}/{project_slug}/jobstore-push-url/?filename={filename}",
+            headers=self.common_request_headers
+        )
+
+        if res.status == 409 or res.status == '409':
+            return None
+
+        return todict(res.data)
     
     def dispatch(self, project_slug, job_description):
         
@@ -110,7 +123,7 @@ class AcceleratorTerminalCliProjectService:
         except AccAPIError as err:
                 raise err
 
-        return res.json()['job_id']
+        return todict(res.data)['job_id']
     
     def get_dataset_template_details(self, project_slug, template_slug):
         
@@ -125,7 +138,7 @@ class AcceleratorTerminalCliProjectService:
             else:
                 raise err
 
-        return res.json()
+        return todict(res.data)
 
     
     def get_multipart_put_create_signed_url(
@@ -148,7 +161,7 @@ class AcceleratorTerminalCliProjectService:
             headers=self.common_request_headers
         )
 
-        return res.json()
+        return todict(res.data)
 
     
 
@@ -162,7 +175,7 @@ class AcceleratorTerminalCliProjectService:
             headers=self.common_request_headers
         )
 
-        data = res.json()
+        data = todict(res.data)
 
         return data['upload_id'], data['app_bucket_id'], data['uniqified_filename']
 
@@ -191,7 +204,7 @@ class AcceleratorTerminalCliProjectService:
             headers=headers
         )
 
-        return res.json()
+        return todict(res.data)
 
     
     def abort_create_multipart_upload(self, project_slug, app_bucket_id, filename, upload_id):

{accli-0.5 → accli-0.5.2}/accli/_version.py

@@ -1,3 +1,3 @@
 # Please strictly put double quote to use this info for git tag 
 # Read developer guide for git tag command with regex
-VERSION = "v0.5" 
+VERSION = "v0.5.2" 

{accli-0.5 → accli-0.5.2}/accli/cli.py

@@ -9,7 +9,7 @@ import importlib.util
 from rich import print
 from typing_extensions import Annotated
 
-from accli.token import save_token_details, get_token, get_server_url, set_github_app_token
+from accli.token import save_token_details, get_token, get_server_url, set_github_app_token, set_project_slug
 
 from accli.CsvRegionalTimeseriesValidator import CsvRegionalTimeseriesValidator
 from ._version import VERSION
@@ -181,7 +181,7 @@ def dispatch(
     root_task_variable: Annotated[str, typer.Argument(help="Root task variable in workflow_file.")],
     server: Annotated[str, typer.Option(help="Accelerator server url.")] = "https://accelerator-api.iiasa.ac.at",
 ):
-    
+    set_project_slug(project_slug)
     access_token = get_token()
 
     server_url = get_server_url()

accli-0.5.2/accli/common.py

@@ -0,0 +1,6 @@
+import json
+
+def todict(bytestring: bytes):
+    return json.loads(
+        bytestring.decode("utf-8")
+    )

{accli-0.5 → accli-0.5.2}/accli/token.py

@@ -61,6 +61,25 @@ def set_github_app_token(github_app_token):
     db = TinyDB(db_path)
     db.update({'github_app_token': github_app_token}, doc_ids=[1])
 
+def set_project_slug(project_slug):
+    db_path = get_db_path()
+    db = TinyDB(db_path)
+    db.update({'project_slug': project_slug}, doc_ids=[1])
+
+def get_project_slug():
+    db_path = get_db_path()
+
+    db = TinyDB(db_path)
+
+    for item in db:
+        project_slug = item.get('project_slug')
+        if project_slug:
+            break
+
+    if not project_slug:
+        print("project slug was not set.")
+    return project_slug
+
 
 def get_server_url():
     db_path = get_db_path()

{accli-0.5 → accli-0.5.2}/accli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: accli
-Version: 0.5
+Version: 0.5.2
 Summary: IIASA Accelerator Client
 Author-email: Wrufesh S <wrufesh@gmail.com>
 License: The MIT License (MIT)
@@ -83,5 +83,3 @@ Please follow [this link.](https://packaging.python.org/en/latest/tutorials/pack
 2. Run 'python scripts/tag.py'
 3. `python -m build`
 4. `twine upload -r pypi -u __token__ -p <password-or-token> ./dist/*`
-
-

{accli-0.5 → accli-0.5.2}/accli.egg-info/SOURCES.txt

@@ -9,6 +9,7 @@ accli/__init__.py
 accli/__main__.py
 accli/_version.py
 accli/cli.py
+accli/common.py
 accli/token.py
 accli.egg-info/PKG-INFO
 accli.egg-info/SOURCES.txt