accli 0.3__tar.gz → 0.4__tar.gz

{accli-0.3/accli.egg-info → accli-0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: accli
-Version: 0.3
+Version: 0.4
 Summary: IIASA Accelerator Client
 Author-email: Wrufesh S <wrufesh@gmail.com>
 License: The MIT License (MIT)
@@ -81,6 +81,7 @@ Please follow [this link.](https://packaging.python.org/en/latest/tutorials/pack
 **Release process**
 1. Commit with right version on accli/_version.py
 2. Run 'python scripts/tag.py'
-3. python -m build
+3. `python -m build`
+4. `twine upload -r pypi -u __token__ -p <password-or-token> ./dist/*`
 
 

{accli-0.3 → accli-0.4}/README.md

@@ -46,6 +46,7 @@ Please follow [this link.](https://packaging.python.org/en/latest/tutorials/pack
 **Release process**
 1. Commit with right version on accli/_version.py
 2. Run 'python scripts/tag.py'
-3. python -m build
+3. `python -m build`
+4. `twine upload -r pypi -u __token__ -p <password-or-token> ./dist/*`
 
 

{accli-0.3 → accli-0.4}/accli/AcceleratorJobProjectService.py

@@ -1,7 +1,11 @@
+import os
+import io
+import typing
 import requests
 import json
 import base64
 import urllib3
+from pathlib import Path
 from typing import List, Tuple
 
 class AccAPIError(Exception):
@@ -16,7 +20,7 @@ class AcceleratorJobProjectService:
     def __init__(
             self,
             user_token, 
-            job_cli_base_url='http://accelerator-api/v1/ajob-cli',
+            server_url='http://accelerator-api.iiasa.ac.at',
             verify_cert=True
         ):
         
@@ -29,7 +33,7 @@ class AcceleratorJobProjectService:
         else:
             self.http_client = urllib3.poolmanager.PoolManager(cert_reqs="CERT_NONE", num_pools=1, retries=retries)
 
-        self.cli_base_url = job_cli_base_url
+        self.cli_base_url = f"{server_url}/v1/ajob-cli"
         self.common_request_headers = {
             'x-authorization': user_token
         }
@@ -92,6 +96,36 @@ class AcceleratorJobProjectService:
         if url:
             resp = self.http_client_request("GET", url, preload_content=False)
             return resp
+        
+    def add_log_file(self, data: bytes, filename):
+        res = self.http_client_request(
+            "GET",
+            f"{self.cli_base_url}/presigned-upload-url/?filename={filename}",
+            headers=self.common_request_headers
+        ).json()
+
+        upload_url = res['upload_url']
+        app_bucket_id = res['app_bucket_id']
+        res_filename = res['filename']
+
+        requests.put(
+            upload_url,
+            data=data,
+            # headers=headers,
+            verify=False,
+        )
+
+        self.http_client_request(
+            "POST",
+            f"{self.cli_base_url}/register-log-file/",
+            json=dict(
+                filename=res_filename,
+                app_bucket_id=app_bucket_id
+            ),
+            headers=self.common_request_headers
+        )
+
+        
 
     def get_multipart_put_create_signed_url(
         self,
@@ -475,4 +509,47 @@ class AcceleratorJobProjectService:
                 )
             ),
             headers=headers
-        )
+        )
+
+class Fs:
+    @staticmethod
+    def write_stream_remote(filestream, dest_filepath, user_token, server_url):
+
+        accelerator_job_service = AcceleratorJobProjectService(
+            user_token, 
+            server_url=server_url,
+            verify_cert=True
+        )
+
+        accelerator_job_service.add_filestream_as_job_output(
+            dest_filepath,
+            filestream
+        )
+
+    @staticmethod
+    def write_stream_local(filestream, dest_filepath):
+        with open(dest_filepath, 'wb') as file:
+            file.write(filestream.getvalue())
+
+    @staticmethod
+    def write_file(source: typing.Union[str, io.BytesIO], dest_filepath):
+        user_token = os.environ.get("ACC_JOB_JOB_TOKEN", None)
+        server_url = os.environ.get("ACC_JOB_GATEWAY_SERVER", None)
+
+        if isinstance(source, io.BytesIO):
+            if (user_token and server_url):
+                Fs.write_stream_remote(source, dest_filepath, user_token, server_url)
+            else:
+                Fs.write_stream_local(source, dest_filepath)
+        else:
+            if type(source) != str:
+                raise ValueError("Unknown source argument")
+            source_path = Path(source)
+            if not source_path.exists():
+                raise ValueError("Source path does not exist")
+
+            with open(source, 'wb') as fstream:
+                if (user_token and server_url):
+                    Fs.write_stream_remote(fstream, dest_filepath, user_token, server_url)
+                else:
+                    Fs.write_stream_local(fstream, dest_filepath)

accli-0.4/accli/AcceleratorTaskDispatcher.py

@@ -0,0 +1,218 @@
+import git
+import os
+import hashlib
+import shutil
+import tempfile
+from typing import Optional, List, Dict
+from pydantic import BaseModel, root_validator
+
+from accli.token import get_github_app_token
+
+FOLDER_JOB_REPO_URL = 'https://github.com/IIASA-Accelerator/wkube-job.git'
+
+def hash_folder(folder_path):
+    md5_hash = hashlib.md5()
+    sha256_hash = hashlib.sha256()
+
+    for root, _, files in os.walk(folder_path):
+        for file_name in files:
+            file_path = os.path.join(root, file_name)
+            try:
+                with open(file_path, 'rb') as f:
+                    # Read and update MD5 and SHA256 hash object with file content
+                    for chunk in iter(lambda: f.read(4096), b""):
+                        md5_hash.update(chunk)
+                        sha256_hash.update(chunk)
+            except IOError:
+                # Handle read error (if any)
+                print(f"Error reading file: {file_path}")
+
+    # Get hexadecimal digest of hashes
+    md5_digest = md5_hash.hexdigest()
+    # sha256_digest = sha256_hash.hexdigest()
+
+    return md5_digest
+
+def copy_tree(src, dst):
+    """Recursively copy from src to dst, excluding .git folders."""
+    for item in os.listdir(src):
+        src_path = os.path.join(src, item)
+        dst_path = os.path.join(dst, item)
+        
+        if os.path.isdir(src_path):
+            if item == '.git':
+                continue
+            os.makedirs(dst_path, exist_ok=True)
+            copy_tree(src_path, dst_path)
+        else:
+            shutil.copy2(src_path, dst_path)
+
+def push_folder_job(dir):
+
+    repo_dir = tempfile.mkdtemp()
+
+    copy_tree(dir, repo_dir)
+    
+    branch_name = hash_folder(repo_dir)
+    
+    repo = git.Repo.init(repo_dir)
+
+    git_token = get_github_app_token()
+
+    remote_url = f'https://x-access-token:{git_token}@{FOLDER_JOB_REPO_URL.split("https://")[1]}'
+    remote = repo.create_remote("accelerator_job_repo", remote_url)
+
+    remote.pull('master')
+
+    try:
+        repo.git.checkout(branch_name)
+    except git.exc.GitCommandError:
+        repo.git.checkout('-b', branch_name)
+
+    repo.git.add('.')
+
+    repo.index.commit(branch_name)
+
+    # TODO see what happens with the same checksum
+    remote.push(branch_name)
+    shutil.rmtree(repo_dir)
+
+    return FOLDER_JOB_REPO_URL, branch_name
+
+
+class JobDispatchModel(BaseModel):
+    is_holder_job: bool = True  
+    
+    execute_cluster: str
+    job_location: str
+    job_args: List
+    job_kwargs: Dict
+
+    required_cores: Optional[float]
+    required_ram: Optional[float]
+    required_storage_local: Optional[float]  
+    
+    # is ignored if it is a callback and child of non free node jobs
+    required_storage_workflow: Optional[float]
+
+    job_secrets: Optional[dict] = {}     
+    
+    timeout: Optional[int]
+    pvc_id: Optional[str]
+    node_id: Optional[str]
+
+    ignore_duplicate_job: bool = False
+    free_node: bool = False             # Only applies to immediate children jobs
+
+    
+    
+    children: List['JobDispatchModel'] = []
+    callback: Optional['JobDispatchModel']
+
+
+class WKubeTaskMeta(BaseModel):
+    required_cores: float
+    required_ram: float
+    required_storage_local: float  
+    
+    # is ignored if it is a callback and child of non free node jobs
+    required_storage_workflow: float     
+
+    job_secrets: Optional[dict] = {}  
+    
+    timeout:int
+
+class WKubeTaskKwargs(BaseModel):
+    docker_image: Optional[str]
+
+    job_folder: Optional[str]
+    
+    repo_url: Optional[str]                # required when docker image is not present
+    repo_branch: Optional[str]             # required when docker image is not present
+    
+    docker_filename: Optional[str]         # when not docker image;
+    base_stack: Optional[str]              # when not github docker file #TODO add enum class of available base stack
+    
+    force_build: bool = False
+
+    command: str                           # may not be present with docker_image # TODO wrap a command in custom script to implement timeout or possibly log ingestion if required.  
+
+    conf: Dict[str,str] = {}
+
+    def dict(self, *args, **kwargs):
+        result = super().dict(*args, **kwargs)
+        if 'job_folder' in result:
+            del result['job_folder']
+        return result
+
+    @root_validator(pre=True)
+    def validate_root(cls, values):
+        if not values.get('docker_image'):
+
+            job_folder = values.get('job_folder')
+
+            if not job_folder: 
+            
+                if not values.get('repo_url'): 
+                    # TODO if has no repo url just set it
+                    raise ValueError("repo_url is required")
+                
+                if not values.get('repo_branch'):
+                    raise ValueError('repo_branch is required')
+            else: 
+                remote_url, branch_name = push_folder_job(job_folder)
+                values['repo_url'] = remote_url
+                values['repo_branch'] = branch_name
+
+            if not values.get('docker_filename'):
+                if not values.get("base_stack"):
+                    raise ValueError("base_stack is required when dockerfile is not defined")
+        return values
+    
+class WKubeTaskPydantic(WKubeTaskMeta, WKubeTaskKwargs):
+   pass
+
+
+class GenericTask:
+    def __init__(self, *args, **kwargs):
+        self.dispatch_model_task: JobDispatchModel
+    
+    def add_child(self, task):
+        if self.__class__ != task.__class__:
+            raise ValueError(f"task should of {self.__class__} class")
+        self.dispatch_model_task.children.append(task.dispatch_model_task)
+
+    def add_callback(self, task):
+        
+        if self.__class__ != task.__class__:
+            raise ValueError(f"task should of {self.__class__} class")
+        self.dispatch_model_task.callback = task.dispatch_model_task
+    
+    @property
+    def description(self):
+        return self.dispatch_model_task.dict()
+
+class WKubeTask(GenericTask):
+    def __init__(self, *t_args, **t_kwargs):
+
+        wkube_task_kwargs = None
+        wkube_task_meta = dict()
+
+        if (t_args or t_kwargs):
+            
+            WKubeTaskPydantic(*t_args, **t_kwargs)
+            wkube_task_kwargs = WKubeTaskKwargs(*t_args, **t_kwargs)
+            wkube_task_meta.update(WKubeTaskMeta(*t_args, **t_kwargs).dict())
+            
+
+        self.dispatch_model_task = JobDispatchModel(
+            is_holder_job=not (t_args or t_kwargs),
+            execute_cluster='WKUBE',
+            job_location='acc_native_jobs.dispatch_wkube_task',
+            job_args=[],
+            job_kwargs= wkube_task_kwargs.dict() if wkube_task_kwargs else dict(),
+            **wkube_task_meta
+        )
+
+
+    

{accli-0.3 → accli-0.4}/accli/AcceleratorTerminalCliProjectService.py

@@ -70,6 +70,30 @@ class AcceleratorTerminalCliProjectService:
 
         return res.json()
     
+    def get_github_app_token(self, project_slug):
+        
+        res = self.http_client_request(
+            "GET", 
+            f"{self.cli_base_url}/{project_slug}/github-app-token/",
+            headers=self.common_request_headers
+        )
+
+        return res.json()
+    
+    def dispatch(self, project_slug, job_description):
+        
+        try:
+            res = self.http_client_request(
+                "POST", 
+                f"{self.cli_base_url}/{project_slug}/jobs/dispatch/",
+                json=job_description,
+                headers=self.common_request_headers
+            )
+        except AccAPIError as err:
+                raise err
+
+        return res.json()['job_id']
+    
     def get_dataset_template_details(self, project_slug, template_slug):
         
         try:

{accli-0.3 → accli-0.4}/accli/__init__.py

@@ -2,7 +2,9 @@ from accli._version import VERSION
 
 from accli.cli import app
 
-from accli.AcceleratorJobProjectService import AcceleratorJobProjectService
+from accli.AcceleratorTaskDispatcher import *
+
+from accli.AcceleratorJobProjectService import AcceleratorJobProjectService, Fs
 
 AjobCliService = AcceleratorJobProjectService
 

{accli-0.3 → accli-0.4}/accli/_version.py

@@ -1,3 +1,3 @@
 # Please strictly put double quote to use this info for git tag 
 # Read developer guide for git tag command with regex
-VERSION = "v0.3" 
+VERSION = "v0.4" 

{accli-0.3 → accli-0.4}/accli/cli.py

@@ -5,83 +5,27 @@ import os
 import typer
 import requests
 import warnings
+import importlib.util
 from rich import print
 from typing_extensions import Annotated
-from tinydb import TinyDB, Query
+
+from accli.token import save_token_details, get_token, get_server_url, set_github_app_token
 
 from accli.CsvRegionalTimeseriesValidator import CsvRegionalTimeseriesValidator
 from ._version import VERSION
 
-
 from accli.AcceleratorTerminalCliProjectService import AcceleratorTerminalCliProjectService
 
 from rich.progress import Progress, SpinnerColumn, TextColumn, ProgressColumn, BarColumn, TaskProgressColumn, TimeElapsedColumn, TimeRemainingColumn
 
 warnings.filterwarnings('ignore')
 
-
 app = typer.Typer(
     add_completion=False, 
     pretty_exceptions_show_locals=False, 
     no_args_is_help=True
 )
 
-def get_db_path():
-
-    home = os.path.expanduser("~")
-    token_directory = f"{home}/.accli"
-
-    if not os.path.exists(token_directory):
-        os.makedirs(token_directory)
-    
-    return f"{token_directory}/data.json"
-
-
-def save_token_details(token, server_url, webcli_url):
-
-    db_path = get_db_path()
-
-    try:
-        os.remove(db_path)
-    except OSError:
-        pass
-
-    db = TinyDB(db_path)
-    db.insert({
-        'token': token,
-        'server_url': server_url,
-        'webcli_url': webcli_url
-    })
-
-def get_token():
-    db_path = get_db_path()
-
-    db = TinyDB(db_path)
-
-    for item in db:
-        token = item.get('token')
-        if token:
-            break
-
-    if not token:
-        print("Token does not exists. Please login.")
-    return token
-
-
-def get_server_url():
-    db_path = get_db_path()
-
-    db = TinyDB(db_path)
-
-    for item in db:
-        server_url = item.get('server_url')
-        if server_url:
-            break
-
-    if not server_url:
-        print("Server url does not exists. Please login.")
-    return server_url
-
 
 def get_size(path):
     size = 0
@@ -158,6 +102,9 @@ def upload(
     folder_name: Annotated[str, typer.Argument(help="Name of the folder to be made in Accelerator project space.")],
     max_workers: Annotated[int, typer.Option(help="Maximum worker pool for multipart upload.")] = os.cpu_count()
 ):
+
+
+    #TODO make user free to put anywere except for reserved folder
     
     if not re.fullmatch(r'[a-zA-Z0-9\-\_]+', folder_name):
         raise ValueError("Folder name is invalid.")
@@ -226,3 +173,50 @@ def validate(
 
     validate()
 
+@app.command()
+def dispatch(
+    project_slug: Annotated[str, typer.Argument(help="Unique Accelerator project slug.")],
+    # experiment_name: Annotated[str, typer.Argument(help="Experiment or simulation name. Please put name in quote it includes space(s).")],
+    workflow_file: Annotated[str, typer.Argument(help="Python workflow filepath.")],
+    root_task_variable: Annotated[str, typer.Argument(help="Root task variable in workflow_file.")],
+    server: Annotated[str, typer.Option(help="Accelerator server url.")] = "https://accelerator-api.iiasa.ac.at",
+):
+    
+    access_token = get_token()
+
+    server_url = get_server_url()
+
+    term_cli_project_service = AcceleratorTerminalCliProjectService(
+        user_token=access_token,
+        server_url=server_url,
+        verify_cert=False
+    )
+
+    github_app_token = term_cli_project_service.get_github_app_token(project_slug)
+    set_github_app_token(github_app_token)
+
+
+    spec = importlib.util.spec_from_file_location("workflow", workflow_file)
+
+    module = importlib.util.module_from_spec(spec)
+
+    spec.loader.exec_module(module)
+
+    job_to_dispatch = getattr(module, root_task_variable)
+
+
+    if not job_to_dispatch:
+        raise ValueError(f"No root task variable found with name {root_task_variable}")
+    
+    print(job_to_dispatch.description)
+
+
+    root_job_id = term_cli_project_service.dispatch(
+        project_slug,
+        job_to_dispatch.description
+    )
+
+    print(f"Dispatched root job #ID: {root_job_id}")
+
+
+

accli-0.4/accli/token.py

@@ -0,0 +1,77 @@
+import os
+from tinydb import TinyDB, Query
+
+def get_db_path():
+
+    home = os.path.expanduser("~")
+    token_directory = f"{home}/.accli"
+
+    if not os.path.exists(token_directory):
+        os.makedirs(token_directory)
+    
+    return f"{token_directory}/data.json"
+
+
+def save_token_details(token, server_url, webcli_url):
+
+    db_path = get_db_path()
+
+    try:
+        os.remove(db_path)
+    except OSError:
+        pass
+
+    db = TinyDB(db_path)
+    db.insert({
+        'token': token,
+        'server_url': server_url,
+        'webcli_url': webcli_url
+    })
+
+def get_token():
+    db_path = get_db_path()
+
+    db = TinyDB(db_path)
+
+    for item in db:
+        token = item.get('token')
+        if token:
+            break
+
+    if not token:
+        print("Token does not exists. Please login.")
+    return token
+
+def get_github_app_token():
+    db_path = get_db_path()
+
+    db = TinyDB(db_path)
+
+    for item in db:
+        token = item.get('github_app_token')
+        if token:
+            break
+
+    if not token:
+        print("Github app token does not exists.")
+    return token
+
+def set_github_app_token(github_app_token):
+    db_path = get_db_path()
+    db = TinyDB(db_path)
+    db.update({'github_app_token': github_app_token}, doc_ids=[1])
+
+
+def get_server_url():
+    db_path = get_db_path()
+
+    db = TinyDB(db_path)
+
+    for item in db:
+        server_url = item.get('server_url')
+        if server_url:
+            break
+
+    if not server_url:
+        print("Server url does not exists. Please login.")
+    return server_url

{accli-0.3 → accli-0.4/accli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: accli
-Version: 0.3
+Version: 0.4
 Summary: IIASA Accelerator Client
 Author-email: Wrufesh S <wrufesh@gmail.com>
 License: The MIT License (MIT)
@@ -81,6 +81,7 @@ Please follow [this link.](https://packaging.python.org/en/latest/tutorials/pack
 **Release process**
 1. Commit with right version on accli/_version.py
 2. Run 'python scripts/tag.py'
-3. python -m build
+3. `python -m build`
+4. `twine upload -r pypi -u __token__ -p <password-or-token> ./dist/*`
 
 

{accli-0.3 → accli-0.4}/accli.egg-info/SOURCES.txt

@@ -5,12 +5,14 @@ README.md
 pyproject.toml
 requirements.txt
 accli/AcceleratorJobProjectService.py
+accli/AcceleratorTaskDispatcher.py
 accli/AcceleratorTerminalCliProjectService.py
 accli/CsvRegionalTimeseriesValidator.py
 accli/__init__.py
 accli/__main__.py
 accli/_version.py
 accli/cli.py
+accli/token.py
 accli.egg-info/PKG-INFO
 accli.egg-info/SOURCES.txt
 accli.egg-info/dependency_links.txt

{accli-0.3 → accli-0.4}/accli.egg-info/requires.txt

@@ -3,3 +3,4 @@ typer[all]
 tinydb
 requests
 jsonschema
+pydantic==1.10.13

{accli-0.3 → accli-0.4}/pyproject.toml

@@ -12,7 +12,7 @@ description = "IIASA Accelerator Client"
 readme = "README.md"
 license = {file = "LICENSE"}
 requires-python = ">=3.7.17"
-dependencies = ['urllib3', 'typer[all]', 'tinydb', 'requests', 'jsonschema']
+dependencies = ['urllib3', 'typer[all]', 'tinydb', 'requests', 'jsonschema', 'pydantic==1.10.13']
 classifiers = [
     "Programming Language :: Python :: 3",
     "License :: OSI Approved :: MIT License",

accli-0.4/requirements.txt

@@ -0,0 +1,8 @@
+urllib3
+requests
+typer[all]
+tinydb
+twine
+jsonschema
+pydantic==1.10.13
+gitpython==3.1.43

accli-0.3/requirements.txt

@@ -1,6 +0,0 @@
-urllib3
-requests
-typer[all]
-tinydb
-twine
-jsonschema