scientiflow-cli 0.1.0__tar.gz

scientiflow_cli-0.1.0/LICENSE.md

@@ -0,0 +1,7 @@
+ScientiFlow CLI Proprietary License
+
+This software is the proprietary property of ScientiFlow. You may not copy, modify, or distribute this software or any part of it without express written permission from ScientiFlow.
+
+Usage of the software is governed by this license and subject to the terms of any commercial agreements with ScientiFlow.
+
+For commercial licensing requests, contact ScientiFlow at scientiflow@gmail.com.

scientiflow_cli-0.1.0/PKG-INFO

@@ -0,0 +1,62 @@
+Metadata-Version: 2.1
+Name: scientiflow-cli
+Version: 0.1.0
+Summary: CLI tool for scientiflow. This application runs on the client side, decodes pipelines, and executes them in the configured order!
+License: Proprietary
+Author: ScientiFlow
+Author-email: scientiflow@gmail.com
+Requires-Python: >=3.12,<4.0
+Classifier: License :: Other/Proprietary License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: cryptography (>=43.0.1,<44.0.0)
+Requires-Dist: pwinput (>=1.0.3,<2.0.0)
+Requires-Dist: requests (>=2.32.3,<3.0.0)
+Description-Content-Type: text/markdown
+
+# Scientiflow
+
+## Setting the ENV variables
+
+```
+# When developing on Windows, run:
+$env:API_BASE="SCIENTIFLOW_BACKEND_URL"     # Set the base URL of the Scientiflow backend
+$env:AUTH_TOKEN="SOME_AUTH_TOKEN"           # Set the AUTH token that you get after logging in
+$env:SCFLOW_DEBUG=1                         # Run in debug mode, using some dummy data
+
+# on Linux, run:
+export API_BASE="SCIENTIFLOW_BACKEND_URL"   # Set the base URL of the Scientiflow backend
+export AUTH_TOKEN="SOME_AUTH_TOKEN"         # Set the AUTH token that you get after logging in
+export SCFLOW_DEBUG=1                       # Run in debug mode, using some dummy data
+```
+
+## Building and installing the python package
+
+```bash
+poetry build
+pip install dist/scientiflow_cli-0.1.0-py3-none-any.whl --force-reinstall
+
+# Now you can run it as:
+python -m scientiflow_cli --help
+```
+
+> Note: The `--force-reinstall` flag is used as a sage-guard, in case an already exising version of the package is installed.
+
+## For the devs
+
+> Note
+> If you need to add a dependency, which this project depends on, use the command `poetry add` instead of doing a pip install. This will ensure that the `pyproject.toml` file is updated with the new dependency, and all the other devs have the same dependencies and dependencies versions.
+
+```bash
+# Activate a poetry shell, and install dependencies
+poetry shell
+
+# Install the dependencies if it's your first time working on the project, using:
+# poetry install
+```
+
+```bash
+# In order to run the package without building, you can do:
+poetry run python -m scientiflow_cli.main --help
+```
+

scientiflow_cli-0.1.0/README.md

@@ -0,0 +1,45 @@
+# Scientiflow
+
+## Setting the ENV variables
+
+```
+# When developing on Windows, run:
+$env:API_BASE="SCIENTIFLOW_BACKEND_URL"     # Set the base URL of the Scientiflow backend
+$env:AUTH_TOKEN="SOME_AUTH_TOKEN"           # Set the AUTH token that you get after logging in
+$env:SCFLOW_DEBUG=1                         # Run in debug mode, using some dummy data
+
+# on Linux, run:
+export API_BASE="SCIENTIFLOW_BACKEND_URL"   # Set the base URL of the Scientiflow backend
+export AUTH_TOKEN="SOME_AUTH_TOKEN"         # Set the AUTH token that you get after logging in
+export SCFLOW_DEBUG=1                       # Run in debug mode, using some dummy data
+```
+
+## Building and installing the python package
+
+```bash
+poetry build
+pip install dist/scientiflow_cli-0.1.0-py3-none-any.whl --force-reinstall
+
+# Now you can run it as:
+python -m scientiflow_cli --help
+```
+
+> Note: The `--force-reinstall` flag is used as a sage-guard, in case an already exising version of the package is installed.
+
+## For the devs
+
+> Note
+> If you need to add a dependency, which this project depends on, use the command `poetry add` instead of doing a pip install. This will ensure that the `pyproject.toml` file is updated with the new dependency, and all the other devs have the same dependencies and dependencies versions.
+
+```bash
+# Activate a poetry shell, and install dependencies
+poetry shell
+
+# Install the dependencies if it's your first time working on the project, using:
+# poetry install
+```
+
+```bash
+# In order to run the package without building, you can do:
+poetry run python -m scientiflow_cli.main --help
+```

scientiflow_cli-0.1.0/pyproject.toml

@@ -0,0 +1,30 @@
+[tool.poetry]
+name = "scientiflow-cli"
+version = "0.1.0"
+description = "CLI tool for scientiflow. This application runs on the client side, decodes pipelines, and executes them in the configured order!"
+authors = ["ScientiFlow <scientiflow@gmail.com>"]
+license = "Proprietary"
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.12"
+requests = "^2.32.3"
+cryptography = "^43.0.1"
+pwinput = "^1.0.3"
+
+[tool.poetry.scripts]
+scientiflow-cli = "scientiflow_cli.main:main"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+# Optional additions
+# Keywords and classifiers go directly in the main section
+keywords = ["cli", "scientific workflows", "automation", "pipeline"]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: Other/Proprietary License",
+    "Development Status :: 4 - Beta",
+    "Operating System :: OS Independent"
+]

scientiflow_cli-0.1.0/scientiflow_cli/__main__.py

@@ -0,0 +1,4 @@
+from scientiflow_cli.main import main
+
+
+main()

scientiflow_cli-0.1.0/scientiflow_cli/cli/auth_utils.py

@@ -0,0 +1,50 @@
+import os
+from cryptography.fernet import Fernet
+
+TOKEN_FILE_PATH = os.path.expanduser("~/.scientiflow/token")
+KEY_FILE_PATH = os.path.expanduser("~/.scientiflow/key")
+
+def setAuthToken(auth_token):
+    try:
+        directory = os.path.dirname(TOKEN_FILE_PATH)
+        if not os.path.exists(directory):
+            os.makedirs(directory, exist_ok=True)
+
+        if not os.path.exists(KEY_FILE_PATH):
+            key = Fernet.generate_key()
+            with open(KEY_FILE_PATH, "wb") as key_file:
+                key_file.write(key)
+            os.chmod(KEY_FILE_PATH, 0o600)
+        else:
+            with open(KEY_FILE_PATH, "rb") as key_file:
+                key = key_file.read()
+
+        fernet = Fernet(key)
+        encrypted_token = fernet.encrypt(auth_token.encode())
+
+        with open(TOKEN_FILE_PATH, "wb") as token_file:
+            token_file.write(encrypted_token)
+        os.chmod(TOKEN_FILE_PATH, 0o600)
+
+        print("Token encrypted and saved securely.")
+    except Exception as e:
+        print(f"Error setting auth token: {e}")
+
+def getAuthToken():
+    if os.path.exists(TOKEN_FILE_PATH) and os.path.exists(KEY_FILE_PATH):
+        try:
+            with open(KEY_FILE_PATH, 'rb') as key_file:
+                encryption_key = key_file.read()
+            
+            fernet = Fernet(encryption_key)
+            
+            with open(TOKEN_FILE_PATH, 'rb') as token_file:
+                encrypted_token = token_file.read()
+            
+            token = fernet.decrypt(encrypted_token).decode()
+            return token
+        except Exception as e:
+            print(f"Failed to decrypt token: {e}")
+            return None
+    else:
+        return None

scientiflow_cli-0.1.0/scientiflow_cli/cli/login.py

@@ -0,0 +1,38 @@
+import requests
+import pwinput
+
+from scientiflow_cli.cli.auth_utils import setAuthToken
+import re
+
+def login_user():
+    pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$'
+    email = input("Enter your email: ")
+    if re.match(pattern, email):
+       is_valid = True
+    else:
+       is_valid = False
+    if is_valid:
+      password = pwinput.pwinput("Enter your password: ")
+      payload = {
+        "email": email,
+        "password": password,
+        "device_name": "Google-Windows",
+        "remember": True
+      }
+    else:
+      print(f"'{email}' is not a valid email.")
+      return
+    app_base_url = "https://www.backend.scientiflow.com/api"
+    login_url = f"{app_base_url}/auth/login"
+    response = requests.post(login_url, json=payload)
+    if response.status_code == 200:
+        print("Login successful!")
+        auth_token = response.json().get("token")
+        if auth_token:
+            setAuthToken(auth_token)
+        else:
+            print("No token received from the server.")
+    else:
+        print("Login failed!")
+        print(f"Status code: {response.status_code}")
+        print(f"Response: {response.text}")

scientiflow_cli-0.1.0/scientiflow_cli/cli/logout.py

@@ -0,0 +1,26 @@
+import os
+import requests
+from scientiflow_cli.cli.auth_utils import getAuthToken
+
+def logout_user():
+    app_base_url = "https://www.backend.scientiflow.com/api"
+    logout_url = f"{app_base_url}/auth/logout"
+    token = getAuthToken()
+    if token:
+        headers = {
+            'Authorization': f'Bearer {token}'
+        }
+        response = requests.post(logout_url, headers=headers)
+        if response.status_code == 200:
+            print("Logout successful!")
+            token_file_path = os.path.expanduser("~/.scientiflow/token")
+            key_file_path = os.path.expanduser("~/.scientiflow/key")
+            os.remove(token_file_path)
+            os.remove(key_file_path)
+            print("Token and key files deleted.")
+        else:
+            print("Logout failed!")
+            print(f"Status code: {response.status_code}")
+            print(f"Response: {response.text}")
+    else:
+        print("Token file not found or decryption failed. Please login to continue.")

scientiflow_cli-0.1.0/scientiflow_cli/cli/status.py

@@ -0,0 +1,26 @@
+import pathlib
+
+
+
+def user_is_authenticated() -> bool:
+    """Check if user is authenticated."""
+    home = pathlib.Path.home()
+    token_file = home / ".scientiflow" / "key"
+
+    if token_file.exists():
+        jwt_token = token_file.read_text()
+
+        #TODO: Implement token verification using backend
+        if (jwt_token == "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VybmFtZSI6ImFkbWluIiwibG9naW5FeHBpcmVzIjoxNzMwMDc4NDU0fQ.Ppa9RsbMfMo7q0ZfhD9To_vArAkmESaDJtUVRmzm920"):
+
+            return True
+
+    return False
+
+
+
+def get_auth_token() -> str:
+    home = pathlib.Path.home()
+    token_file = home / ".scientiflow" / "key"
+
+    return token_file.read_text()

scientiflow_cli-0.1.0/scientiflow_cli/main.py

@@ -0,0 +1,66 @@
+import os
+import argparse
+from scientiflow_cli.cli.login import login_user#, user_has_auth_token, get_auth_token
+from scientiflow_cli.cli.login import login_user
+from scientiflow_cli.cli.logout import logout_user
+from scientiflow_cli.pipeline.get_jobs import get_jobs
+from scientiflow_cli.utils.file_manager import create_job_dirs, get_job_files
+from scientiflow_cli.pipeline.container_manager import get_job_containers
+from scientiflow_cli.utils.mock import mock_jobs
+from scientiflow_cli.services.executor import execute_jobs
+from scientiflow_cli.cli.auth_utils import getAuthToken
+
+
+AUTH_TOKEN = os.getenv("AUTH_TOKEN", None)
+
+def main():
+    parser = argparse.ArgumentParser(description="Scientiflow Agent CLI")
+
+    parser.add_argument('--login', action='store_true', help="Login using your scientiflow credentials")
+    parser.add_argument('--logout', action='store_true', help="Logout from scientiflow")
+    # parser.add_argument('--list-jobs', action='store_true', help="Get jobs to execute")
+    parser.add_argument('--get-files', action='store_true', help="Get user files")
+    parser.add_argument('--get-containers', action='store_true', help="Download containers for the user")
+    parser.add_argument('--mock-pipeline-decode', action='store_true', help="Decode the mock pipeline")
+    parser.add_argument('--execute-jobs', action='store_true', help="Fetch and execute pending jobs")
+    args = parser.parse_args()
+
+
+    try:
+        # We're checking here that if command is either login or logout, then we don't need to check for auth token
+        # since these two are the only two commands allowed to run without an auth token
+        # if not args.login and not args.logout:
+        #     print("Session not found. Please login to continue")
+        #     return
+        if args.login:
+            login_user()
+        
+        elif args.logout:
+            logout_user()
+        # elif args.list_jobs:
+        #     get_jobs(auth_token = getAuthToken())
+
+        elif args.get_files:
+            get_job_files(auth_token = AUTH_TOKEN)
+        elif args.get_containers:
+            get_job_containers(auth_token = AUTH_TOKEN)
+
+        # This is a mock pipeline decode. This is used to test the functions that are used to decode the pipeline
+        # elif args.mock_pipeline_decode:
+        #     for job in mock_jobs:
+        #         create_job_dirs(job)
+        #         get_job_files(auth_token = AUTH_TOKEN, job = job)
+        #         get_job_containers(auth_token = AUTH_TOKEN, job = job)
+
+        elif args.execute_jobs:
+            execute_jobs(auth_token=getAuthToken())
+        else:
+            print("No arguments specified. Use --help to see available options")
+    
+    except Exception as e:
+        print("Error: ", e)
+        return
+
+
+if __name__ == "__main__":
+    main()

scientiflow_cli-0.1.0/scientiflow_cli/pipeline/container_manager.py

@@ -0,0 +1,62 @@
+import os
+import requests
+import subprocess
+from pathlib import Path
+
+API_BASE = "https://www.backend.scientiflow.com/api"
+
+def get_job_containers(auth_token: str,  job: dict) -> None:
+    base_dir = Path(job['server']['base_directory'])
+    containers_dir = base_dir / "containers"
+    project_job_id = job['project_job']['id']
+    if not containers_dir.exists():
+        containers_dir.mkdir()
+
+    # Get names of containers already available in the user's machine
+    avail_containers: set[str] = {item.name for item in containers_dir.iterdir() if item.is_file()}
+
+    headers = {"Authorization": f"Bearer {auth_token}"}
+    params = {"project_job_id": project_job_id}
+    response = requests.get(f"{API_BASE}/agent-application/get-user-containers", headers=headers, params=params)
+    try:
+        if response.status_code == 200:
+            container_info = response.json()
+
+            if not container_info:
+                print("[X] No containers found for current User / Project")
+                return
+            
+            current_pipeline_containers = set(container_info["current_pipeline_containers"])
+            user_all_containers = set(container_info["user_all_unique_containers"])
+
+            # Remove containers that are not needed but are present on the current machine
+            containers_to_remove = avail_containers - user_all_containers
+            for container_name in containers_to_remove:
+                container_path = base_dir / "containers" / container_name
+                container_path.unlink() # Unlinking means deleting
+                
+            # Download containers which are not present on the user's machine
+            containers_to_download = current_pipeline_containers - avail_containers
+
+            for container_name in container_info['container_image_details']:
+                if container_name['image_name'] in containers_to_download:
+                    command = f"singularity pull {container_name['image_name']}.sif {container_name['sylabs_uri']}"
+                    command = command.split()
+                    print(f"[+] Downloading container {container_name['image_name']}... ", end="")
+                    subprocess.run(command, check=True, cwd=containers_dir)
+                    print("Done")
+        
+        else:
+            print(f"Error fetching container info! Error code: {response.status_code}")
+                
+    except subprocess.CalledProcessError:
+        print("[x] Error executing singularity commands. Try checking your singularity installation")
+        return
+
+    except requests.exceptions.JSONDecodeError:
+        print("[X] No containers found for current User / Project")
+        return
+    
+    except Exception as e:
+        print(f"Error: {e}")    
+    

scientiflow_cli-0.1.0/scientiflow_cli/pipeline/decode_and_execute.py

@@ -0,0 +1,111 @@
+import subprocess
+import re
+from typing import Dict, List, Any
+from scientiflow_cli.services.status_updater import update_job_status
+
+class PipelineExecutor:
+    def __init__(self, base_dir: str, project_job_id: int, project_title: str, job_dir_name: str, nodes: List[Dict[str, Any]], edges: List[Dict[str, str]], environment_variables: Dict[str, str]):
+        self.base_dir = base_dir
+        self.project_job_id = project_job_id
+        self.project_title = project_title
+        self.job_dir_name = job_dir_name
+        self.nodes = nodes
+        self.edges = edges
+        self.environment_variables = environment_variables
+
+        # Create a mapping of nodes
+        self.nodes_map = {node['id']: node for node in nodes}
+
+        # Create adjacency list based on edges
+        self.adj_list = {node['id']: [] for node in nodes}
+        for edge in edges:
+            self.adj_list[edge['source']].append(edge['target'])
+
+        # Find the root (the node with no incoming edges)
+        all_nodes = set(self.nodes_map.keys())
+        target_nodes = {edge['target'] for edge in edges}
+        self.root_nodes = all_nodes - target_nodes
+
+    def replace_variables(self, command: str) -> str:
+        # Function to replace each match with the corresponding value from the dictionary
+        def replacer(match):
+            variable_name = match.group(1)  # Extract the variable name (without ${})
+            return self.environment_variables.get(variable_name, f"${{{variable_name}}}")
+
+        pattern = r'\$\{(\w+)\}'  # This regex pattern finds placeholders like ${<variable>}
+        result = re.sub(pattern, replacer, command)
+        return result
+
+    def execute_command(self, command: str) -> Dict[str, str]:
+        res = {"status": None, "result": ""}
+        try:
+            res["result"] = subprocess.run(command, capture_output=True, text=True, check=True, shell=True).stdout
+            res["status"] = "SUCCESS"
+        except FileNotFoundError as fnf_error:
+            res["result"] = f"File not found: {fnf_error}"
+            res["status"] = "FAILURE"
+            update_job_status(self.project_job_id, "FAILURE")
+            raise SystemExit("Terminating the program.")
+        except subprocess.CalledProcessError as cpe:
+            res["status"] = "FAILURE"
+            res["result"] = f"Script failed with error: {cpe.stderr}"
+            update_job_status(self.project_job_id, "FAILURE")
+            raise SystemExit("Terminating the program.")
+        except Exception as e:
+            res["status"] = "FAILURE"
+            res["result"] = f"Error: {e}"
+            update_job_status(self.project_job_id, "FAILURE")
+            raise SystemExit("Terminating the program.")
+        return res
+
+    def dfs(self, node: str):
+        current_node = self.nodes_map[node]
+        
+        if current_node['type'] == "splitterParent":
+            collector = None
+            for splitter_child in self.adj_list[node]:
+                if self.nodes_map[splitter_child]['data']['active'] == True:
+                    collector = self.dfs(splitter_child)
+            else:
+                if collector and self.adj_list[collector]:
+                    return self.dfs(self.adj_list[collector][0])
+                else:
+                    return
+
+        elif current_node['type'] == "splitter-child":
+            if current_node['data']['active'] == True:
+                if self.adj_list[node]:
+                    return self.dfs(self.adj_list[node][0])
+                else:
+                    return
+
+        elif current_node['type'] == "terminal":
+            commands = current_node['data']['commands']
+            for command in commands:
+                cmd = self.replace_variables(command.get('command'))
+                if cmd:
+                    res = self.execute_command(f"cd {self.base_dir}/{self.project_title}/{self.job_dir_name} && singularity exec {self.base_dir}/containers/{current_node['data']['software']}.sif {cmd}")
+                    print(cmd, ":", res["status"], res["result"])
+            if self.adj_list[node]:
+                return self.dfs(self.adj_list[node][0])
+            else:
+                return
+
+        elif current_node['type'] == "collector":
+            if self.adj_list[node]:
+                return node
+            else:
+                return
+
+    def decode_and_execute_pipeline(self):
+        update_job_status(self.project_job_id, "RUNNING")
+        if self.root_nodes:
+            # Start DFS from the first root node
+            root_node = next(iter(self.root_nodes))
+            self.dfs(root_node)
+        update_job_status(self.project_job_id, "SUCCESS")
+
+# External function to initiate the pipeline execution
+def decode_and_execute_pipeline(base_dir: str, project_job_id: int, project_title: str, job_dir_name: str, nodes: List[Dict[str, Any]], edges: List[Dict[str, str]], environment_variables: Dict[str, str]):
+    executor = PipelineExecutor(base_dir, project_job_id, project_title, job_dir_name, nodes, edges, environment_variables)
+    executor.decode_and_execute_pipeline()

scientiflow_cli-0.1.0/scientiflow_cli/pipeline/get_jobs.py

@@ -0,0 +1,34 @@
+import requests
+from pathlib import Path
+
+
+API_BASE = "https://www.backend.scientiflow.com/api"
+
+
+def get_jobs(auth_token: str) -> list[dict]:
+    headers = { "Authorization": f"Bearer {auth_token}"}
+    response = requests.get(f"{API_BASE}/agent-application/check-jobs-to-execute", headers=headers)
+    
+    if not response.status_code == 200:
+        print("Error fetching jobs - Invalid response")
+        return []
+    
+    try:
+        jobs = response.json()
+        # breakpoint()
+        if len(jobs) == 0:
+            print("No jobs to execute")
+            return []
+        else:
+            print("\n{:<8} {:<20} {:<20}".format("Job ID", "Project Title", "Job Title"))
+            print("======   =============        =========")
+            for index, job in enumerate(response.json(), start=1):
+                project_title = job['project']['project_title']
+                job_title = job['project_job']['job_title']
+                print("{:<8} {:<20} {:<20}".format(index, project_title, job_title))
+            print("\n")
+            return jobs
+
+    except requests.exceptions.JSONDecodeError:
+        print("Error fetching jobs - Invalid JSON")
+        return []