machineconfig 1.92__py3-none-any.whl → 1.94__py3-none-any.whl

machineconfig/cluster/distribute.py

@@ -0,0 +1,280 @@
+"""
+Distributed Computing
+"""
+
+
+from typing import Optional, Any, Callable, Union
+from math import ceil, floor
+from enum import Enum
+from dataclasses import dataclass
+import psutil
+import numpy as np
+
+from crocodile.core import randstr, List as L, Struct as S, install_n_import
+from crocodile.file_management import P, Save
+from crocodile.meta import SSH, Terminal
+from machineconfig.cluster.remote_machine import RemoteMachine, RemoteMachineConfig, WorkloadParams, LAUNCH_METHOD
+from rich.console import Console
+# from platform import system
+# import time
+# from rich.progress import track
+
+
+console = Console()
+
+
+class LoadCriterion(Enum):
+    cpu = "cpu"
+    ram = "ram"
+    product = "cpu * ram"
+    cpu_norm = "cpu_norm"
+    ram_norm = "ram_norm"
+    product_norm = "cpu_norm * ram_norm"
+
+
+@dataclass
+class MachineSpecs:
+    cpu: float
+    ram: float
+    product: float
+    cpu_norm: float
+    ram_norm: float
+    product_norm: float
+    @staticmethod
+    def get_this_machine_specs():
+        cpu, ram = psutil.cpu_count(), psutil.virtual_memory().total / 2 ** 30
+        assert cpu is not None
+        return MachineSpecs(cpu=cpu, ram=ram, product=cpu * ram, cpu_norm=cpu, ram_norm=ram, product_norm=cpu * ram)
+
+
+class ThreadLoadCalculator:
+    """relies on relative values to a referenc machine specs.
+    Runs multiple instances of code per machine. Useful if code doesn't run faster with more resources avaliable.
+    equal distribution across instances of one machine"""
+    def __init__(self, num_jobs: Optional[int] = None, load_criterion: LoadCriterion = LoadCriterion.cpu, reference_specs: Optional[MachineSpecs] = None):
+        self.num_jobs = num_jobs
+        self.load_criterion = load_criterion
+        self.reference_specs: MachineSpecs = MachineSpecs.get_this_machine_specs() if reference_specs is None else reference_specs
+    def __getstate__(self): return self.__dict__
+    def __setstate__(self, state: dict[str, Any]): self.__dict__.update(state)
+    def get_num_threads(self, machine_specs: MachineSpecs) -> int:
+        if self.num_jobs is None: return 1
+        res = int(floor(self.num_jobs * (machine_specs.__dict__[self.load_criterion.name] / self.reference_specs.__dict__[self.load_criterion.name])))
+        return 1 if res == 0 else res
+
+
+class MachineLoadCalculator:
+    def __init__(self, max_num: int = 1000, load_criterion: LoadCriterion = LoadCriterion.product, load_ratios_repr: str = ""):
+        self.load_ratios: list[float] = []
+        self.load_ratios_repr = load_ratios_repr
+        self.max_num: int = max_num
+        self.load_criterion = load_criterion
+    def __getstate__(self) -> dict[str, Any]: return self.__dict__
+    def __setstate__(self, d: dict[str, Any]) -> None: self.__dict__.update(d)
+    def get_workload_params(self, machines_specs: list[MachineSpecs], threads_per_machine: list[int]) -> list[WorkloadParams]:
+        """Note: like thread divider in parallelize function, the behaviour is to include the edge cases on both ends of subsequent intervals."""
+        tmp: list[WorkloadParams] = []
+        idx_so_far = 0
+        for machine_index, (machine_specs, a_threads_per_machine) in enumerate(zip(machines_specs, threads_per_machine)):
+            load_value = machine_specs.__dict__[self.load_criterion.name]
+            self.load_ratios.append(load_value)
+            idx1 = idx_so_far
+            idx2 = self.max_num if machine_index == len(threads_per_machine) - 1 else (floor(load_value * self.max_num) + idx1)
+            if idx2 > self.max_num:
+                print(machines_specs, '\n\n', threads_per_machine)
+                print(f"All values: {tmp=}, {idx_so_far=}, {idx1=}, {idx2=}, {self.max_num=}, {a_threads_per_machine=}, {machine_index=}, {machine_specs=}, {load_value=}, {self.load_ratios=}, {self.load_ratios_repr=}")
+                raise ValueError(f"idx2 ({idx2}) > max_num ({self.max_num})")
+            idx_so_far = idx2
+            tmp.append(WorkloadParams(idx_start=idx1, idx_end=idx2, idx_max=self.max_num, jobs=a_threads_per_machine))
+        return tmp
+
+
+class Cluster:
+    def __getstate__(self) -> dict[str, Any]:
+        state = self.__dict__
+        state["func"] = None
+        return state
+    def __setstate__(self, state: dict[str, Any]) -> None: self.__dict__.update(state)
+    def save(self) -> P:
+        path = self.root_dir.joinpath("cluster.Cluster.pkl")
+        Save.pickle(obj=self.__getstate__(), path=path)
+        return path
+    @staticmethod
+    def load(job_id: str, base: Optional[str] = None) -> 'Cluster': return Cluster.get_cluster_path(job_id=job_id, base=base).joinpath("cluster.Cluster.pkl").readit()
+    @staticmethod
+    def get_cluster_path(job_id: str, base: Union[str, P, None] = None):
+        if base is None: base_obj = P.home().joinpath("tmp_results/remote_machines")
+        else: base_obj = P(base)
+        return base_obj.joinpath(f"job_id__{job_id}")
+    def __init__(self,
+                 func: Callable[..., Any],
+                 ssh_params: list[dict[str, Any]],
+                 remote_machine_config: RemoteMachineConfig,
+                 func_kwargs: Optional[dict[str, Any]] = None,
+                 # workload_params: list[WorkloadParams] or None = None,
+                 thread_load_calc: Optional[ThreadLoadCalculator] = None,
+                 # machine_load_calc=None,
+                 ditch_unavailable_machines: bool = False,
+                 description: str = "",
+                 job_id: Optional[str] = None,
+                 base_dir: Union[str, P, None] = None):
+        self.job_id = job_id or randstr(noun=True)
+        self.root_dir = self.get_cluster_path(self.job_id, base=base_dir)
+        self.results_downloaded = False
+
+        self.thread_load_calc: ThreadLoadCalculator = thread_load_calc or ThreadLoadCalculator()
+        self.machine_load_calc: MachineLoadCalculator = MachineLoadCalculator(load_criterion=LoadCriterion[self.thread_load_calc.load_criterion.name + "_norm"], )
+
+        sshz: list[SSH] = []
+        for an_ssh_params in ssh_params:
+            try:
+                tmp = SSH(**an_ssh_params)
+                sshz.append(tmp)
+            except Exception as ex:
+                print(f"Couldn't connect to {an_ssh_params}")
+                if ditch_unavailable_machines: continue
+                else: raise Exception(f"Couldn't connect to {an_ssh_params}") from ex  # type: ignore # pylint: disable=W0719
+
+        # lists of similar length:
+        self.sshz: list[SSH] = sshz
+        self.machines: list[RemoteMachine] = []
+        self.machines_specs: list[MachineSpecs] = []
+        self.threads_per_machine: list[int] = []
+        self.remote_machine_kwargs: RemoteMachineConfig = remote_machine_config
+        self.workload_params: list[WorkloadParams] = []
+
+        self.description: str = description
+        self.func = func
+        self.func_kwargs = func_kwargs if func_kwargs is not None else {}
+
+        # fire options
+        self.machines_per_tab: int = 1
+        self.window_number: int = 2
+
+    def __repr__(self): return "Cluster with following machines:\n" + "\n".join([repr(item) for item in (self.machines if self.machines else self.sshz)])
+    def print_func_kwargs(self):
+        print("\n" * 2)
+        console.rule(title="kwargs of functions to be run on machines")
+        for an_ssh, a_kwarg in zip(self.sshz, self.workload_params):
+            S(a_kwarg.__dict__).print(as_config=True, title=an_ssh.get_remote_repr())
+    def print_commands(self, launch_method: LAUNCH_METHOD):
+        print("\n" * 2)
+        console.rule(title="Commands to run on each machine:")
+        for machine in self.machines:
+            print(f"{repr(machine)} ==> {machine.file_manager.get_fire_command(launch_method=launch_method)}")
+
+    def generate_standard_kwargs(self) -> None:
+        if self.workload_params:
+            self.print_func_kwargs()
+            print(self.workload_params, len(self.workload_params), type(self.workload_params))
+            print("workload_params is not None, so not generating standard kwargs")
+            return None
+        cpus: list[float] = []
+        rams: list[float] = []
+        for an_ssh in self.sshz:
+            res = an_ssh.run_py("import psutil; print(psutil.cpu_count(), psutil.virtual_memory().total)", verbose=False).op
+            try: cpus.append(int(res.split(' ')[0]))
+            except ValueError as ve:
+                print(f"Couldn't get cpu count from {an_ssh}")
+                raise ValueError(f"Couldn't get cpu count from {an_ssh.get_remote_repr()}") from ve
+            rams.append(ceil(int(res.split(' ')[1]) / 2 ** 30))
+        total_cpu = np.array(cpus).sum()
+        total_ram = np.array(rams).sum()
+        total_product = (np.array(cpus) * np.array(rams)).sum()
+
+        self.machines_specs = [MachineSpecs(cpu=a_cpu, ram=a_ram, product=a_cpu * a_ram, cpu_norm=a_cpu / total_cpu, ram_norm=a_ram / total_ram, product_norm=a_cpu * a_ram / total_product) for a_cpu, a_ram in zip(cpus, rams)]
+        self.threads_per_machine = [self.thread_load_calc.get_num_threads(machine_specs=machine_specs) for machine_specs in self.machines_specs]
+        self.workload_params = self.machine_load_calc.get_workload_params(machines_specs=self.machines_specs, threads_per_machine=self.threads_per_machine)
+        self.print_func_kwargs()
+
+    def viz_load_ratios(self) -> None:
+        if not self.workload_params: raise RuntimeError("func_kwargs_list is None. You need to run generate_standard_kwargs() first.")
+        plt = install_n_import("plotext")
+        names = L(self.sshz).apply(lambda x: x.get_remote_repr(add_machine=True)).list
+
+        plt.simple_multiple_bar(names, [[machine_specs.cpu for machine_specs in self.machines_specs], [machine_specs.ram for machine_specs in self.machines_specs]], title="Resources per machine", labels=["#cpu threads", "memory size"])
+        plt.show()
+        print("")
+        plt.simple_bar(names, self.machine_load_calc.load_ratios, width=100, title=f"Load distribution for machines using criterion `{self.machine_load_calc.load_criterion}`")
+        plt.show()
+
+        tmp = S(dict(zip(names, L((np.array(self.machine_load_calc.load_ratios) * 100).round(1)).apply(lambda x: f"{int(x)}%")))).print(as_config=True, justify=75, return_str=True)
+        assert isinstance(tmp, str)
+        self.machine_load_calc.load_ratios_repr = tmp
+        print(self.machine_load_calc.load_ratios_repr)
+        # self.workload_params.
+        print("\n")
+
+    def submit(self) -> None:
+        if not self.workload_params: raise RuntimeError("You need to generate standard kwargs first.")
+        for idx, (a_workload_params, an_ssh) in enumerate(zip(self.workload_params, self.sshz)):
+            desc = self.description + f"\nLoad Ratios on machines:\n{self.machine_load_calc.load_ratios_repr}"
+            # if self.remote_machine_kwargs is not None:
+            config = self.remote_machine_kwargs
+            config.__dict__.update(dict(description=desc, job_id=self.job_id + f"_{idx}", base_dir=self.root_dir, workload_params=a_workload_params, ssh_obj=an_ssh))
+            # else: config = RemoteMachineConfig(description=desc, job_id=self.job_id + f"_{idx}", base_dir=self.root_dir.as_posix(), workload_params=a_workload_params, ssh_obj=an_ssh)
+            m = RemoteMachine(func=self.func, func_kwargs=self.func_kwargs, config=config)
+            m.generate_scripts()
+            m.submit()
+            self.machines.append(m)
+        try: self.save()
+        except Exception as re:
+            print(re)
+            print("Couldn't pickle cluster object")
+        # self.print_commands()
+
+    def open_mux(self, machines_per_tab: int = 1, window_number: Optional[int] = None):
+        self.machines_per_tab = machines_per_tab
+        self.window_number = window_number if window_number is not None else 0  # randstr(length=3, lower=False, upper=False)
+        cmd = f"wt -w {self.window_number} "
+        for idx, m in enumerate(self.machines):
+
+            sub_cmd = m.get_session_manager().get_new_session_ssh_command(ssh=m.ssh, sess_name=m.job_params.session_name)
+            if idx == 0: cmd += f""" new-tab --title '{str(m.ssh.hostname) + str(idx)}' pwsh -Command "{sub_cmd}" `;"""  # avoid new tabs despite being even index
+            elif idx % self.machines_per_tab == 0: cmd += f""" new-tab --title {str(m.ssh.hostname) + str(idx)} pwsh -Command "{sub_cmd}" `;"""
+            else: cmd += f""" split-pane --horizontal --size {1 / self.machines_per_tab} pwsh -Command "{sub_cmd}" `;"""
+
+        print("Terminal launch command:\n", cmd)
+        if cmd.endswith("`;"): cmd = cmd[:-2]
+        Terminal().run_async(*cmd.replace("`;", ";").split(" "))  # `; only for powershell, cmd is okay for ; as it is not a special character
+        rm_last = self.machines[-1]
+        rm_last.get_session_manager().asssert_session_started(ssh=rm_last.ssh, sess_name=rm_last.job_params.session_name)
+
+    def fire(self, machines_per_tab: int = 1, window_number: Optional[int] = None, run: bool = False):
+        self.open_mux(machines_per_tab=machines_per_tab, window_number=window_number)
+        for m in self.machines:
+            m.fire(run=run, open_console=False)
+
+    def run(self, run: bool = False, machines_per_tab: int = 1, window_number: Optional[int] = None):
+        self.generate_standard_kwargs()
+        self.viz_load_ratios()
+        print(self)
+        self.submit()
+        self.fire(run=run, machines_per_tab=machines_per_tab, window_number=window_number)
+        self.save()
+        return self
+
+    def check_job_status(self) -> None: L(self.machines).apply(lambda machine: machine.check_job_status())
+    def download_results(self):
+        if self.results_downloaded:
+            print(f"All results downloaded to {self.root_dir} 🤗")
+            return True
+        for idx, a_m in enumerate(self.machines):
+            _ = idx
+            if a_m.results_path is None:
+                print(f"Results are not ready for machine {a_m}.")
+                print("Try to run `.check_job_status()` to check if the job is done and obtain results path.")
+                continue
+            # results_folder = P(a_m.results_path).expanduser()
+            if a_m.results_downloaded is False:
+                print("\n")
+                console.rule(f"Downloading results from {a_m}")
+                print("\n")
+                a_m.download_results(target=None)  # TODO another way of resolve multiple machines issue is to create a directory at downlaod_results time.
+        if L(self.machines).results_downloaded.to_numpy().sum() == len(self.machines):
+            print(f"All results downloaded to {self.root_dir} 🤗")
+            self.results_downloaded = True
+
+
+if __name__ == '__main__':
+    pass

machineconfig/cluster/file_manager.py

@@ -0,0 +1,238 @@
+
+from rich import inspect
+from rich.console import Console
+import pandas as pd
+
+from crocodile.core import Struct as S, install_n_import
+from crocodile.file_management import P, Save
+from crocodile.meta import MACHINE
+from machineconfig.cluster.loader_runner import JOB_STATUS, LAUNCH_METHOD, JobStatus
+
+from typing import Union, Any
+import time
+import os
+import platform
+
+console = Console()
+
+
+class FileManager:
+    running_path          = P("~/tmp_results/remote_machines/file_manager/running_jobs.pkl")
+    queue_path            = P("~/tmp_results/remote_machines/file_manager/queued_jobs.pkl")
+    history_path          = P("~/tmp_results/remote_machines/file_manager/history_jobs.pkl")
+    shell_script_path_log = P("~/tmp_results/remote_machines/file_manager/last_cluster_script.txt")
+    default_base          = P("~/tmp_results/remote_machines/jobs")
+    @staticmethod
+    def from_pickle(path: Union[str, P]):
+        fm = FileManager(job_id='1', remote_machine_type='Windows', lock_resources=True, max_simulataneous_jobs=1, base=None)
+        fm.__setstate__(dict(P(path).expanduser().readit()))
+        return fm
+    def __getstate__(self): return self.__dict__
+    def __setstate__(self, state: dict[str, Any]): self.__dict__ = state
+    def __init__(self, job_id: str, remote_machine_type: MACHINE, lock_resources: bool, max_simulataneous_jobs: int = 1, base: Union[str, P, None] = None):
+        """Log files to track execution process:
+        * A text file that cluster deletes at the begining then write to at the end of each job.
+        * pickle of Machine and clusters objects.
+        """
+        # EVERYTHING MUST REMAIN IN RELATIVE PATHS
+        self.remote_machine_type = remote_machine_type
+        self.job_id = job_id
+        self.max_simulataneous_jobs = max_simulataneous_jobs
+        self.lock_resources = lock_resources
+
+        self.submission_time = pd.Timestamp.now()
+
+        self.base_dir = P(base).collapseuser() if bool(base) else FileManager.default_base
+        status: JOB_STATUS
+        status = 'queued'
+        self.job_root = self.base_dir.joinpath(f"{status}/{self.job_id}")
+    @property
+    def py_script_path(self): return self.job_root.joinpath("python/cluster_wrap.py")
+    @property
+    def cloud_download_py_script_path(self): return self.job_root.joinpath("python/download_data.py")
+    @property
+    def shell_script_path(self): return self.job_root.joinpath("shell/cluster_script" + {"Windows": ".ps1", "Linux": ".sh"}[self.remote_machine_type])  # noqa: E501
+    @property
+    def kwargs_path(self): return self.job_root.joinpath("data/func_kwargs.pkl")
+    @property
+    def file_manager_path(self): return self.job_root.joinpath("data/file_manager.pkl")
+    @property
+    def remote_machine_path(self): return self.job_root.joinpath("data/remote_machine.Machine.pkl")
+    @property
+    def remote_machine_config_path(self): return self.job_root.joinpath("data/remote_machine_config.pkl")
+    @property
+    def execution_log_dir(self): return self.job_root.joinpath("logs")
+    def get_fire_command(self, launch_method: LAUNCH_METHOD):
+        _ = launch_method
+        script_path = self.shell_script_path.expanduser()
+        # if launch_method == "remotely": pass  # shell_script is already repared for target machine.
+        # else:
+        if platform.system() == "Windows" and script_path.name.endswith(".sh"):
+            tmp = script_path.with_suffix(".ps1")
+            tmp.write_text(script_path.read_text(), encoding="utf-8", newline=None)
+            script_path = tmp
+        if platform.system() == "Linux" and script_path.name.endswith(".ps1"):
+            tmp = script_path.with_suffix(".sh")
+            tmp.write_text(script_path.read_text(), encoding="utf-8", newline='\n')
+            script_path = tmp
+        return f". {script_path}"
+    def fire_command_to_clip_memory(self, launch_method: LAUNCH_METHOD):
+        print("Execution command copied to clipboard 📋")
+        print(self.get_fire_command(launch_method=launch_method)); install_n_import("clipboard").copy(self.get_fire_command(launch_method=launch_method))
+        print("\n")
+    def get_job_status(self, session_name: str, tab_name: str) -> JOB_STATUS:
+        pid_path = self.execution_log_dir.expanduser().joinpath("pid.txt")
+        tmp = self.execution_log_dir.expanduser().joinpath("status.txt").read_text()
+        status: JOB_STATUS = tmp  # type: ignore
+        if status == "running":
+            if not pid_path.exists():
+                print(f"Something wrong happened to job `{self.job_id}`. Its status log file says `{status}`, but pid_path doesn't exists. Moving to failed.")
+                status = 'failed'
+                self.execution_log_dir.expanduser().joinpath("status.txt").write_text(status)
+                return status
+            pid: int = int(pid_path.read_text().rstrip())
+            import psutil
+            try: proc = psutil.Process(pid=pid)
+            except psutil.NoSuchProcess:
+                print(f"Something wrong happened to job `{self.job_id}`.. Its status log file says `{status}`, but its declared `{pid=}` is dead. Moving to failed.")
+                status = 'failed'
+                self.execution_log_dir.expanduser().joinpath("status.txt").write_text(status)
+                return status
+            command = " ".join(proc.cmdline())
+            if self.job_id not in command:
+                print(f"Something wrong happened to job `{self.job_id}`. Its status log file says `{status}` but the `{pid=}` declared seem to belong to a different process as indicated by the firing command `{command=}`. Moving to failed.")
+                status = 'failed'
+                self.execution_log_dir.expanduser().joinpath("status.txt").write_text(status)
+                return status
+            print(f"Job `{self.job_id}` is running with {pid=} & {session_name=} & {tab_name=}.")
+            return status
+        return status
+
+    def add_to_queue(self, job_status: JobStatus):
+        try:
+            queue_file: list[JobStatus] = self.queue_path.expanduser().readit()
+        except FileNotFoundError:
+            print("Queue file was deleted by the locking job, creating an empty one and saving it.")
+            queue_file = []
+            Save.pickle(obj=queue_file, path=self.queue_path.expanduser())
+        job_ids = [job.job_id for job in queue_file]
+        if self.job_id not in job_ids:
+            print(f"Adding this job {self.job_id} to the queue and saving it. {len(queue_file)=}")
+            queue_file.append(job_status)
+            Save.pickle(obj=queue_file, path=self.queue_path.expanduser())
+        return queue_file
+
+    def get_resources_unlocking(self):  # this one works at shell level in case python script failed.
+        return f"""
+rm {self.running_path.collapseuser().as_posix()}
+echo "Unlocked resources"
+"""
+
+    def secure_resources(self):
+        if self.lock_resources is False: return True
+        this_job = JobStatus(job_id=self.job_id, pid=os.getpid(), submission_time=self.submission_time, start_time=None, status='locked')
+        sleep_time_mins = 10
+        lock_status = 'locked'
+        while lock_status == 'locked':
+
+            try: running_file: list[JobStatus] = self.running_path.expanduser().readit()
+            except FileNotFoundError:
+                print("Running file was deleted by the locking job, making one.")
+                running_file = []
+                Save.pickle(obj=running_file, path=self.running_path.expanduser())
+
+            queue_file = self.add_to_queue(job_status=this_job)
+
+            if len(running_file) < self.max_simulataneous_jobs:
+                lock_status = 'unlocked'
+                break
+
+            # --------------- Clearning up queue_file from dead processes -----------------
+            import psutil
+            next_job_in_queue = queue_file[0]  # only consider the first job in the queue
+            try: _ = psutil.Process(next_job_in_queue.pid)
+            except psutil.NoSuchProcess:
+                print(f"Next job in queue {next_job_in_queue} has no associated process, removing it from the queue.")
+                queue_file.pop(0)
+                Save.pickle(obj=queue_file, path=self.queue_path.expanduser())
+                continue
+
+            # --------------- Clearning up running_file from dead processes -----------------
+            found_dead_process = False
+            assert len(running_file) > 0, "Running file is empty. This should not happen. There should be a break before this point."
+
+            for running_job in running_file:
+                try: proc = psutil.Process(pid=running_job.pid)
+                except psutil.NoSuchProcess:
+                    print(f"Locking process with pid {running_job.pid} is dead. Ignoring this lock file.")
+                    S(running_job.__dict__).print(as_config=True, title="Ignored Lock File Details")
+                    running_file.remove(running_job)
+                    Save.pickle(obj=running_file, path=self.running_path.expanduser())
+                    found_dead_process = True
+                    continue  # for for loop
+                attrs_txt = ['status', 'memory_percent', 'exe', 'num_ctx_switches',
+                             'ppid', 'num_threads', 'pid', 'cpu_percent', 'create_time', 'nice',
+                             'name', 'cpu_affinity', 'cmdline', 'username', 'cwd']
+                # if self.remote_machine_type == 'Windows': attrs_txt += ['num_handles']
+                # environ, memory_maps, 'io_counters'
+                attrs_objs = ['memory_info', 'memory_full_info', 'cpu_times', 'ionice', 'threads', 'open_files', 'connections']
+                inspect(S(proc.as_dict(attrs=attrs_objs)), value=False, title=f"Process holding the Lock (pid = {running_job.pid})", docs=False, sort=False)
+                inspect(S(proc.as_dict(attrs=attrs_txt)), value=False, title=f"Process holding the Lock (pid = {running_job.pid})", docs=False, sort=False)
+
+            if found_dead_process: continue  # repeat while loop logic.
+            running_job = running_file[0]  # arbitrary job in the running file.
+            assert running_job.start_time is not None, f"Running job {running_job} has no start time. This should not happen."
+
+            this_specs = {"Submission time": this_job.submission_time, "Time now": pd.Timestamp.now(),
+                          "Time spent waiting in the queue so far 🛌": pd.Timestamp.now() - this_job.submission_time,
+                          f"Time consumed by locking job so far (job_id = {running_job.job_id}) so far ⏰": pd.Timestamp.now() - running_job.start_time}
+            S(this_specs).print(as_config=True, title=f"This Job `{this_job.job_id}` Details")
+            console.rule(title=f"Resources are locked by another job `{running_job.job_id}`. Sleeping for {sleep_time_mins} minutes. 😴", style="bold red", characters="-")
+            print("\n")
+            time.sleep(sleep_time_mins * 60)
+        self.write_lock_file(job_status=this_job)
+        console.print(f"Resources are locked by this job `{self.job_id}`. Process pid = {os.getpid()}.", highlight=True)
+
+    def write_lock_file(self, job_status: JobStatus):
+        job_status.start_time = pd.Timestamp.now()
+        queue_path = self.queue_path.expanduser()
+        try: queue_file: list[JobStatus] = queue_path.readit()
+        except FileNotFoundError as fne: raise FileNotFoundError(f"Queue file {queue_path} does not exist. This method should not be called in the first place.") from fne
+
+        if job_status in queue_file: queue_file.remove(job_status)
+        print("Removed current job from waiting queue and added it to the running queue. Saving both files.")
+        Save.pickle(obj=queue_file, path=queue_path)
+
+        running_path = self.running_path.expanduser()
+        try: running_file: list[JobStatus] = running_path.readit()
+        except FileNotFoundError as fne: raise FileNotFoundError(f"Queue file {running_path} does not exist. This method should not be called in the first place.") from fne
+
+        assert job_status not in running_file, f"Job status {job_status} is already in the running file. This should not happen."
+        assert len(running_file) < self.max_simulataneous_jobs, f"Number of running jobs ({len(running_file)}) is greater than the maximum allowed ({self.max_simulataneous_jobs}). This method should not be called in the first place."
+        running_file.append(job_status)
+        Save.pickle(obj=running_file, path=running_path)
+
+    def unlock_resources(self):
+        if self.lock_resources is False: return True
+        running_file: list[JobStatus] = self.running_path.expanduser().readit()
+        for job_status in running_file:
+            if job_status.job_id == self.job_id:
+                this_job = job_status
+                break
+        else:
+            print(f"Job {self.job_id} is not in the running file. This should not happen. The file is corrupt.")
+            this_job = None
+        if this_job is not None:
+            running_file.remove(this_job)
+        console.print(f"Resources have been released by this job `{self.job_id}`. Saving new running file")
+        Save.pickle(path=self.running_path.expanduser(), obj=running_file)
+        start_time = pd.to_datetime(self.execution_log_dir.expanduser().joinpath("start_time.txt").readit(), utc=False)
+        end_time = pd.Timestamp.now()
+        item = {"job_id": self.job_id, "start_time": start_time, "end_time": end_time, "submission_time": self.submission_time}
+        hist_file = self.history_path.expanduser()
+        if hist_file.exists(): hist = hist_file.readit()
+        else: hist = []
+        hist.append(item)
+        print(f"Saved history file to {hist_file} with {len(hist)} items.")
+        Save.pickle(obj=hist, path=hist_file)
+        # this is further handled by the calling script in case this function failed.

machineconfig/cluster/job_params.py

@@ -0,0 +1,148 @@
+
+
+
+import git
+from crocodile.file_management import P
+
+from typing import Optional, Callable, Union, Any
+from machineconfig.cluster.remote_machine import WorkloadParams
+from dataclasses import dataclass
+import getpass
+import platform
+
+
+@dataclass
+class JobParams:
+    """What Python script needs to run the job. This will be dynamically injected into the script. Notice that all attrs are either strings or integers."""
+    description: str
+    ssh_repr: str
+    ssh_repr_remote: str
+    error_message: str
+    session_name: str
+    tab_name: str
+    file_manager_path: str
+
+    repo_path_rh: str
+    file_path_rh: str
+    file_path_r: str
+    func_module: str
+    func_class: Optional[str] = None  # the callable might be a function on its own, not a method of a class.
+    func_name: Optional[str] = None  # the job might be running a script as is, no particular method.
+
+    def auto_commit(self):
+        from git.repo import Repo
+        repo = Repo(P(self.repo_path_rh).expanduser(), search_parent_directories=True)
+        # do a commit if the repo is dirty
+        if repo.is_dirty():
+            repo.git.add(update=True)
+            repo.index.commit(f"CloudManager auto commit by {getpass.getuser()}@{platform.node()}")
+            print(f"⚠️ Repo {repo.working_dir} was dirty, auto-committed.")
+        else: print(f"✅ Repo {repo.working_dir} was clean, no auto-commit.")
+
+    def is_installabe(self) -> bool: return True if "setup.py" in P(self.repo_path_rh).expanduser().absolute().listdir().apply(str) else False
+    @staticmethod
+    def from_empty() -> 'JobParams':
+        return JobParams(repo_path_rh="", file_path_rh="", file_path_r="", func_module="", func_class="", func_name="", description="", ssh_repr="", ssh_repr_remote="", error_message="", session_name="", tab_name="", file_manager_path="")
+    @staticmethod
+    def from_func(func: Union[Callable[[Any], Any], P, str]) -> 'JobParams':
+        # if callable(self.func): executed_obj = f"""**{self.func.__name__}** from *{P(self.func.__code__.co_filename).collapseuser().as_posix()}*"""  # for email.
+        if callable(func) and not isinstance(func, P):
+            func_name = func.__name__
+            func_module = func.__module__
+            if func_module == "<run_path>":  # function imported through readpy module.
+                func_module = P(func.__globals__['__file__']).name
+            assert func_module != "__main__", f"Function must be defined in a module, not in __main__. Consider importing `{func.__name__}` or, restart this session and import the contents of this module."
+            if func.__name__ != func.__qualname__:
+                # print(f"Passed function {func} is a method of a class.")
+                func_file, func_class = P(func.__code__.co_filename), func.__qualname__.split(".")[0]
+            else:
+                # print(f"Passed function {func} is not a method of a class.")
+                func_file, func_class = P(func.__code__.co_filename), None
+        elif type(func) is str or type(func) is P:
+            func_file = P(func)
+            # func = None
+            func_class = None
+            func_name = None
+            func_module = func_file.stem
+        else: raise TypeError(f"Passed function {func} is not a callable or a path to a python file.")
+        try:
+            repo_path = P(git.Repo(func_file, search_parent_directories=True).working_dir)
+            func_relative_file = func_file.relative_to(repo_path)
+        except Exception as e:
+            print(e)
+            repo_path, func_relative_file = func_file.parent, func_file.name
+        return JobParams(repo_path_rh=repo_path.collapseuser().as_posix(), file_path_rh=repo_path.collapseuser().joinpath(func_relative_file).collapseuser().as_posix(),
+                         file_path_r=P(func_relative_file).as_posix(),
+                         func_module=func_module, func_class=func_class, func_name=func_name,
+                         description="", ssh_repr="", ssh_repr_remote="", error_message="",
+                         session_name="", tab_name="", file_manager_path="")
+
+    def get_execution_line(self, workload_params: Optional[WorkloadParams], parallelize: bool, wrap_in_try_except: bool) -> str:
+        # P(self.repo_path_rh).name}.{self.file_path_r.replace(".py", '').replace('/', '.')#
+        # if func_module is not None:
+        #     # noinspection PyTypeChecker
+        #     module = __import__(func_module, fromlist=[None])
+        #     exec_obj = module.__dict__[func_name] if not bool(func_class) else getattr(module.__dict__[func_class], func_name)
+        # elif func_name is not None:
+        #     # This approach is not conducive to parallelization since "mymod" is not pickleable.
+        #     module = SourceFileLoader("mymod", P.home().joinpath(rel_full_path).as_posix()).load_module()  # loading the module assumes its not a script, there should be at least if __name__ == __main__ wrapping any script.
+        #     exec_obj = getattr(module, func_name) if not bool(func_class) else getattr(getattr(module, func_class), func_name)
+        # else:
+        #     module = P.home().joinpath(rel_full_path).readit()  # uses runpy to read .py files.
+        #     exec_obj = module  # for README.md generation.
+
+        if workload_params is not None: base = f"""
+workload_params = WorkloadParams(**{workload_params.__dict__})
+repo_path = P(rf'{self.repo_path_rh}').expanduser().absolute()
+file_root = P(rf'{self.file_path_rh}').expanduser().absolute().parent
+tb.sys.path.insert(0, repo_path.str)
+tb.sys.path.insert(0, file_root.str)
+"""
+        else: base = ""
+
+        # loading function ===============================================================
+        if self.func_name is not None:
+            if self.func_class is None: base += f"""
+from {self.func_module.replace('.py', '')} import {self.func_name} as func
+"""
+            elif self.func_class is not None:  # type: ignore
+                base += f"""
+from {self.func_module.replace('.py', '')} import {self.func_class} as {self.func_class}
+func = {self.func_class}.{self.func_name}
+"""
+        else: base = f"""
+res = None  # in case the file did not define it.
+# --------------------------------- SCRIPT AS IS
+{P(self.file_path_rh).expanduser().read_text()}
+# --------------------------------- END OF SCRIPT AS IS
+"""
+
+        if workload_params is not None and parallelize is False: base += """
+res = func(workload_params=workload_params, **func_kwargs)
+"""
+        elif workload_params is not None and parallelize is True: base += f"""
+kwargs_workload = {list(workload_params.split_to_jobs().apply(lambda a_kwargs: a_kwargs.__dict__))}
+workload_params = []
+for idx, x in enumerate(kwargs_workload):
+    S(x).print(as_config=True, title=f"Instance {{idx}}")
+    workload_params.append(WorkloadParams(**x))
+print("\\n" * 2)
+res = L(workload_params).apply(lambda a_workload_params: func(workload_params=a_workload_params, **func_kwargs), jobs={workload_params.jobs})
+"""
+        else: base += """
+res = func(**func_kwargs)
+"""
+
+        if wrap_in_try_except:
+            import textwrap
+            base = textwrap.indent(base, " " * 4)
+            base = f"""
+try:
+{base}
+except Exception as e:
+    print(e)
+    params.error_message = str(e)
+    res = None
+
+"""
+        return base