machineconfig 1.9__py3-none-any.whl → 1.92__py3-none-any.whl

machineconfig/utils/scheduling.py

@@ -62,7 +62,7 @@ class Report:
     status: str
 
     @classmethod
-    def from_path(cls, path: P, return_default_if_not_found: bool = False):
+    def from_path(cls, path: P, return_default_if_not_found: bool=False):
         if not path.exists():
             if return_default_if_not_found:
                 return Report(name=path.parent.name, start=datetime(year=2000, month=1, day=1), end=datetime(year=2000, month=1, day=1), status="NA")
@@ -108,7 +108,7 @@ def read_task_from_dir(path: P):
     return task
 
 
-def main(root: Optional[str] = None, ignore_conditions: bool = True):
+def main(root: Optional[str] = None, ignore_conditions: bool=True):
     if root is None: root_resolved = SCHEDULER_DEFAULT_ROOT
     else: root_resolved = P(root).expanduser().absolute()
     tasks_dirs = root_resolved.search(files=False, folders=True).filter(lambda x: x.joinpath("task.py").exists())
@@ -168,7 +168,7 @@ def should_task_run(task: Task, tolerance_mins: int = 1440) -> tuple[bool, Optio
 def run_task(task: Task) -> Report:
     start_time = datetime.now()
 
-    shell_script = get_shell_script_executing_python_file(python_file=task.task_root.joinpath("task.py").str, ve_name=task.venv)
+    shell_script = get_shell_script_executing_python_file(python_file=task.task_root.joinpath("task.py").to_str(), ve_name=task.venv)
     shell_script_root = P.tmp().joinpath(f"tmp_scripts/scheduler/{task.name}").create()
     try:
         if platform.system() == 'Windows':

machineconfig/utils/utils.py

@@ -3,9 +3,9 @@
 Utils
 """
 
-from crocodile.file_management import P, randstr
+from crocodile.core import List as L
+from crocodile.file_management import P, randstr, PLike
 from crocodile.meta import Terminal
-from crocodile.core import install_n_import
 # import crocodile.environment as env
 import machineconfig
 from rich.text import Text
@@ -17,13 +17,12 @@ import subprocess
 from typing import Optional, Union, TypeVar, Iterable
 
 
-LIBRARY_ROOT = P(machineconfig.__file__).resolve().parent  # .replace(P.home().str.lower(), P.home().str)
+LIBRARY_ROOT = P(machineconfig.__file__).resolve().parent  # .replace(P.home().to_str().lower(), P.home().str)
 REPO_ROOT = LIBRARY_ROOT.parent.parent
-PROGRAM_PATH = (P.tmp().joinpath("shells/python_return_command") + (".ps1" if platform.system() == "Windows" else ".sh")).create(parents_only=True)
+PROGRAM_PATH = (P.home().joinpath("tmp_results", "shells", "python_return_command") + (".ps1" if platform.system() == "Windows" else ".sh"))
 CONFIG_PATH = P.home().joinpath(".config/machineconfig")
 INSTALL_VERSION_ROOT = CONFIG_PATH.joinpath("cli_tools_installers/versions")
-INSTALL_TMP_DIR = P.tmp(folder="tmp_installers")
-
+INSTALL_TMP_DIR = P.home().joinpath("tmp_results", "tmp_installers")
 DEFAULTS_PATH = P.home().joinpath("dotfiles/machineconfig/defaults.ini")
 
 
@@ -31,8 +30,7 @@ T = TypeVar("T")
 
 
 def choose_cloud_interactively() -> str:
-    from crocodile.core import List as L
-    print(f"Listing Remotes ... ")
+    print("Listing Remotes ... ")
     tmp = Terminal().run("rclone listremotes").op_if_successfull_or_default(strict_returcode=False)
     # consider this: remotes = Read.ini(P.home().joinpath(".config/rclone/rclone.conf")).sections()
     if isinstance(tmp, str):
@@ -40,8 +38,8 @@ def choose_cloud_interactively() -> str:
 
     else: raise ValueError(f"Got {tmp} from rclone listremotes")
     if len(remotes) == 0:
-        raise RuntimeError(f"You don't have remotes. Configure your rclone first to get cloud services access.")
-    cloud: str = choose_one_option(msg="WHICH CLOUD?", options=list(remotes), default=remotes[0], fzf=True)
+        raise RuntimeError("You don't have remotes. Configure your rclone first to get cloud services access.")
+    cloud: str=choose_one_option(msg="WHICH CLOUD?", options=list(remotes), default=remotes[0], fzf=True)
     return cloud
 
 
@@ -71,63 +69,78 @@ def sanitize_path(a_path: P) -> P:
 
 def match_file_name(sub_string: str, search_root: Optional[P] = None) -> P:
     """Look up current directory for file name that matches the passed substring."""
-    root = search_root if search_root is not None else P.cwd()
-    print(f"Searching for {sub_string} in {root}")
-    search_results = root.absolute().search(f"*{sub_string}*.py", r=True)
+    search_root_obj = search_root if search_root is not None else P.cwd()
+    search_root_obj = search_root_obj.absolute()
+    print(f"Searching for {sub_string} in {search_root_obj}")
+
+    search_root_objects = search_root_obj.search("*", not_in=["links", ".venv", ".git", ".idea", ".vscode", "node_modules", "__pycache__"])
+    search_results: L[P] = L([a_search_root_obj.search(f"*{sub_string}*", r=True) for a_search_root_obj in search_root_objects]).reduce(lambda x, y: x + y)  # type: ignore
+    search_results = search_results.filter(lambda x: x.suffix in (".py", ".sh", ".ps1"))
+
     if len(search_results) == 1:
         path_obj = search_results.list[0]
     elif len(search_results) > 1:
-        choice = choose_one_option(msg=f"Search results are ambiguous or non-existent", options=search_results.list, fzf=True)
+        msg = "Search results are ambiguous or non-existent, choose manually:"
+        print(msg)
+        choice = choose_one_option(msg=msg, options=search_results.list, fzf=True)
         path_obj = P(choice)
     else:
         # let's do a final retry with sub_string.small()
         sub_string_small = sub_string.lower()
         if sub_string_small != sub_string:
+            print("Retrying with small letters")
             return match_file_name(sub_string=sub_string_small)
         from git.repo import Repo
         from git.exc import InvalidGitRepositoryError
         try:
-            repo = Repo(root, search_parent_directories=True)
+            repo = Repo(search_root_obj, search_parent_directories=True)
             repo_root_dir = P(repo.working_dir)
-            if repo_root_dir != root:  # may be user is in a subdirectory of the repo root, try with root dir.
+            if repo_root_dir != search_root_obj:  # may be user is in a subdirectory of the repo root, try with root dir.
+                print("Retrying with root repo instea of cwd")
                 return match_file_name(sub_string=sub_string, search_root=repo_root_dir)
             else:
-                root = repo_root_dir
+                search_root_obj = repo_root_dir
         except InvalidGitRepositoryError:
             pass
 
-        if check_tool_exists("fzf"):
+        if check_tool_exists(tool_name="fzf"):
             try:
-                search_res = subprocess.run(f"cd '{root}'; fzf --filter={sub_string}", stdout=subprocess.PIPE, text=True, check=True, shell=True).stdout.split("\n")[:-1]
+                print(f"Using fd to searching for `{sub_string}` in `{search_root_obj}` ...")
+                fzf_cmd = f"cd '{search_root_obj}'; fd --type f --strip-cwd-prefix | fzf --filter={sub_string}"
+                search_res = subprocess.run(fzf_cmd, stdout=subprocess.PIPE, text=True, check=True, shell=True).stdout.split("\n")[:-1]
             except subprocess.CalledProcessError as cpe:
-                print(f"Failed at fzf search with {sub_string} in {root}.\n{cpe}")
+                print(f"Failed at fzf search with {sub_string} in {search_root_obj}.\n{cpe}")
                 msg = f"\n{'--' * 50}\n💥 Path {sub_string} does not exist. No search results\n{'--' * 50}\n"
                 raise FileNotFoundError(msg) from cpe
-            if len(search_res) == 1: return root.joinpath(search_res[0])
+
+            if len(search_res) == 1: return search_root_obj.joinpath(search_res[0])
             else:
+                print("Tring with raw fzf search ...")
                 try:
-                    res = subprocess.run(f"cd '{root}'; fzf --query={sub_string}", check=True, stdout=subprocess.PIPE, text=True, shell=True).stdout.strip()
+                    # res = subprocess.run(f"cd '{search_root_obj}'; fzf --query={sub_string}", check=True, stdout=subprocess.PIPE, text=True, shell=True).stdout.strip()
+                    res = subprocess.run(f"cd '{search_root_obj}'; fd | fzf --query={sub_string}", check=True, stdout=subprocess.PIPE, text=True, shell=True).stdout.strip()
                 except subprocess.CalledProcessError as cpe:
-                    print(f"Failed at fzf search with {sub_string} in {root}. {cpe}")
+                    print(f"Failed at fzf search with {sub_string} in {search_root_obj}. {cpe}")
                     msg = f"\n{'--' * 50}\n💥 Path {sub_string} does not exist. No search results\n{'--' * 50}\n"
                     raise FileNotFoundError(msg) from cpe
-                return root.joinpath(res)
+                return search_root_obj.joinpath(res)
+
         msg = f"\n{'--' * 50}\n💥 Path {sub_string} does not exist. No search results\n{'--' * 50}\n"
         raise FileNotFoundError(msg)
     print(f"\n{'--' * 50}\n🔗 Matched `{sub_string}` ➡️ `{path_obj}`\n{'--' * 50}\n")
     return path_obj
 
 
-def choose_one_option(options: Iterable[T], header: str = "", tail: str = "", prompt: str = "", msg: str = "",
-                      default: Optional[T] = None, fzf: bool = False, custom_input: bool = False) -> T:
+def choose_one_option(options: Iterable[T], header: str="", tail: str="", prompt: str="", msg: str="",
+                      default: Optional[T] = None, fzf: bool=False, custom_input: bool=False) -> T:
     choice_key = display_options(msg=msg, options=options, header=header, tail=tail, prompt=prompt,
                                  default=default, fzf=fzf, multi=False, custom_input=custom_input)
     assert not isinstance(choice_key, list)
     return choice_key
 
 
-def choose_multiple_options(options: Iterable[T], header: str = "", tail: str = "", prompt: str = "", msg: str = "",
-                            default: Optional[T] = None, custom_input: bool = False) -> list[T]:
+def choose_multiple_options(options: Iterable[T], header: str="", tail: str="", prompt: str="", msg: str="",
+                            default: Optional[T] = None, custom_input: bool=False) -> list[T]:
     choice_key = display_options(msg=msg, options=options, header=header, tail=tail, prompt=prompt,
                                  default=default, fzf=True, multi=True,
                                  custom_input=custom_input)
@@ -135,14 +148,14 @@ def choose_multiple_options(options: Iterable[T], header: str = "", tail: str =
     return [choice_key]
 
 
-def display_options(msg: str, options: Iterable[T], header: str = "", tail: str = "", prompt: str = "",
-                    default: Optional[T] = None, fzf: bool = False, multi: bool = False, custom_input: bool = False) -> Union[T, list[T]]:
-    # TODO: replace with https://github.com/tmbo/questionary  # also see https://github.com/charmbracelet/gum
+def display_options(msg: str, options: Iterable[T], header: str="", tail: str="", prompt: str="",
+                    default: Optional[T] = None, fzf: bool=False, multi: bool=False, custom_input: bool=False) -> Union[T, list[T]]:
+    # TODO: replace with https://github.com/tmbo/questionary
+    # # also see https://github.com/charmbracelet/gum
     tool_name = "fzf"
     options_strings: list[str] = [str(x) for x in options]
     default_string = str(default) if default is not None else None
     if fzf and check_tool_exists(tool_name):
-        install_n_import("pyfzf")
         from pyfzf.pyfzf import FzfPrompt
         fzf_prompt = FzfPrompt()
         nl = "\n"
@@ -163,7 +176,7 @@ def display_options(msg: str, options: Iterable[T], header: str = "", tail: str
         console = Console()
         if default is not None:
             assert default in options, f"Default `{default}` option not in options `{list(options)}`"
-            default_msg = Text(f" <<<<-------- DEFAULT", style="bold red")
+            default_msg = Text(" <<<<-------- DEFAULT", style="bold red")
         else: default_msg = Text("")
         txt = Text("\n" + msg + "\n")
         for idx, key in enumerate(options):
@@ -176,10 +189,10 @@ def display_options(msg: str, options: Iterable[T], header: str = "", tail: str
 
         if choice_string == "":
             if default_string is None:
-                print(f"Default option not available!")
+                print("Default option not available!")
                 return display_options(msg=msg, options=options, header=header, tail=tail, prompt=prompt, default=default, fzf=fzf, multi=multi, custom_input=custom_input)
             choice_idx = options_strings.index(default_string)
-            assert default is not None, f"🧨 Default option not available!"
+            assert default is not None, "🧨 Default option not available!"
             choice_one: T = default
         else:
             try:
@@ -198,12 +211,12 @@ def display_options(msg: str, options: Iterable[T], header: str = "", tail: str
                 elif custom_input:
                     return choice_string  # type: ignore #TODO: fix this
                 else: raise ValueError(f"Unknown choice. {choice_string}") from te
-        print(f"{choice_idx}: {choice_one}", f"<<<<-------- CHOICE MADE")
+        print(f"{choice_idx}: {choice_one}", "<<<<-------- CHOICE MADE")
         if multi: return [choice_one]
     return choice_one
 
 
-def symlink(this: P, to_this: P, prioritize_to_this: bool = True):
+def symlink_func(this: P, to_this: P, prioritize_to_this: bool=True):
     """helper function. creates a symlink from `this` to `to_this`.
     What can go wrong?
     depending on this and to_this existence, one will be prioretized depending on overwrite value.
@@ -222,14 +235,32 @@ def symlink(this: P, to_this: P, prioritize_to_this: bool = True):
             else: this.move(path=to_this)  # this exists, to_this doesn't, this is prioritized.
     else:  # this doesn't exist.
         if not to_this.exists(): to_this.touch()  # we have to touch it (file) or create it (folder)
-    if platform.system() == "Windows": _ = install_n_import("win32api", "pywin32")  # this is crucial for windows to pop up the concent window in case python was not run as admin.
     try:
         # print(f"Linking {this} ➡️ {to_this}")
         P(this).symlink_to(target=to_this, verbose=True, overwrite=True)
     except Exception as ex: print(f"Failed at linking {this} ➡️ {to_this}.\nReason: {ex}")
 
 
-def get_shell_script_executing_python_file(python_file: str, func: Optional[str] = None, ve_name: str = "ve", strict_execution: bool = True):
+def symlink_copy(this: P, to_this: P, prioritize_to_this: bool=True):
+    this = P(this).expanduser().absolute()
+    to_this = P(to_this).expanduser().absolute()
+    if this.is_symlink(): this.delete(sure=True)  # delete if it exists as symblic link, not a concrete path.
+    if this.exists():  # this is a problem. It will be resolved via `overwrite`
+        if prioritize_to_this is True:  # it *can* be deleted, but let's look at target first.
+            if to_this.exists():  # this exists, to_this as well. to_this is prioritized.
+                this.append(f".orig_{randstr()}", inplace=True)  # rename is better than deletion
+            else: this.move(path=to_this)  # this exists, to_this doesn't. to_this is prioritized.
+        elif prioritize_to_this is False:  # don't sacrefice this, sacrefice to_this.
+            if to_this.exists(): this.move(path=to_this, overwrite=True)  # this exists, to_this as well, this is prioritized.   # now we are readly to make the link
+            else: this.move(path=to_this)  # this exists, to_this doesn't, this is prioritized.
+    else:  # this doesn't exist.
+        if not to_this.exists(): to_this.touch()  # we have to touch it (file) or create it (folder)
+    try:
+        to_this.copy(path=this, overwrite=True, verbose=True)
+    except Exception as ex: print(f"Failed at linking {this} ➡️ {to_this}.\nReason: {ex}")
+
+
+def get_shell_script_executing_python_file(python_file: str, func: Optional[str] = None, ve_name: str="ve", strict_execution: bool=True):
     if func is None: exec_line = f"""python {python_file}"""
     else: exec_line = f"""python -m fire {python_file} {func}"""
     shell_script = f"""
@@ -247,7 +278,15 @@ deactivate || true
     return shell_script
 
 
-def get_shell_file_executing_python_script(python_script: str, ve_name: str = "ve", verbose: bool = True):
+def get_shell_script(shell_script: str):
+    if platform.system() == "Linux": suffix = ".sh"
+    elif platform.system() == "Windows": suffix = ".ps1"
+    else: raise NotImplementedError(f"Platform {platform.system()} not implemented.")
+    shell_file = P.tmp().joinpath("tmp_scripts", "shell", randstr() + suffix).create(parents_only=True).write_text(shell_script)
+    return shell_file
+
+
+def get_shell_file_executing_python_script(python_script: str, ve_name: str, verbose: bool=True):
     if verbose:
         python_script = f"""
 code = r'''{python_script}'''
@@ -257,15 +296,13 @@ try:
 except ImportError: print(code)
 """ + python_script
     python_file = P.tmp().joinpath("tmp_scripts", "python", randstr() + ".py").create(parents_only=True).write_text(python_script)
-    shell_script = get_shell_script_executing_python_file(python_file=python_file.str, ve_name=ve_name)
-    if platform.system() == "Linux": suffix = ".sh"
-    elif platform.system() == "Windows": suffix = ".ps1"
-    else: raise NotImplementedError(f"Platform {platform.system()} not implemented.")
-    shell_file = P.tmp().joinpath("tmp_scripts", "shell", randstr() + suffix).create(parents_only=True).write_text(shell_script)
+    shell_script = get_shell_script_executing_python_file(python_file=python_file.to_str(), ve_name=ve_name)
+    shell_file = get_shell_script(shell_script)
     return shell_file
 
 
-def write_shell_script(program: str, desc: str = "", preserve_cwd: bool = True, display: bool = True, execute: bool = False):
+# def write_shell_script(program: str, desc: str="", preserve_cwd: bool=True, display: bool=True, execute: bool=False):
+def write_shell_script(program: str, desc: str, preserve_cwd: bool, display: bool, execute: bool):
     if preserve_cwd:
         if platform.system() == "Windows":
             program = "$orig_path = $pwd\n" + program + "\ncd $orig_path"
@@ -274,13 +311,13 @@ def write_shell_script(program: str, desc: str = "", preserve_cwd: bool = True,
     if display:
         print(f"Executing {PROGRAM_PATH}")
         print_code(code=program, lexer="shell", desc=desc)
-    if platform.system() == 'Windows': PROGRAM_PATH.create(parents_only=True).write_text(program)
-    else: PROGRAM_PATH.create(parents_only=True).write_text(f"{program}")
-    if execute: Terminal().run(f". {PROGRAM_PATH}", shell="powershell").print_if_unsuccessful(desc="Executing shell script", strict_err=True, strict_returncode=True)
+    PROGRAM_PATH.create(parents_only=True).write_text(program)
+    if execute:
+        Terminal().run(f". {PROGRAM_PATH}", shell="powershell").print_if_unsuccessful(desc="Executing shell script", strict_err=True, strict_returncode=True)
     return None
 
 
-def print_code(code: str, lexer: str, desc: str = ""):
+def print_code(code: str, lexer: str, desc: str):
     if lexer == "shell":
         if platform.system() == "Windows": lexer = "powershell"
         elif platform.system() == "Linux": lexer = "sh"
@@ -314,7 +351,7 @@ def check_tool_exists(tool_name: str, install_script: Optional[str] = None) -> b
 
     try:
         _tmp = subprocess.check_output([cmd, tool_name], stderr=subprocess.DEVNULL)
-        res: bool = True
+        res: bool=True
     except (subprocess.CalledProcessError, FileNotFoundError):
         res = False
     if res is False and install_script is not None:
@@ -327,13 +364,12 @@ def check_tool_exists(tool_name: str, install_script: Optional[str] = None) -> b
 def get_ssh_hosts() -> list[str]:
     from paramiko import SSHConfig
     c = SSHConfig()
-    c.parse(open(P.home().joinpath(".ssh/config").str, encoding="utf-8"))
+    c.parse(open(P.home().joinpath(".ssh/config").to_str(), encoding="utf-8"))
     return list(c.get_hostnames())
-def choose_ssh_host(multi: bool = True): return display_options(msg="", options=get_ssh_hosts(), multi=multi, fzf=True)
-
+def choose_ssh_host(multi: bool=True): return display_options(msg="", options=get_ssh_hosts(), multi=multi, fzf=True)
 
 
-def check_dotfiles_version_is_beyond(commit_dtm: str, update: bool = False):
+def check_dotfiles_version_is_beyond(commit_dtm: str, update: bool=False):
     dotfiles_path = str(P.home().joinpath("dotfiles"))
     from git import Repo
     repo = Repo(path=dotfiles_path)
@@ -345,10 +381,54 @@ def check_dotfiles_version_is_beyond(commit_dtm: str, update: bool = False):
     if res is False and update is True:
         print(f"Updating dotfiles because {dtm} < {datetime.fromisoformat(commit_dtm)}")
         from machineconfig.scripts.python.cloud_repo_sync import main
-        main(cloud=None, path=dotfiles_path, push=False)
+        main(cloud=None, path=dotfiles_path)
     return res
 
 
+def build_links(target_paths: list[tuple[PLike, str]], repo_root: PLike):
+    """Build symboic links from various relevant paths (e.g. data) to `repo_root/links/<name>` to facilitate easy access from
+    tree explorer of the IDE.
+    """
+    target_dirs_filtered: list[tuple[P, str]] = []
+    for a_dir, a_name in target_paths:
+        a_dir_obj = P(a_dir).resolve()
+        if not a_dir_obj.exists():
+            a_dir_obj.mkdir(parents=True, exist_ok=True)
+        target_dirs_filtered.append((a_dir_obj, a_name))
+
+    import git
+    repo = git.Repo(repo_root, search_parent_directories=True)
+    root_maybe = repo.working_tree_dir
+    assert root_maybe is not None
+    repo_root_obj = P(root_maybe)
+    tmp_results_root = P.home().joinpath("tmp_results", "tmp_data", repo_root_obj.name)
+    tmp_results_root.mkdir(parents=True, exist_ok=True)
+    target_dirs_filtered.append((tmp_results_root, "tmp_results"))
+
+    for a_target_path, a_name in target_dirs_filtered:
+        links_path = repo_root_obj.joinpath("links", a_name)
+        links_path.symlink_to(target=a_target_path)
+
+
+def wait_for_jobs_to_finish(root: P, pattern: str, wait_for_n_jobs: int, max_wait_minutes: float) -> bool:
+    wait_finished: bool=False
+    import time
+    t0 = time.time()
+    while not wait_finished:
+        parts = root.search(pattern, folders=False, r=False)
+        counter  =  len(parts)
+        if counter == wait_for_n_jobs:
+            wait_finished = True
+            print(f"{counter} Jobs finished. Exiting.")
+            return True
+        if (time.time() - t0) > 60 * max_wait_minutes:
+            print(f"Waited for {max_wait_minutes} minutes. Exiting.")
+            return False
+        print(f"{counter} Jobs finished. Waiting for {wait_for_n_jobs - counter} / {wait_for_n_jobs} jobs to finish, sleeping for 60 seconds.")
+        time.sleep(60)
+    return False
+
+
 if __name__ == '__main__':
     # import typer
     # typer.run(check_tool_exists)