machineconfig 1.5__py3-none-any.whl → 1.8__py3-none-any.whl

machineconfig/utils/procs.py

@@ -1,64 +1,114 @@
-
-import psutil
-import pandas as pd
-from pytz import timezone
-from machineconfig.utils.utils import display_options
-
-
-pd.options.display.max_rows = 10000
-
-
-class ProcessManager:
-    def __init__(self):
-        process_info = []
-        for proc in psutil.process_iter():
-            try:
-                mem_usage_mb = proc.memory_info().rss / (1024 * 1024)
-                process_info.append([proc.pid, proc.name(), proc.username(), proc.cpu_percent(), mem_usage_mb, proc.status(), proc.create_time()])
-            except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): pass
-        df = pd.DataFrame(process_info)
-        df.columns = ['pid', 'name', 'username', 'cpu_percent', 'memory_usage_mb', 'status', 'create_time']
-        df['create_time'] = pd.to_datetime(df['create_time'], unit='s', utc=True).apply(lambda x: x.tz_convert(timezone('Australia/Adelaide')))
-        df = df.sort_values(by='memory_usage_mb', ascending=False).reset_index(drop=True)
-        self.df = df
-
-    def choose_and_kill(self):
-        options = str(self.df).split("\n")[1:]
-        res = display_options(options=str(self.df).split("\n"), msg="", fzf=True, multi=True)
-        indices = [options.index(val) for val in res]
-        sub_df = self.df.iloc[indices]
-        print(self.df)
-        print(sub_df)
-        self.kill(pids=sub_df.pid) if input("Confirm kill? y/[n] ").lower() == "y" else print("Not killing")
-
-    def filter_and_kill(self, name=None):
-        df_sub = self.df.query(f"name == '{name}' ").sort_values(by='create_time', ascending=True)
-        self.kill(pids=df_sub.pid)
-
-    def kill(self, names: list or None = None, pids: list or None = None):
-        if names is None and pids is None:
-            raise ValueError('names and pids cannot both be None')
-        if names is None: names = []
-        if pids is None: pids = []
-        for name in names:
-            rows = self.df[self.df['name'] == name]
-            if len(rows) > 0:
-                for pid in rows['pid'].values:
-                    psutil.Process(pid).kill()
-                    age = pd.Timestamp.now(tz="Australia/Adeliade") - pd.to_datetime(rows["create_time"].values[0], unit="s", utc=True).tz_convert(timezone("Australia/Adelaide"))
-                    print(f'Killed process {name} with pid {pid}. It lived {age}.')
-            else:
-                print(f'No process named {name} found')
-        for pid in pids:
-            try:
-                proc = psutil.Process(pid)
-                proc.kill()
-                age = pd.Timestamp.now(tz="Australia/Adelaide") - pd.to_datetime(proc.create_time(), unit="s", utc=True).tz_convert(timezone("Australia/Adelaide"))
-                print(f'Killed process with pid {pid} and name {proc.name()}.  It lived {age}.')
-            except psutil.NoSuchProcess:
-                print(f'No process with pid {pid} found')
-
-
-if __name__ == '__main__':
-    pass
-
+
+"""Procs
+"""
+import psutil
+import pandas as pd
+from tqdm import tqdm
+from pytz import timezone
+from machineconfig.utils.utils import display_options
+from typing import Optional
+
+pd.options.display.max_rows = 10000
+
+
+def get_processes_accessing_file(path: str):
+    res: dict[int, list[str]] = {}
+    for proc in tqdm(psutil.process_iter()):
+        try:
+            files = proc.open_files()
+        except psutil.AccessDenied:
+            continue
+        tmp = [file.path for file in files if path in file.path]
+        if len(tmp) > 0:
+            res[proc.pid] = tmp
+    df = pd.DataFrame(res.items(), columns=['pid', 'files'])
+    return df
+
+
+def kill_process(name: str):
+    for proc in psutil.process_iter():
+        if proc.name() == name:
+            proc.kill()
+
+
+class ProcessManager:
+    def __init__(self):
+        process_info = []
+        for proc in tqdm(psutil.process_iter(), desc="Reading Processes"):
+            try:
+                mem_usage_mb = proc.memory_info().rss / (1024 * 1024)
+                process_info.append([proc.pid, proc.name(), proc.username(), proc.cpu_percent(), mem_usage_mb, proc.status(), proc.create_time(), " ".join(proc.cmdline())])
+            except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): pass
+        df = pd.DataFrame(process_info)
+        df.columns = pd.Index(['pid', 'name', 'username', 'cpu_percent', 'memory_usage_mb', 'status', 'create_time', 'command'])
+        df['create_time'] = pd.to_datetime(df['create_time'], unit='s', utc=True).apply(lambda x: x.tz_convert(timezone('Australia/Adelaide')))
+        df = df.sort_values(by='memory_usage_mb', ascending=False).reset_index(drop=True)
+        self.df = df
+
+    def choose_and_kill(self):
+        options = str(self.df).split("\n")[1:]
+        res = display_options(options=str(self.df).split("\n"), msg="", fzf=True, multi=True)
+        indices = [options.index(val) for val in res]
+        sub_df = self.df.iloc[indices]
+        print(self.df)
+        print(sub_df)
+        from crocodile.core import Struct
+        for idx, (_, row) in enumerate(sub_df.iterrows()):
+            Struct(row.to_dict()).print(as_config=True, title=f"Process {idx}")
+        kill_all = input("🔪 Confirm killing ALL? y/[n] ").lower() == "y"
+        if kill_all:
+            self.kill(pids=sub_df.pid.to_list())
+            return
+        kill_by_index = input("🔫 Kill by index? 1 4 ... /[n] ")
+        if kill_by_index != "":
+            indices = [int(val) for val in kill_by_index.split(" ")]
+            sub_sub_df = sub_df.iloc[indices]
+            for idx2, row in sub_sub_df.iterrows():
+                Struct(row.to_dict()).print(as_config=True, title=f"Process {idx2}")
+            _ = self.kill(pids=sub_sub_df.pid.to_list()) if input("Confirm kill? y/[n] ").lower() == "y" else None
+        print("🫠🐔 Not killing any process.")
+
+    def filter_and_kill(self, name: Optional[str] = None):
+        _ = 20
+        df_sub = self.df.query(f"name == '{name}' ").sort_values(by='create_time', ascending=True)
+        self.kill(pids=df_sub.pid.to_list())
+
+    def kill(self, names: Optional[list[str]] = None, pids: Optional[list[int]] = None, commands: Optional[list[str]] = None):
+        if names is None and pids is None and commands is None:
+            raise ValueError('names, pids and commands cannot all be None')
+        if names is None: names = []
+        if pids is None: pids = []
+        if commands is None: commands = []
+        for name in names:
+            rows = self.df[self.df['name'] == name]
+            if len(rows) > 0:
+                for _idx, a_row in rows.iterrows():
+                    psutil.Process(a_row.pid).kill()
+                    print(f'💀 Killed process {name} with pid {a_row.pid}. It lived {get_age(a_row.create_time)}. RIP 🪦💐')
+            else: print(f'No process named {name} found')
+        for pid in pids:
+            try:
+                proc = psutil.Process(pid)
+                proc.kill()
+                print(f'💀 Killed process with pid {pid} and name {proc.name()}.  It lived {get_age(proc.create_time())}. RIP 🪦💐')
+            except psutil.NoSuchProcess: print(f'No process with pid {pid} found')
+        for command in commands:
+            rows = self.df[self.df['command'].str.contains(command)]
+            if len(rows) > 0:
+                for _idx, a_row in rows.iterrows():
+                    psutil.Process(a_row.pid).kill()
+                    print(f'💀 Killed process with `{command}` in its command & pid = {a_row.pid}. It lived {get_age(a_row.create_time)}. RIP 🪦💐')
+            else: print(f'No process has `{command}` in its command.')
+
+
+def get_age(create_time: float):
+    try: age = pd.Timestamp.now(tz="Australia/Adelaide") - pd.to_datetime(create_time, unit="s", utc=True).tz_convert(timezone("Australia/Adelaide"))
+    except Exception as e:
+        try: age = pd.Timestamp.now() - pd.to_datetime(create_time, unit="s", utc=True).tz_localize(tz=None)
+        except Exception as ee:  # type: ignore
+            return f"unknown due to {ee} and {e}"
+    return age
+
+
+if __name__ == '__main__':
+    pass

machineconfig/utils/scheduling.py

@@ -0,0 +1,188 @@
+
+
+"""Task scheduler
+"""
+
+from crocodile.file_management import P, Read, str2timedelta, Save
+# from crocodile.meta import Terminal
+from machineconfig.utils.utils import get_shell_script_executing_python_file
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+import platform
+import subprocess
+from typing import Optional
+# from crocodile.meta import Scheduler
+
+
+SCHEDULER_DEFAULT_ROOT = P.home().joinpath("dotfiles/scripts/.scheduler")
+SUCCESS = "success"
+DEFAULT_CONFIG = """
+[specs]
+frequency = 30d
+start = 2024-01-01 01:00
+
+[runtime]
+venv = ve
+"""
+
+
+class Register:
+    def __init__(self, root: str):
+        self.root = P(root).expanduser().absolute()
+
+    def register_runtime(self, frequency_months: int = 1):
+        start, end = self.get_report_start_end_datetimes(frequency_months=frequency_months)
+        report = Report(name="runtime", start=start, end=end, status="success")
+        report.to_path(self.root.joinpath("runtime.ini"))
+        return report
+
+    def read_runtime(self):
+        return Report.from_path(self.root.joinpath("runtime.ini"))
+
+    @staticmethod
+    def format_date(date: datetime):
+        return str(date.year)[2:] + "-" + str(date.month).zfill(2)
+    @staticmethod
+    def get_report_start_end_datetimes(frequency_months: int):
+        now = datetime.now()
+        import numpy as np
+        chunks = np.arange(start=1, stop=12, step=frequency_months)
+        chunk_now_start_month: int = chunks[chunks <= now.month][-1]
+        chunk_now_start = datetime(year=now.year, month=chunk_now_start_month, day=1)
+        from dateutil.relativedelta import relativedelta
+        previous_chunk_start = chunk_now_start - relativedelta(months=frequency_months)
+        return previous_chunk_start, chunk_now_start
+
+
+@dataclass
+class Report:
+    name: str
+    start: datetime
+    end: datetime
+    status: str
+
+    @classmethod
+    def from_path(cls, path: P, return_default_if_not_found: bool = False):
+        if not path.exists():
+            if return_default_if_not_found:
+                return Report(name=path.parent.name, start=datetime(year=2000, month=1, day=1), end=datetime(year=2000, month=1, day=1), status="NA")
+            else:
+                raise ValueError(f"Could not find report at {path}")
+        ini = Read.ini(path)['report']
+        return cls(
+            name=ini["name"],
+            start=datetime.fromisoformat(ini["start"]),
+            end=datetime.fromisoformat(ini["end"]),
+            status=ini["status"],
+        )
+    def to_path(self, path: P):
+        Save.ini(path=path, obj={'report': {
+            'name': self.name,
+            'start': self.start.isoformat(),
+            'end': self.end.isoformat(),
+            'status': str(self.status),
+        }})
+
+
+@dataclass
+class Task:
+    name: str
+    task_root: P
+    frequency: timedelta
+    start: datetime
+    venv: str
+    @property
+    def report_path(self):
+        return self.task_root.joinpath("report.ini")
+
+
+def read_task_from_dir(path: P):
+    tasks_config = Read.ini(path.joinpath("config.ini"))
+    task = Task(name=path.name,
+                task_root=path,
+                frequency=str2timedelta(tasks_config["specs"]["frequency"]),
+                start=datetime.fromisoformat(tasks_config["specs"]["start"]),
+                venv=tasks_config["runtime"]["venv"],
+                # output_dir=P(a_task_section["output_dir"]).expanduser().absolute(),
+                )
+    return task
+
+
+def main(root: Optional[str] = None, ignore_conditions: bool = True):
+    if root is None: root_resolved = SCHEDULER_DEFAULT_ROOT
+    else: root_resolved = P(root).expanduser().absolute()
+    tasks_dirs = root_resolved.search(files=False, folders=True).filter(lambda x: x.joinpath("task.py").exists())
+
+    print(root_resolved)
+    tasks: list[Task] = []
+    for a_dir in tasks_dirs:
+        tasks.append(read_task_from_dir(a_dir))
+
+    from machineconfig.utils.utils import choose_multiple_options
+    import pandas as pd
+    df_res = pd.DataFrame([Report.from_path(path=a_task.report_path).__dict__ for a_task in tasks])
+    tasks_chosen_raw = choose_multiple_options(df_res.to_markdown().splitlines(), "Choose tasks to run")
+    tasks_chosen = [tasks[int(a_task_chosen.split("|")[1])] for a_task_chosen in tasks_chosen_raw]
+
+    result: list[Report] = []
+    for a_task in tasks_chosen:
+        if not ignore_conditions:
+            answer, report = should_task_run(a_task)
+        else:
+            answer, report = True, None
+        if answer: report = run_task(a_task)
+        else:
+            assert report is not None
+        result.append(report)
+
+    df_res = pd.DataFrame([r.__dict__ for r in result])
+    print(df_res)
+    # root_resolved.joinpath("task_report.md").write_text(df_res.to_markdown(), encoding="utf-8")
+    return ""
+
+
+def should_task_run(task: Task, tolerance_mins: int = 1440) -> tuple[bool, Optional[Report]]:
+    if not task.report_path.exists():
+        print(f"Task {task.name} has no record of being run before, running now...")
+        return True, None
+    old_report = Report.from_path(task.report_path)
+    time_since_execution = datetime.now() - old_report.end
+    if time_since_execution > task.frequency:
+        print(f"⚠️ Task {task.name} has not been run for {time_since_execution}, It is mean to run every {task.frequency}. running now if time is okay ...")
+    elif old_report.status != SUCCESS:
+        print(f"⚠️ Task {task.name} last run failed, running now if time is okay ...")
+    else:
+        print(f"Task `{task.name}` was run successfully {time_since_execution} ago, skipping...")
+        return False, old_report
+
+    suitable_run_time = task.start.time()
+    time_now = datetime.now().time()
+    min_diff = abs(suitable_run_time.hour - time_now.hour) * 60 + abs(suitable_run_time.minute - time_now.minute)
+    if not min_diff < tolerance_mins:
+        status = f"⌚ Time now is not suitable for running task {task.name} (Ideally, it should be run at {suitable_run_time})"
+        print(status)
+        return False, old_report
+    return True, old_report
+
+
+def run_task(task: Task) -> Report:
+    start_time = datetime.now()
+
+    shell_script = get_shell_script_executing_python_file(python_file=task.task_root.joinpath("task.py").str, ve_name=task.venv)
+    shell_script_root = P.tmp().joinpath(f"tmp_scripts/scheduler/{task.name}").create()
+    try:
+        if platform.system() == 'Windows':
+            shell_script = shell_script_root.joinpath("run.ps1").write_text(shell_script)
+            subprocess.run(['powershell', '-ExecutionPolicy', 'Unrestricted', shell_script], check=True)
+        elif platform.system() == 'Linux':
+            shell_script = shell_script_root.joinpath("run.sh").write_text(shell_script)
+            subprocess.run(['bash', shell_script], check=True)
+        else: res = f"Error: Unsupported platform {platform.system()}."
+        res = SUCCESS
+    except subprocess.CalledProcessError: res = f"Error: The script {shell_script_root} failed to run."
+    except Exception as e: res = f"Error: An unexpected error occurred while running the script {shell_script_root}: {e}"
+
+    end_time = datetime.now()
+    report = Report(name=task.name, start=start_time, end=end_time, status=res.replace('\n', '_NL_').strip().replace('=', '_eq_'))
+    report.to_path(task.report_path)
+    return report