falconry 0.1.2__py3-none-any.whl

falconry/__init__.py

@@ -0,0 +1,5 @@
+from .manager import manager  # NOQA
+from .job import job  # NOQA
+from . import translate  # NOQA
+from . import cli  # NOQA
+from .schedd_wrapper import ScheddWrapper  # NOQA

falconry/cli.py

@@ -0,0 +1,54 @@
+import sys
+import select
+import logging
+from enum import Enum
+from typing import Dict, Optional, Tuple
+
+log = logging.getLogger('falconry')
+
+
+# helper class to define the status
+class InputState(Enum):
+    UNKNOWN = -1
+    SUCCESS = 0
+    TIMEOUT = 1
+
+
+def input_checker(
+    validOptions: Dict[str, str],
+    timeout: int = 60,
+    message: str = "Following options available:",
+    silent: bool = False,
+) -> Tuple[InputState, Optional[str]]:
+    """Helper function to get input from user
+
+    Arguments:
+        validOptions (Dict[str, str]): dictionary of valid options
+        timeout (int, optional): timeout in seconds. Defaults to 60.
+        message (str, optional): message to print before options.
+        Defaults to "Following options available:".
+        silent (bool, optional): silent mode. Defaults to False.
+
+    Returns:
+        Tuple[InputState, Optional[str]]: returns the state of the input
+    """
+    if message != "" and not silent:
+        log.info(message)
+    for opt, desc in validOptions.items():
+        if desc != "" and not silent:
+            log.info(f"{opt} - {desc}")
+
+    i, o, e = select.select([sys.stdin], [], [], timeout)
+    if i:
+        inp = sys.stdin.readline().strip()
+        if inp in validOptions.keys():
+            return InputState.SUCCESS, inp
+
+        if not silent:
+            log.info(f"Unknown state {inp}!")
+        return InputState.UNKNOWN, inp
+
+    else:
+        if not silent:
+            log.info("Timed out ...")
+        return InputState.TIMEOUT, None

falconry/job.py

@@ -0,0 +1,383 @@
+import htcondor
+import os
+import logging
+
+from typing import List, Dict, Any
+
+from . import translate
+from .schedd_wrapper import ScheddWrapper
+
+log = logging.getLogger('falconry')
+
+
+class job:
+    """Submits and holds a single job and all relevant information
+
+    The schedd can be imported as:
+
+    .. code-block:: python
+
+        from falconry import ScheddWrapper
+        schedd = ScheddWrapper()
+
+    or from the manager:
+
+    .. code-block:: python
+
+        from falconry import manager
+        mgr = manager.Manager(mgrDir, mgrMsg)
+        schedd = mgr.schedd
+
+    Currently planning to keep single job per clusterID,
+    since group submittion would significantly complicate resubmitting.
+    HTCondor does not seem to allow for re-submittion of single ProcId,
+    so one would have to first connect specific arguments to specific ProcIds
+    and then resubmit individual jobs anyway.
+
+
+    Arguments:
+        name (str): name of the job for easy identification
+        schedd (ScheddWrapper): HTCondor schedd wrapper
+    """
+
+    def __init__(self, name: str, schedd: ScheddWrapper) -> None:
+
+        # first, define HTCondor schedd wrapper
+        self.schedd = schedd
+
+        # name of the job for easy identification
+        self.name = name
+
+        # since we will be resubmitting, job IDs are kept as a list
+        self.clusterIDs: List[str] = []
+
+        # add a decoration to the job to hold dependencies
+        self.dependencies: List["job"] = []
+
+        # configuration of the jobs
+        self.config: Dict[str, str] = {}
+
+        # to setup initial state (done/submitted and so on)
+        self.reset()
+
+    def set_simple(self, exe: str, logPath: str):
+        """Sets up a simple job with only executable and a path to log files
+
+        Arguments:
+            exe (str): path to the executable
+            logPath (str): path to the log files
+        """
+
+        # htcondor defines job as a dict
+        cfg = {
+            "executable": exe,
+            "output": logPath + "/" + self.name + "/$(ClusterId).out",
+            "error": logPath + "/" + self.name + "/$(ClusterId).err",
+            "log": logPath + "/" + self.name + "/$(ClusterId).log",
+        }
+        self.config = cfg
+
+        # create the directory for the log
+        logDir = os.getcwd() + "/" + logPath + "/" + self.name + "/"
+        if not os.path.exists(logDir):
+            os.makedirs(logDir)
+
+        # setup flags:
+        self.reset()
+
+    def save(self) -> Dict[str, Any]:
+        """Returns a dictionary containing all relevant job information
+        to be saved to a file.
+
+        Returns:
+            Dict[str, Any]: dictionary containing job information
+        """
+        # first rewrite dependencies using names
+        depNames = [j.name for j in self.dependencies]
+        jobDict = {
+            "clusterIDs": self.clusterIDs,
+            "config": self.config,
+            "depNames": depNames,
+            "done": "false",
+        }
+        # to test if job is done takes long time
+        # because log file needs to be checked
+        # so its best to save this status
+        if self.done:
+            jobDict["done"] = "true"
+        return jobDict
+
+    def load(self, jobDict: Dict[str, Any]) -> None:
+        """Loads a job from a dictionary created using the save function.
+
+        Arguments:
+            jobDict (Dict[str, Any]): dictionary containing job information
+        """
+
+        # TODO: define proper "jobDict checker"
+        if "clusterIDs" not in jobDict.keys() and "config" not in jobDict.keys():
+            log.error("Job dictionary in a wrong form")
+            raise SystemError
+
+        # the htcondor version of the configuration
+        self.config = jobDict["config"]
+
+        # setup flags:
+        self.reset()
+        # tmp backwards compatibility
+        if "done" in jobDict and jobDict["done"] == "true":
+            log.debug("Job is already done")
+            self.done = True
+
+        # set cluster IDs
+        self.clusterIDs = jobDict["clusterIDs"]
+        # if not empty, the job has been already submitted at least once
+        if len(self.clusterIDs):
+            self.htjob = htcondor.Submit(self.config)
+            self.clusterID = self.clusterIDs[-1]
+            self.logFile = self.config["log"].replace(
+                "$(ClusterId)", str(self.clusterID)
+            )
+            self.outFile = self.config["output"].replace(
+                "$(ClusterId)", str(self.clusterID)
+            )
+            self.errFile = self.config["error"].replace(
+                "$(ClusterId)", str(self.clusterID)
+            )
+            self.submitted = True
+
+    def reset(self) -> None:
+        """Resets job flags"""
+        self.submitted = False
+        self.skipped = False
+        self.failed = False
+        self.done = False
+
+    def add_job_dependency(self, *args: "job") -> None:
+        """Add dependencies to the job.
+
+        Arguments:
+            *args (List["job"]): list of jobs
+        """
+        self.dependencies.extend(list(args))
+
+    # submit the job
+    def submit(self, force: bool = False) -> None:
+        """Submits the job to HTCondor if either the job is not submitted,
+        the force flag is set or the job failed.
+
+        Arguments:
+            force (bool, optional): force submission. Defaults to False.
+        """
+        # TODO: raise error if problem
+
+        # force: for cases when the job status was checked
+        # e.g. when retrying
+        # this is can save a lot of time because
+        # failed job required the log file to be read.
+        # Should not be used by users, only internally.
+
+        # first check if job was not submitted before:
+        if not force and self.clusterIDs != []:
+            status = self.get_status()
+            if status == 12 or status < 0 or status == 10:
+                log.info("Job %s failed and will be resubmitted.", self.name)
+            else:
+                log.info(
+                    "The job is %s, not submitting", translate.statusMessage[status]
+                )
+                return
+        else:
+            # the htcondor version of the configuration
+            self.htjob = htcondor.Submit(self.config)
+
+        # Of course the submit has different capitalization here ...
+        self.submit_result = self.schedd.submit(self.htjob)
+        self.clusterID = self.submit_result.cluster()
+
+        self.clusterIDs.append(self.clusterID)
+        log.info("Submitting job %s with id %s", self.name, self.clusterID)
+        log.debug(self.config)
+        self.logFile = self.config["log"].replace("$(ClusterId)", str(self.clusterID))
+        self.outFile = self.config["output"].replace(
+            "$(ClusterId)", str(self.clusterID)
+        )
+        self.errFile = self.config["error"].replace("$(ClusterId)", str(self.clusterID))
+
+        # reset job properties
+        self.reset()
+        self.submitted = True
+
+    def release(self) -> bool:
+        """Releases held job"""
+        if self.clusterIDs == []:
+            return False
+        self.schedd.act(
+            htcondor.JobAction.Release, "ClusterId == " + str(self.clusterID)
+        )
+        log.info("Releasing job %s with id %s", self.name, self.clusterID)
+        return True
+
+    def remove(self) -> bool:
+        """Removes the job from HTCondor"""
+        if self.clusterIDs == []:
+            return False
+        self.schedd.act(
+            htcondor.JobAction.Remove, "ClusterId == " + str(self.clusterID)
+        )
+        log.info("Removing job %s with id %s", self.name, self.clusterID)
+        return True
+
+    def get_info(self) -> Dict[str, Any]:
+        """Returns information about the job
+
+        Returns:
+            Dict[str, Any]: dictionary containing job information
+        """
+        # check if job has an ID
+        if self.clusterIDs == []:
+            log.error("Trying to list info for a job which was not submitted")
+            raise SystemError
+
+        constr = "ClusterId == " + str(self.clusterID)
+        # get all job info of running job
+        ads = self.schedd.query(constraint=constr)
+
+        # if the job finished, query will be empty and we have to use history
+        # because condor is stupid, it returns and iterator (?),
+        # so just returning first element
+        # TODO: add more to projection
+        if ads == []:
+            for ad in self.schedd.history(constraint=constr, projection=["JobStatus"]):
+                return ad
+
+        # check if only one job was returned
+        if len(ads) != 1:
+            # empty job is probably job finished a long ago
+            # return specific code -999 and let get_status function
+            # sort the rest from the log files
+            if ads == []:
+                return {"JobStatus": -999}
+            else:
+                log.error(
+                    "HTCondor returned more than one jobs for given ID, this should not happen!"
+                )
+                log.error("Job %s with id %u", self.name, self.clusterID)
+                print(ads)
+                raise SystemError
+
+        return ads[0]  # we take only single job, so return onl the first eleement
+
+    def get_status(self) -> int:
+        """Returns status of the job, as defined in translate.py
+
+        Returns:
+            int: status of the job
+        """
+
+        # First check if the job is skipped or not even submitted
+        if self.skipped:
+            return 8
+        elif self.done:
+            return 4
+        elif self.clusterIDs == []:  # job was not even submitted
+            return 9
+        elif not os.path.isfile(self.logFile):
+            return 10
+
+        status_log = self._get_status_log()
+        if status_log != 0:  # 0 for unknown so try from condor
+            return status_log
+
+        cndr_status = self._get_status_condor()
+        # If job is incomplete, simply return the status:
+        if cndr_status != 4 and cndr_status != -999:
+            return cndr_status
+
+        log.error("Unknown output of job %s!", self.name)
+        return 0
+
+    def _get_status_condor(self) -> int:
+        """Returns status of the job, as defined in condor
+
+        Returns:
+            int: status of the job
+        """
+        return self.get_info()["JobStatus"]
+
+    def _get_status_log(self) -> int:
+        """Gets status from the log file
+
+        Returns:
+            int: status of the job
+        """
+        # Check log file to determine if job finished with an error
+        with open(self.logFile, 'r') as fl:
+            search = fl.read()
+
+        # User abortion is special case
+        if "Job was aborted by the user" in search:
+            # I think this is the same as 3 but need to check
+            return 12
+
+        # Sometimes `removed` is not properly saved
+        # (probably when continuing after long time?)
+        # so here alternative way
+        if "SYSTEM_PERIODIC_REMOVE" in search or "Job was aborted" in search:
+            return 3
+
+        # Otherwise check `"Job terminated"`. If the log does not contain it
+        # its unknown state
+        if "Job terminated" not in search:
+            return 0
+
+        # Evaluate `"Job terminated"`
+        searchSplit = search.split("\n")
+        for line in searchSplit:
+            if "Normal termination (return value" in line:
+                line = line.rstrip()  # remove '\n' at end of line
+                status = int(line.split("value")[1].strip()[:-1])
+
+                if status == 0:
+                    self.done = True
+                    return 4  # success
+
+                log.debug(f"Job failed {status}")
+                self.failed = True
+                # Positive  values reserved for falconry states,
+                # so return as negative
+                return -status
+        return 11  # no "Normal termination for Job terminated"
+
+    def set_custom(self, dict: Dict[str, str]) -> None:
+        """Sets custom configuration for the job from a dictionary
+
+        Arguments:
+            dict (Dict[str, str]): dictionary containing job configuration
+        """
+        for key, item in dict.items():
+            self.config[key] = item
+
+    def set_time(self, runTime: int, useRequestRuntime: bool = False) -> None:
+        """Sets time limit for the job.
+
+        For some clusters (DESY), RequestRuntime is used instead of MaxRuntime,
+        to use it set useRequestRuntime to `True`.
+
+        Arguments:
+            runTime (int): time limit in seconds
+            useRequestRuntime (bool, optional): use RequestRuntime option. Defaults to False.
+        """
+        self.config["+MaxRuntime"] = str(runTime)
+        # RequestRuntime seems to be DESY specific option and does not work
+        # e.g. in Prague (jobs get held), so this option is false by default
+        if useRequestRuntime:
+            self.config["+RequestRuntime"] = str(runTime)
+
+    def set_arguments(self, args: str) -> None:
+        """Sets arguments for the job
+
+        Arguments:
+            args (str): arguments for the job
+        """
+        self.config["arguments"] = args

falconry/manager.py

@@ -0,0 +1,624 @@
+import logging
+import json
+import ijson
+import os
+import shutil
+import sys
+import traceback
+import datetime
+import select
+from time import sleep
+
+from typing import Dict, Any, Tuple, Optional
+
+from .job import job
+from . import translate
+from . import cli
+from .schedd_wrapper import ScheddWrapper
+
+log = logging.getLogger('falconry')
+
+
+class counter:
+    # just holds few variables used in status print
+    def __init__(self):
+        self.waiting = 0
+        self.notSub = 0
+        self.idle = 0
+        self.run = 0
+        self.failed = 0
+        self.done = 0
+        self.skipped = 0
+        self.removed = 0
+        self.held = 0
+
+    def __eq__(self, other):
+        return self.__dict__ == other.__dict__
+
+
+class manager:
+    """Manager holds all jobs and periodically checks their status.
+
+    It also take care of dependent jobs,
+    submitting jobs when all dependencies are satisfied.
+    These are handled as decorations of the job.
+
+    Arguments:
+        mgrDir (str): directory where the manager stores the jobs
+        mgrMsg (str): message to be saved in the save file
+        maxJobIdle (int): maximum number of idle jobs
+    """
+
+    reservedNames = ["Message", "Command"]
+
+    def __init__(self, mgrDir: str, mgrMsg: str = "", maxJobIdle: int = -1):
+        log.info("MONITOR: INIT")
+
+        # Initialize the manager, maily getting the htcondor schedd
+        self.schedd = ScheddWrapper()
+
+        # job collection
+        self.jobs: Dict[str, job] = {}
+
+        # now create a directory where the info about jobs will be save
+        if not os.path.exists(mgrDir):
+            os.makedirs(mgrDir)
+        self.dir = mgrDir
+        self.saveFileName = self.dir + "/data.json"
+        self.mgrMsg = mgrMsg
+        self.command = " ".join(sys.argv)
+
+        self.maxJobIdle = maxJobIdle
+        self.curJobIdle = 0
+
+    # check if save file already exists
+    def check_savefile_status(self) -> Tuple[bool, Optional[str]]:
+        """Checks if the save file already exists. If it does, asks the user
+        whether to load existing jobs or start new ones.
+
+        Returns:
+            Tuple[bool, Optional[str]]: (True, 'l') if load, (True, 'n') if new,
+            (False, None) if error
+        """
+        if os.path.exists(self.saveFileName):
+            log.warning(f"Manager directory {self.dir} already exists!")
+            state, var = cli.input_checker(
+                {"l": "Load existing jobs", "n": "Start new jobs"}
+            )
+
+            # Simplify the output for user interface
+            # both unknown/timeout have the same result
+            if state == cli.InputState.SUCCESS:
+                return True, var
+            return False, var
+        return True, "n"  # automatically assume new
+
+    def ask_for_message(self):
+        """Asks user for a message to be saved in the save file for bookkeeping."""
+
+        log.info("Enter a message to be saved in the save file " "for bookkeeping.")
+        i, o, e = select.select([sys.stdin], [], [], 60)
+        if i:
+            self.mgrMsg = sys.stdin.readline().strip()
+
+    def add_job(self, j: job, update: bool = False):
+        """Adds a job to the manager. If the job already exists and `update` is
+        `True`, it will be updated.
+
+        Arguments:
+            j (job): job to be added
+            update (bool, optional): whether to update the job if it already
+            exists. Defaults to False.
+        """
+        # some reserved names, to simplify saving later
+        if j.name in manager.reservedNames:
+            log.error("Name %s  is reserved! Exiting ...", j.name)
+            raise SystemExit
+
+        # first check if the jobs already exists
+        if j.name in self.jobs.keys():
+            if not update:
+                log.error("Job %s already exists! Exiting ...", j.name)
+                raise SystemExit
+            else:
+                log.info(f"Updating job {j.name}.")
+
+        self.jobs[j.name] = j
+
+    def save(self, quiet: bool = False):
+        """Saves the current status of the jobs to a json file.
+
+        If `quiet` is `True`, it will not print any messages and
+        will not make a time-stamped copy of the save file.
+
+        Arguments:
+            quiet (bool, optional): whether to print messages. Defaults to False.
+        """
+        if not quiet:
+            log.info("Saving current status of jobs")
+        output: Dict[str, Any] = {
+            "Message": self.mgrMsg,
+            "Command": self.command,
+        }
+        for name, j in self.jobs.items():
+            output[name] = j.save()
+
+        # save with a timestamp as a suffix, create sym link
+        current_time = datetime.datetime.now().strftime("%Y%m%d_%H%M_%S")
+        fileLatest = f"{self.saveFileName}.latest"
+        fileSuf = f"{self.saveFileName}.{current_time}"  # only if not quiet
+
+        with open(fileLatest, "w") as f:
+            json.dump(output, f, indent=2)
+            if not quiet:
+                log.info("Success! Making copy with time-stamp.")
+                if not os.path.exists(fileSuf):
+                    shutil.copyfile(fileLatest, fileSuf)
+                else:
+                    raise FileExistsError(
+                        f"Destination file {fileSuf} already exists. "
+                        "This should not be possible."
+                    )
+
+        # not necessary to remove, but maybe better to be sure its not broken
+        if os.path.exists(self.saveFileName):
+            os.remove(self.saveFileName)
+        os.symlink(fileLatest.split("/")[-1], self.saveFileName)
+
+    def load(self, retryFailed: bool = False):
+        """Loads the saved status of the jobs from a json file
+        provided by the user.
+
+        Arguments:
+            retryFailed (bool, optional): whether to retry the failed jobs.
+            Defaults to False.
+        """
+        log.info("Loading past status of jobs")
+        with open(self.dir + "/data.json", "r") as f:
+            depNames = {}
+            for name, jobDict in ijson.kvitems(f, ""):
+                if name in manager.reservedNames:
+                    continue
+                log.debug("Loading job %s", name)
+
+                # create a job
+                j = job(name, self.schedd)
+                j.load(jobDict)
+
+                # add it to the manager
+                self.add_job(j, update=True)
+
+                # decorate the list of names of the dependencies
+                depNames[j.name] = jobDict["depNames"]
+
+        # Now that jobs are defined, dependencies can be recreated
+        # also resubmit jobs which failed
+        for j in self.jobs.values():
+            dependencies = [self.jobs[name] for name in depNames[j.name]]
+            j.add_job_dependency(*dependencies)
+
+        # Retry failed jobs
+        # Since this changes the status and submits
+        # jobs, add safequard in case of crash
+        # to save up-to-date state
+        if retryFailed:
+            try:
+                for j in self.jobs.values():
+                    self._check_resubmit(j, True)
+            except KeyboardInterrupt:
+                log.error("Manager interrupted with keyboard!")
+                log.error("Saving and exitting ...")
+                self.save()
+                self.print_failed()
+                sys.exit(0)
+            except Exception:
+                log.error("Error ocurred when running manager!")
+                traceback.print_exc(file=sys.stdout)
+                self.save()
+                self.print_failed()
+                sys.exit(1)
+
+    # print names of all failed jobs
+    def print_running(self, printLogs: bool = False):
+        log.info("Printing running jobs:")
+        for name, j in self.jobs.items():
+            if j.get_status() == 2:
+                log.info("%s (id %u)", name, j.clusterID)
+                if printLogs:
+                    log.info(f"log: {j.logFile}")
+                    log.info(f"out: {j.outFile}")
+                    log.info(f"err: {j.errFile}")
+
+    # print names of all failed jobs
+    def print_failed(self, printLogs: bool = False):
+        """Prints names of all failed jobs.
+
+        Arguments:
+            printLogs (bool, optional): whether to print paths to logs.
+                Defaults to False.
+        """
+        log.info("Printing failed jobs:")
+        for name, j in self.jobs.items():
+            if j.get_status() < 0:
+                log.info("%s (id %u)", name, j.clusterID)
+                if printLogs:
+                    log.info(f"log: {j.logFile}")
+                    log.info(f"out: {j.outFile}")
+                    log.info(f"err: {j.errFile}")
+        # TODO: maybe separate failed and removed?
+        log.info("Printing removed jobs:")
+        for name, j in self.jobs.items():
+            if j.get_status() == 3:
+                log.info("%s (id %u)", name, j.clusterID)
+                if printLogs:
+                    log.info(f"log: {j.logFile}")
+                    log.info(f"out: {j.outFile}")
+                    log.info(f"err: {j.errFile}")
+
+    def _check_dependence(self):
+        """Checks if all dependencies of a job are done. If so, it will submit
+        the job. If any of the dependencies failed, it will add the job to the
+        skipped list.
+        """
+
+        # TODO: consider if not submitted jobs in a special list
+        for name, j in self.jobs.items():
+            # only check jobs which are neither submitted nor skipped
+            if j.submitted or j.skipped:
+                continue
+
+            # if ready submit, single not done dependency leads to isReady=False
+            isReady = True
+            for tarJob in j.dependencies:
+                # if any job is not done, do not submit
+                if tarJob.done:
+                    continue
+
+                isReady = False
+
+                if tarJob.skipped or tarJob.failed:
+                    log.error(
+                        f"Job {name} depends on job {tarJob.name} which either failed or was skipped! Skipping ..."
+                    )
+                    j.skipped = True
+
+                status = tarJob.get_status()
+                if status == 3:
+                    log.error(
+                        f"Job {name} depends on job {tarJob.name} which is {translate.statusMessage[status]}! Skipping ..."
+                    )
+                    j.skipped = True
+
+                break
+
+            if isReady:
+                # Check if we did not reach maximum number of submitted jobs
+                if self.maxJobIdle != -1 and self.curJobIdle > self.maxJobIdle:
+                    break  # break because it does not make sense to check any other jobs now
+                j.submit()
+                self.curJobIdle += 1  # Add the jobs as a idle for now
+
+    def _check_resubmit(self, j: job, retryFailed: bool = False):
+        """Checks if a job should be resubmitted due to some known problems.
+
+        Arguments:
+            j (job): job to check
+            retryFailed (bool, optional): whether to also retry failed jobs.
+                Defaults to False.
+        """
+        status = j.get_status()
+        if status > 0:
+            log.debug("Job %s has status %s", j.name, translate.statusMessage[status])
+        if status == 12:
+            log.warning(
+                f"Error! Job {j.name} (id {j.clusterID}) failed due to condor, rerunning"
+            )
+            j.submit(force=True)
+        elif retryFailed and status < 0:
+            log.warning(
+                f"Error! Job {j.name} (id {j.clusterID}) failed and will be retried, rerunning"
+            )
+            j.submit(force=True)
+        elif retryFailed and status == 3:
+            log.warning(
+                f"Error! Job {j.name} (id {j.clusterID}) was removed and will be retried, rerunning"
+            )
+            j.submit(force=True)
+        elif retryFailed and j.submitted and (status == 9 or status == 10):
+            log.warning(
+                f"Error! Job {j.name} was not submitted succesfully (probably...), rerunning"
+            )
+            j.submit(force=True)
+
+        elif retryFailed and j.skipped:
+            log.warning(
+                f"Error! Job {j.name} was skipped and will be retried, rerunning"
+            )
+            j.skipped = False
+
+    def _count_jobs(self, c: counter):
+        """Counts the number of jobs with different status.
+        Resubmits jobs which failed due to condor problems.
+
+        Arguments:
+            c (counter): counter object to count the jobs
+        """
+
+        maxLength = 0
+        for name, j in self.jobs.items():
+            printStr = f"Checking {name}\t\t\t\t\t\t\r"
+            if len(printStr) > maxLength:
+                maxLength = len(printStr)
+            print(printStr, end='')
+
+            self._count_job(c, j)
+
+            print(" " * maxLength + "\r", flush=True, end='')
+
+    def _count_job(self, c: counter, j: job):
+        """Updates the counter object with the status of a single job.
+        Also resubmits jobs which failed due to condor problems.
+
+        Arguments:
+            c (counter): counter object to update
+            j (job): job to check
+        """
+
+        # first check if job is not submitted, skipped or done
+        if j.skipped:
+            c.skipped += 1
+            return
+        if not j.submitted:
+            c.waiting += 1
+            return
+        if j.done:
+            c.done += 1
+            return
+
+        #  resubmit job which failed due to condor problems
+        self._check_resubmit(j)
+
+        # count job with different status
+        status = j.get_status()
+        if status == 9 or status == 10:
+            c.notSub += 1
+        elif status == 1:
+            c.idle += 1
+        elif status == 2:
+            c.run += 1
+        elif status < 0:
+            c.failed += 1
+        elif status == 4:
+            c.done += 1
+        elif status == 5:
+            c.held += 1
+        elif status == 3:
+            c.removed += 1
+
+    def _start_cli(self, sleep_time: int = 60):
+        """Starts the manager, iteratively checking status of jobs.
+
+        Arguments:
+            sleep_time (int, optional): time to sleep between checks.
+                Defaults to 60.
+        """
+        # TODO: maybe add flag to save for each check? or every n-th check?
+
+        log.info("MONITOR: START")
+
+        c = counter()
+        event_counter = 0
+        while True:
+
+            log.info(
+                f"|-Checking status of jobs [{datetime.datetime.now()}]----------------|",
+            )
+
+            cOld = c
+            c = counter()
+            self._count_jobs(c)
+
+            # if no job is waiting nor running, finish the manager
+            if not (c.waiting + c.notSub + c.idle + c.run > 0):
+                break
+
+            # only printout if something changed:
+            if c != cOld:
+                sleep(0.2)  # the printing sometimes breaks here, adding delay helps...
+                log.info(
+                    "| nsub: {0:>4} | hold: {1:>5} | fail: {2:>6} | rem: {3:>6} | skip: {4:>5} |".format(
+                        c.notSub, c.held, c.failed, c.removed, c.skipped
+                    )
+                )
+                log.info(
+                    "| wait: {0:>6} | idle: {1:>4} | RUN: {2:>5} | DONE: {3:>6} | TOT: {4:>6} |".format(
+                        c.waiting, c.idle, c.run, c.done, len(self.jobs)
+                    )
+                )
+
+                # Update current idle of jobs managed by manager.
+                # All new jobs submitted jobs in `check_dependence`
+                # will increase this number, that why we create different
+                # variable than `c.idle`
+                self.curJobIdle = c.idle
+
+                # checking dependencies and submitting ready jobs
+                self._check_dependence()
+                self.save(quiet=True)
+
+                # instead of sleeping wait for input
+                log.info(
+                    "|-Enter 'h' to show all commands, e.g. to resubmit or show failed jobs|"
+                )
+
+            # save with timestamp every 30 events
+            # most important for first event when first
+            # batch of jobs is defined
+            if event_counter % 30 == 0:
+                self.save()
+            event_counter += 1
+
+            self._cli_interface(sleep_time)
+
+        log.info("MONITOR: FINISHED")
+
+    def _cli_interface(self, sleep_time: int = 60):
+        """CLI interface for the manager.
+
+        Arguments:
+            sleep_time (int, optional): time to sleep between checks.
+                Defaults to 60.
+        """
+        print('>>>> ', end='', flush=True)
+        state, var = cli.input_checker(
+            {
+                "h": "",
+                "s": "",
+                "f": "",
+                "x": "",
+                "ff": "",
+                "retry all": "",
+                "r": "",
+                "rr": "",
+            },
+            silent=True,
+            timeout=sleep_time,
+        )
+        if state == cli.InputState.TIMEOUT:
+            print('\r    \r', end='', flush=True)
+        elif state == cli.InputState.SUCCESS:
+            if var == "f":
+                self.print_failed()
+            elif var == "s":
+                self.save()
+            elif var == "h":
+                log.info(
+                    "|-Enter 'f' to show failed jobs, 'ff' to also show log paths----------|"
+                )
+                log.info(
+                    "|-Enter 'r' to show running jobs, 'rr' to also show log paths---------|"
+                )
+                log.info(
+                    "|-Enter 'x' to exit, 's' to save or 'retry all' to retry all failed---|"
+                )
+                self._cli_interface(sleep_time)
+            elif var == "ff":
+                self.print_failed(True)
+            elif var == "r":
+                self.print_running()
+            elif var == "rr":
+                self.print_running(True)
+            elif var == "x":
+                log.info("MONITOR: EXITING")
+                return
+            elif var == "retry all":
+                for j in self.jobs.values():
+                    self._check_resubmit(j, True)
+
+    def _start_gui(self, sleepTime: int = 60):
+        """Starts the manager with GUI, iteratively checking status of jobs.
+
+        This is only experimental!
+
+        Arguments:
+            sleepTime (int, optional): time to sleep between checks.
+                Defaults to 60.
+        """
+
+        log.warning("GUI version is only experimental!")
+        import tkinter as tk
+
+        window = tk.Tk()
+        window.title("Falconry monitor")
+        frm_counter = tk.Frame()
+
+        def quick_label(name: str, x: int, y: int = 0):
+            lbl = tk.Label(master=frm_counter, width=10, text=name)
+            lbl.grid(row=y, column=x)
+            return lbl
+
+        quick_label("Not sub.:", 0)
+        quick_label("Idle:", 1)
+        quick_label("Running:", 2)
+        quick_label("Failed:", 3)
+        quick_label("Done:", 4)
+        quick_label("Waiting:", 5)
+        quick_label("Skipped:", 6)
+        quick_label("Removed:", 7)
+        labels = {}
+        labels["ns"] = quick_label("0", 0, 1)
+        labels["i"] = quick_label("0", 1, 1)
+        labels["r"] = quick_label("0", 2, 1)
+        labels["f"] = quick_label("0", 3, 1)
+        labels["d"] = quick_label("0", 4, 1)
+        labels["w"] = quick_label("0", 5, 1)
+        labels["s"] = quick_label("0", 6, 1)
+        labels["rm"] = quick_label("0", 1)
+
+        frm_counter.grid(row=0, column=0)
+
+        def tk_count():
+            c = counter()
+            self._count_jobs(c)
+            labels["ns"]["text"] = f"{c.notSub}"
+            labels["i"]["text"] = f"{c.idle}"
+            labels["r"]["text"] = f"{c.run}"
+            labels["f"]["text"] = f"{c.failed}"
+            labels["d"]["text"] = f"{c.done}"
+            labels["w"]["text"] = f"{c.waiting}"
+            labels["s"]["text"] = f"{c.skipped}"
+            labels["rm"]["text"] = f"{c.removed}"
+
+            # if no job is waiting nor running, finish the manager
+            # TODO: add condition (close on finish)
+            # if not (c.waiting + c.notSub + c.idle + c.run > 0):
+            #    window.destroy()
+
+            # checking dependencies and submitting ready jobs
+            self._check_dependence()
+
+            window.after(1000 * sleepTime, tk_count)
+
+        tk_count()
+        log.info("MONITOR: START")
+        window.mainloop()
+        log.info("MONITOR: FINISHED")
+
+    def start(self, sleepTime: int = 60, gui: bool = False):
+        """Starts the manager, iteratively checking status of jobs.
+
+        Makes sure to save the current state of jobs
+        in case of interupt or crash.
+
+        Arguments:
+            sleepTime (int, optional): time to sleep between checks.
+                Defaults to 60.
+            gui (bool, optional): whether to use GUI. Defaults to False.
+                GUI is experimental!
+        """
+        try:
+            if gui:
+                self._start_gui(sleepTime)
+            else:
+                self._start_cli(sleepTime)
+        except KeyboardInterrupt:
+            log.error("Manager interrupted with keyboard!")
+            log.error("Saving and exitting ...")
+            self.save()
+            self.print_failed()
+            sys.exit(0)
+        except Exception as e:
+            log.error("Error ocurred when running manager!")
+            log.error(str(e))
+            self.save()
+            self.print_failed()
+            sys.exit(1)
+
+    def start_safe(self, sleepTime: int = 60, gui: bool = False):
+        """Deprecated! Use `start` instead!"""
+        log.warning(
+            "IMPORTANT! `start_safe` is now renamed as `start`. "
+            "Change your scripts as `start_safe` will be removed "
+            "in next version!"
+        )
+        self.start(sleepTime, gui)

falconry/schedd_wrapper.py

@@ -0,0 +1,47 @@
+import htcondor  # for submitting jobs, querying HTCondor daemons, etc.
+import logging
+import functools
+from typing import Callable, Any
+import time
+
+log = logging.getLogger('falconry')
+
+
+class ScheddWrapper:
+    """Wrapper to allow reload of schedd"""
+    def __init__(self) -> None:
+        self.schedd = htcondor.Schedd()
+
+    # Here the typing did not work properly ...
+    def schedd_check(func: Callable[["ScheddWrapper"], Any]):  # type: ignore
+        @functools.wraps(func)
+        def wrapper(self, *args, **kwargs):
+            try:
+                return func(self, *args, **kwargs)
+            except htcondor.HTCondorIOError:
+                log.warning("Possible problem with scheduler, waiting a bit and reloading schedd ...")
+                time.sleep(60)
+                self.schedd = htcondor.Schedd()
+                return func(self, *args, **kwargs)
+        return wrapper
+
+    """Reimplementing all the used functions"""
+    @schedd_check
+    def transaction(self):
+        return self.schedd.transaction()
+
+    @schedd_check
+    def act(self, *args, **kwargs):
+        return self.schedd.act(*args, **kwargs)
+
+    @schedd_check
+    def query(self, *args, **kwargs):
+        return self.schedd.query(*args, **kwargs)
+
+    @schedd_check
+    def history(self, *args, **kwargs):
+        return self.schedd.history(*args, **kwargs)
+
+    @schedd_check
+    def submit(self, *args, **kwargs):
+        return self.schedd.submit(*args, **kwargs)

falconry/translate.py

@@ -0,0 +1,17 @@
+
+# Translation of the Job status
+statusMessage = {
+    0: "UNKNOWN",
+    1: "IDLE",
+    2: "RUNNING",
+    3: "REMOVED",
+    4: "COMPLETE",
+    5: "HELD",
+    6: "TRANSPORTING",
+    7: "SUSPENDED",
+    8: "SKIPPED",
+    9: "NOT SUBMITTED",
+    10: "LOG FILE MISSING",
+    11: "ABNORMAL TERMINATION",
+    12: "ABORTED BY USER"
+}

falconry-0.1.2.dist-info/METADATA

@@ -0,0 +1,58 @@
+Metadata-Version: 2.4
+Name: falconry
+Version: 0.1.2
+Summary: A lightweight python package to create and manage your HTCondor jobs.
+Project-URL: Documentation, https://falconry.readthedocs.io/en/stable/
+Project-URL: Repository, https://github.com/fnechans/falconry
+Project-URL: Issues, https://github.com/fnechans/falconry/issues
+Author-email: Filip Nechansky <filip.nechansky@protonmail.com>
+License-File: LICENSE
+Classifier: Development Status :: 4 - Beta
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.8
+Requires-Dist: htcondor>=24.2.1
+Requires-Dist: ijson>=3.3.0
+Description-Content-Type: text/markdown
+
+# falconry
+
+![Python package](https://github.com/fnechans/falconry/workflows/Python%20package/badge.svg)
+[![Documentation Status](https://readthedocs.org/projects/falconry/badge/?version=latest)](https://falconry.readthedocs.io/en/latest/?badge=latest)
+
+## Introduction
+
+Falconry is lightweight python package to create and manage your [HTCondor](https://github.com/htcondor/) jobs.
+It handles things like job submission, dependent jobs, and job status checking. It periodically saves progress,
+so even if you disconnect or htcondor crashes, you can continue where you left off.
+
+Detailed documentation can be found on [ReadTheDocs](https://falconry.readthedocs.io/en/latest/index.html). You can also check `example.py` for an example of usage. Package has to be first installed using pip as described in section on [installation](#installation-using-pip).
+
+## Instalation using pip
+
+Falconry can be installed using pip:
+
+    $ pip3 install falconry
+
+## Installation from source
+
+To install falconry, simply call following in the repository directory:
+
+    $ pip3 install --user -e .
+
+Then you can include the package in your project simply by adding:
+
+    import falconry
+
+### Installing python3 API for HTCondor
+
+The package  requires htcondor API to run. One can simply do:
+
+    $ python3 -m pip  install --user -r requirements.txt
+
+though it might be better to install in virtual environment.

falconry-0.1.2.dist-info/RECORD

@@ -0,0 +1,10 @@
+falconry/__init__.py,sha256=LlVXOIZn1vOA46jO4Xi96W3qxWAdOK5qiAjid_IknCo,174
+falconry/cli.py,sha256=7dMrRzn3maF82pCovpO7I4jyhyFzqWG7HWBKu0kfxJE,1520
+falconry/job.py,sha256=OU5dncDrFp8pB3YA-PAp7NvwALqm6ha-vbVOvKw4hC8,13033
+falconry/manager.py,sha256=rIn9DoUb43NMxTVMW6-J6xWxuwHrhGW6IZ1Cul3E-N4,21831
+falconry/schedd_wrapper.py,sha256=Y9oWnPcmbdDDhLeaeGQX907qt9r7l11z31e6G3GnjA4,1461
+falconry/translate.py,sha256=3eHbHA9dW0HapNRJkcl28EZ61XXfOxdtcCmrIBZA0LM,327
+falconry-0.1.2.dist-info/METADATA,sha256=-JYe85EGHa1aNvLCzjzzgNB2ZZIYd2GXenk49fBRZtU,2329
+falconry-0.1.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+falconry-0.1.2.dist-info/licenses/LICENSE,sha256=SGozOSvKOihd2xzAhxloeYpU10cr7daz69N73JTtIs8,1071
+falconry-0.1.2.dist-info/RECORD,,

falconry-0.1.2.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any

falconry-0.1.2.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Filip Nechansky
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.