falconry 0.1.2__tar.gz

falconry-0.1.2/.flake8

@@ -0,0 +1,7 @@
+[flake8]
+max-complexity = 12
+max-line-length = 127
+count = True
+statistics = True
+import-order-style = google
+exclude = docs/conf.py

falconry-0.1.2/.github/workflows/publish-to-test-pypi.yml

@@ -0,0 +1,74 @@
+name: Publish Python 🐍 distribution 📦 to PyPI and TestPyPI
+
+on: push
+
+jobs:
+  build:
+    name: Build distribution 📦
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.x"
+    - name: Install pypa/build
+      run: >-
+        python3 -m
+        pip install
+        build
+        --user
+    - name: Build a binary wheel and a source tarball
+      run: python3 -m build
+    - name: Store the distribution packages
+      uses: actions/upload-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+
+  publish-to-testpypi:
+    name: Publish Python 🐍 distribution 📦 to TestPyPI
+    # if: startsWith(github.ref, 'refs/tags/')  # only publish to PyPI on tag pushes
+    needs:
+    - build
+    runs-on: ubuntu-latest
+
+    environment:
+      name: testpypi
+      url: https://test.pypi.org/p/falconry
+
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Publish distribution 📦 to TestPyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        repository-url: https://test.pypi.org/legacy/
+
+  publish-to-pypi:
+    name: >-
+      Publish Python 🐍 distribution 📦 to PyPI
+    if: startsWith(github.ref, 'refs/tags/')  # only publish to PyPI on tag pushes
+    needs:
+    - build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/falconry
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Publish distribution 📦 to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1

falconry-0.1.2/.github/workflows/python-package.yml

@@ -0,0 +1,43 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Python package
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install flake8 mypy
+        #pytest
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --statistics
+        # run mypy for type checking
+        mypy
+    # TODO: implement pytest (will require mock):
+    #- name: Test with pytest
+    #  run: |
+    #    pytest

falconry-0.1.2/.readthedocs.yaml

@@ -0,0 +1,35 @@
+# Read the Docs configuration file for Sphinx projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the OS, Python version and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+    # You can also specify other tool versions:
+    # nodejs: "20"
+    # rust: "1.70"
+    # golang: "1.20"
+
+# Build documentation in the "docs/" directory with Sphinx
+sphinx:
+  configuration: docs/conf.py
+  # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs
+  # builder: "dirhtml"
+  # Fail on all warnings to avoid broken references
+  # fail_on_warning: true
+
+# Optionally build your docs in additional formats such as PDF and ePub
+# formats:
+#   - pdf
+#   - epub
+
+# Optional but recommended, declare the Python requirements required
+# to build your documentation
+# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+    - requirements: docs/requirements.txt

falconry-0.1.2/CONTRIBUTING.md

@@ -0,0 +1,19 @@
+# Contributing to falconry
+
+Any contributions or feedback are welcome, please use github issues to report any problems.
+
+## Development
+
+I use flake8 and mypy to check for syntax errors, style and correct typing. These are checked in the CI but you can run them locally.
+
+Syntax + undefined names:
+
+    python3 -m  flake8 . --count --select=E9,F63,F7,F82 --show-source
+
+Any other problems, those do not cause pipeline to fail, only show warning:
+
+    python3 -m  flake8 . --count --statistics
+
+And finally mypy for type errors:
+
+    python3 -m mypy

falconry-0.1.2/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Filip Nechansky
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

falconry-0.1.2/PKG-INFO

@@ -0,0 +1,58 @@
+Metadata-Version: 2.4
+Name: falconry
+Version: 0.1.2
+Summary: A lightweight python package to create and manage your HTCondor jobs.
+Project-URL: Documentation, https://falconry.readthedocs.io/en/stable/
+Project-URL: Repository, https://github.com/fnechans/falconry
+Project-URL: Issues, https://github.com/fnechans/falconry/issues
+Author-email: Filip Nechansky <filip.nechansky@protonmail.com>
+License-File: LICENSE
+Classifier: Development Status :: 4 - Beta
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.8
+Requires-Dist: htcondor>=24.2.1
+Requires-Dist: ijson>=3.3.0
+Description-Content-Type: text/markdown
+
+# falconry
+
+![Python package](https://github.com/fnechans/falconry/workflows/Python%20package/badge.svg)
+[![Documentation Status](https://readthedocs.org/projects/falconry/badge/?version=latest)](https://falconry.readthedocs.io/en/latest/?badge=latest)
+
+## Introduction
+
+Falconry is lightweight python package to create and manage your [HTCondor](https://github.com/htcondor/) jobs.
+It handles things like job submission, dependent jobs, and job status checking. It periodically saves progress,
+so even if you disconnect or htcondor crashes, you can continue where you left off.
+
+Detailed documentation can be found on [ReadTheDocs](https://falconry.readthedocs.io/en/latest/index.html). You can also check `example.py` for an example of usage. Package has to be first installed using pip as described in section on [installation](#installation-using-pip).
+
+## Instalation using pip
+
+Falconry can be installed using pip:
+
+    $ pip3 install falconry
+
+## Installation from source
+
+To install falconry, simply call following in the repository directory:
+
+    $ pip3 install --user -e .
+
+Then you can include the package in your project simply by adding:
+
+    import falconry
+
+### Installing python3 API for HTCondor
+
+The package  requires htcondor API to run. One can simply do:
+
+    $ python3 -m pip  install --user -r requirements.txt
+
+though it might be better to install in virtual environment.

falconry-0.1.2/README.md

@@ -0,0 +1,36 @@
+# falconry
+
+![Python package](https://github.com/fnechans/falconry/workflows/Python%20package/badge.svg)
+[![Documentation Status](https://readthedocs.org/projects/falconry/badge/?version=latest)](https://falconry.readthedocs.io/en/latest/?badge=latest)
+
+## Introduction
+
+Falconry is lightweight python package to create and manage your [HTCondor](https://github.com/htcondor/) jobs.
+It handles things like job submission, dependent jobs, and job status checking. It periodically saves progress,
+so even if you disconnect or htcondor crashes, you can continue where you left off.
+
+Detailed documentation can be found on [ReadTheDocs](https://falconry.readthedocs.io/en/latest/index.html). You can also check `example.py` for an example of usage. Package has to be first installed using pip as described in section on [installation](#installation-using-pip).
+
+## Instalation using pip
+
+Falconry can be installed using pip:
+
+    $ pip3 install falconry
+
+## Installation from source
+
+To install falconry, simply call following in the repository directory:
+
+    $ pip3 install --user -e .
+
+Then you can include the package in your project simply by adding:
+
+    import falconry
+
+### Installing python3 API for HTCondor
+
+The package  requires htcondor API to run. One can simply do:
+
+    $ python3 -m pip  install --user -r requirements.txt
+
+though it might be better to install in virtual environment.

falconry-0.1.2/docs/conf.py

@@ -0,0 +1,174 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# falconry documentation build configuration file, created by
+# sphinx-quickstart on Wed Sep 23 13:48:31 2020.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import sys
+from os.path import abspath, dirname, join
+sys.path.insert(0, abspath(join(dirname(__file__), '../src')))
+
+autodoc_mock_imports = ['ijson', 'htcondor']
+
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    "sphinx.ext.autodoc",
+    "sphinx.ext.napoleon",
+    "sphinx_rtd_theme",
+]
+
+html_theme = "sphinx_rtd_theme"
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = 'falconry'
+copyright = '2020-2024, Filip Nechansky'
+author = 'Filip Nechansky'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = '0.1.2'
+# The full version, including alpha/beta/rc tags.
+release = '0.1.2'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = False
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+# html_theme = 'alabaster'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# This is required for the alabaster theme
+# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars
+html_sidebars = {
+    '**': [
+        'relations.html',  # needs 'show_related': True theme option to display
+        'searchbox.html',
+    ]
+}
+
+
+# -- Options for HTMLHelp output ------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'falconrydoc'
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'falconry.tex', 'falconry Documentation',
+     'Filip Nechansky', 'manual'),
+]
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'falconry', 'falconry Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'falconry', 'falconry Documentation',
+     author, 'falconry', 'One line description of project.',
+     'Miscellaneous'),
+]

falconry-0.1.2/docs/index.rst

@@ -0,0 +1,22 @@
+.. falconry documentation master file, created by
+   sphinx-quickstart on Wed Sep 23 13:48:31 2020.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to falconry's documentation!
+====================================
+
+HTCondor is powerful tool for managment of jobs on computation clusters. It, and especially its python API, can be a bit complicated to use for an usual user.
+
+The goal of falconry is to have a lightweight wrapper around the HTCondor python API to run jobs. In addition, it offers a manager, which automatically submits and controls jobs, and is able to handle dependent jobs. This way one can submit large number of inter-connected jobs without having to manually run or check anything.
+
+Falconry is running on python 3 and can be found on `GitHub <https://github.com/fnechans/falconry>`_.
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   usage/install
+   usage/quickstart
+   usage/job
+   usage/manager

falconry-0.1.2/docs/requirements.txt

@@ -0,0 +1 @@
+sphinx_rtd_theme

falconry-0.1.2/docs/usage/install.rst

@@ -0,0 +1,18 @@
+============
+Installation
+============
+
+The package  requires htcondor API to run. However, the dependency cannot be linked directly because the condor version depends on the version of htcondor your cluster uses.
+
+To install the python API for condor using pip: ::
+
+    $ python3 -m pip install --user requirements.txt
+
+To install falconry, simply call following in the repository directory: ::
+
+    $ python3 -m pip install --user -e .
+
+Then you can include the package in your project simply by adding: ::
+
+    import falconry
+

falconry-0.1.2/docs/usage/job.rst

@@ -0,0 +1,8 @@
+.. _job:
+
+===
+Job
+===
+
+.. autoclass:: falconry.job
+    :members:

falconry-0.1.2/docs/usage/manager.rst

@@ -0,0 +1,8 @@
+.. _manager:
+
+=======
+Manager
+=======
+
+.. autoclass:: falconry.manager
+    :members:

falconry-0.1.2/docs/usage/quickstart.rst

@@ -0,0 +1,84 @@
+==========
+Quickstart
+==========
+
+---
+Job
+---
+
+Basic unit of the falconry is a job, which simply mantains all properties and submition.  It can be imported simply as::
+
+    from falconry import job
+
+Jobs require an HTCondor schedd. There is more convenient way to acquire it in the ``manager`` class mentioned later on, for now let's set it up directly::
+
+    from falconry import ScheddWrapper
+    schedd = ScheddWrapper()
+
+which should automatically pick-up the local schedd. The job definition then needs a name - useful for identification with a larger number of jobs - and the schedd::
+
+    j = job(name, schedd)
+
+There are several ways to initialize the job properties, but for a simple job, one can use a predefined function ``simple_job``::
+
+    j.set_simple(executablaPath, logFilesPath)
+
+which only requires path to the executable and path to a dir where the log files will be saves. Both path can be relative wrt. to the directory where the python script is run.
+
+One can setup the expected run time with ``set_time(runtime)`` defined in seconds::
+
+    j.set_time(3600)
+
+Generally, one can add or overwrite any options to the job using ``set_custom(options)`` function where options are simply dictionary::
+
+    j.set_custom({"arguments": " --out X"})
+
+And then to submit the job simply::
+
+    j.submit()
+
+More details on job setup can be found in the :ref:`job` module documentation.
+
+-------
+Manager
+-------
+
+When launching large number of jobs, especially with some dependencies between them, it is convenient to use manager class. It handles all the jobs, queues and automatically submits those which are ready.
+
+The manager can be imported as::
+
+    from falconry import manager
+    mgr = manager(dir)
+
+It automatically finds local schedd, so jobs can be then initialized as::
+
+    from falconry import job
+    j = job(name, mgr.schedd)
+
+without need to import HTCondor. To add a job to the manager simply do::
+
+    mgr.add_job(j)
+
+If you want job to start after certain other jobs finish (dependency), add them first to the job::
+
+    j.add_dependency(j1, j2, j3)
+
+The manager will then start the job once all dependencies are succesfully finished. This means that jobs without dependencies are submitted automatically, no need to call ``job.submit()``
+
+Now, start the manager with following command::
+
+    mgr.start(checkTime)
+
+where the ``checkTime`` specifies time in seconds in between checks of job status. After each interval, it will print status of each jobs and submit those waiting in queue if dependencies are satisfied.
+
+However, user may want to interupt the programm, or there may be a crash. For that reason falconry periodically saves all managed jobs in a data.json file via ``save()`` function of the manager. To load previous instance of the manager then simply call::
+
+    mgr.load()
+
+More details on manager setup can be found in the :ref:`manager` module documentation.
+
+---------------
+Example program
+---------------
+
+An example of a complete implemenation can be found in `example.py <https://github.com/fnechans/falconry/blob/master/example.py>`_, which puts all these features together. It also uses command line parser to make the usage more convenient. E.g. it automatically loads previous instance if ``--cont`` command line argument is used.