cr-hydra 0.2.0__tar.gz

cr_hydra-0.2.0/MANIFEST.in

@@ -0,0 +1 @@
+include debian/*

cr_hydra-0.2.0/PKG-INFO

@@ -0,0 +1,7 @@
+Metadata-Version: 2.1
+Name: cr_hydra
+Version: 0.2.0
+Summary: CRHydra - distributed CRTomo computing
+Author: Maximilian Weigand
+Author-email: mweigand@geo.uni-bonn.de
+License: MIT

cr_hydra-0.2.0/debian/changelog

@@ -0,0 +1,5 @@
+cr-hydra (1.0-0) UNRELEASED; urgency=medium
+
+  * Initial release.
+
+ -- mweigand <mweigand@nathan>  Thu, 27 Aug 2020 15:51:56 +0200

cr_hydra-0.2.0/debian/compat

@@ -0,0 +1 @@
+9

cr_hydra-0.2.0/debian/control

@@ -0,0 +1,14 @@
+Source: cr-hydra
+Section: python
+Maintainer: Maximilian Weigand <mweigand@geo.uni-bonn.de>
+Architecture: all
+Build-Depends: debhelper (>= 9), dh-python,
+ python3-all-dev (>= 3.1.2-8~),
+ python-setuptools (>= 0.6b3-1~), python3-setuptools,
+Depends: ${python3:Depends}, ${misc:Depends}
+Description: cr_hydra program suite for distributed CRTomo inversions
+
+Package: python3-crhydra
+Architecture: all
+Description: cr_hydra program suite for distributed CRTomo inversions
+

cr_hydra-0.2.0/debian/rules

@@ -0,0 +1,7 @@
+#! /usr/bin/make -f
+
+#export DH_VERBOSE = 1
+export PYBUILD_NAME = cr_hydra
+
+%:
+	dh $@ --with python3 --buildsystem=pybuild

cr_hydra-0.2.0/lib/cr_hydra/__init__.py

@@ -0,0 +1,7 @@
+import logging
+
+logging.basicConfig(
+    level=logging.INFO,
+    format='{asctime} - {name} - %{levelname} - {message}',
+    style='{',
+)

cr_hydra-0.2.0/lib/cr_hydra/settings.py

@@ -0,0 +1,42 @@
+"""Manage settings, for now only by means of a config file
+"""
+import os
+import configparser
+
+
+def _find_best_config_file():
+    """Look for a configuration file in certain locations of the file system
+
+    Returns
+    -------
+    filename : None|str
+        None if no config file was found, otherwise an absolute path to the
+        configuration file
+    """
+
+    # look at these file paths in increasing priority
+    locations = [
+        '/etc/crhydra/crhydra.cfg',
+        os.getenv('HOME') + os.sep + '.crhydra.cfg',
+        os.getcwd() + os.sep + 'crhydra.cfg',
+    ]
+
+    config_file = None
+    for filename in locations:
+        if os.path.isfile(filename):
+            config_file = os.path.abspath(filename)
+    return config_file
+
+
+def get_config():
+    """Return the configuration of crhydra, if a config file is available
+    """
+    config_file = _find_best_config_file()
+    if config_file is None:
+        raise IOError(
+            'No config file was found! It is required to supply the database' +
+            ' credentials!'
+        )
+    config = configparser.ConfigParser()
+    config.read(config_file)
+    return config

cr_hydra-0.2.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

cr_hydra-0.2.0/setup.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+import os
+import glob
+
+from setuptools import setup
+
+version_long = '0.2.0'
+
+# generate entry points
+entry_points = {'console_scripts': []}
+scripts = [os.path.basename(script)[0:-3] for script in glob.glob('src/*.py')]
+for script in scripts:
+    print(script)
+    entry_points['console_scripts'].append(
+        '{0} = {0}:main'.format(script)
+    )
+
+print(scripts, entry_points)
+
+if __name__ == '__main__':
+    setup(
+        name='cr_hydra',
+        version=version_long,
+        description='CRHydra - distributed CRTomo computing',
+        author='Maximilian Weigand',
+        author_email='mweigand@geo.uni-bonn.de',
+        license='MIT',
+        # url='https://github.com/geophysics-ubonn/reda',
+        packages=['cr_hydra', ],
+        package_dir={
+            '': 'src',
+            'cr_hydra': 'lib/cr_hydra',
+        },
+        py_modules=scripts,
+        entry_points=entry_points,
+        install_requires=[
+            'sqlalchemy >= 2.0',
+            'psycopg2-binary',
+            'ipython',
+            'numpy',
+            'pandas',
+        ],
+    )

cr_hydra-0.2.0/src/cr_hydra.egg-info/PKG-INFO

@@ -0,0 +1,7 @@
+Metadata-Version: 2.1
+Name: cr-hydra
+Version: 0.2.0
+Summary: CRHydra - distributed CRTomo computing
+Author: Maximilian Weigand
+Author-email: mweigand@geo.uni-bonn.de
+License: MIT

cr_hydra-0.2.0/src/cr_hydra.egg-info/SOURCES.txt

@@ -0,0 +1,18 @@
+MANIFEST.in
+setup.py
+debian/changelog
+debian/compat
+debian/control
+debian/rules
+lib/cr_hydra/__init__.py
+lib/cr_hydra/settings.py
+src/crh_add.py
+src/crh_get_file.py
+src/crh_retrieve.py
+src/crh_worker.py
+src/cr_hydra.egg-info/PKG-INFO
+src/cr_hydra.egg-info/SOURCES.txt
+src/cr_hydra.egg-info/dependency_links.txt
+src/cr_hydra.egg-info/entry_points.txt
+src/cr_hydra.egg-info/requires.txt
+src/cr_hydra.egg-info/top_level.txt

cr_hydra-0.2.0/src/cr_hydra.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

cr_hydra-0.2.0/src/cr_hydra.egg-info/entry_points.txt

@@ -0,0 +1,5 @@
+[console_scripts]
+crh_add = crh_add:main
+crh_get_file = crh_get_file:main
+crh_retrieve = crh_retrieve:main
+crh_worker = crh_worker:main

cr_hydra-0.2.0/src/cr_hydra.egg-info/requires.txt

@@ -0,0 +1,5 @@
+ipython
+numpy
+pandas
+psycopg2-binary
+sqlalchemy>=2.0

cr_hydra-0.2.0/src/cr_hydra.egg-info/top_level.txt

@@ -0,0 +1,5 @@
+cr_hydra
+crh_add
+crh_get_file
+crh_retrieve
+crh_worker

cr_hydra-0.2.0/src/crh_add.py

@@ -0,0 +1,322 @@
+#!/usr/bin/env python
+# *-* coding: utf-8 *-*
+"""Look for all unfinished tomodirs in the present directory (and
+subdirectories), e called.
+"""
+import logging
+import hashlib
+import io
+import shutil
+import uuid
+import os
+import datetime
+import json
+import tarfile
+import platform
+
+from sqlalchemy import create_engine
+from sqlalchemy import text
+from optparse import OptionParser
+import IPython
+
+from cr_hydra.settings import get_config
+IPython
+
+logger = logging.getLogger(__name__)
+
+
+def handle_cmd_options():
+    parser = OptionParser()
+    # parser.add_option(
+    #     '-t', "--threads",
+    #     dest="number_threads",
+    #     type="int",
+    #     help="number of threads EACH CRMod/CRTomo instance uses. If not " +
+    #     "set, will be determined automatically",
+    #     default=None,
+    # )
+
+    # parser.add_option(
+    #     "-n", "--number",
+    #     dest="number_processes",
+    #     help="How many CRMod/CRTomo instances to start in parallel. " +
+    #     "Default: number of detected CPUs/2",
+    #     type='int',
+    #     default=None,
+    # )
+
+    parser.add_option(
+        "-f", "--force",
+        dest="force_registration",
+        help="Assume that no other crh_add is running" +
+        " and add not fully initialized simulations",
+        action='store_true',
+    )
+
+    (options, args) = parser.parse_args()
+    return options
+
+
+def is_tomodir(subdirectories):
+    """provided with the subdirectories of a given directory, check if this is
+    a tomodir
+    """
+    required = (
+        'exe',
+        'config',
+        'rho',
+        'mod',
+        'inv'
+    )
+    is_tomodir = True
+    for subdir in required:
+        if subdir not in subdirectories:
+            is_tomodir = False
+    return is_tomodir
+
+
+def check_if_needs_modeling(tomodir):
+    """check of we need to run CRMod in a given tomodir
+    """
+    required_files = (
+        'config' + os.sep + 'config.dat',
+        'rho' + os.sep + 'rho.dat',
+        'grid' + os.sep + 'elem.dat',
+        'grid' + os.sep + 'elec.dat',
+        'exe' + os.sep + 'crmod.cfg',
+    )
+
+    not_allowed = (
+        'mod' + os.sep + 'volt.dat',
+    )
+    needs_modeling = True
+    for filename in not_allowed:
+        if os.path.isfile(tomodir + os.sep + filename):
+            needs_modeling = False
+
+    for filename in required_files:
+        full_file = tomodir + os.sep + filename
+        if not os.path.isfile(full_file):
+            needs_modeling = False
+
+    return needs_modeling
+
+
+def check_if_needs_inversion(tomodir):
+    """check of we need to run CRTomo in a given tomodir
+
+    Parameters
+    ----------
+    tomodir : str
+        Tomodir to check
+
+    Returns
+    -------
+    needs_inversion : bool
+        True if not finished yet
+    """
+    required_files = (
+        'grid' + os.sep + 'elem.dat',
+        'grid' + os.sep + 'elec.dat',
+        'exe' + os.sep + 'crtomo.cfg',
+    )
+
+    needs_inversion = True
+
+    for filename in required_files:
+        if not os.path.isfile(tomodir + os.sep + filename):
+            needs_inversion = False
+
+    # check for crmod OR modeling capabilities
+    if not os.path.isfile(tomodir + os.sep + 'mod' + os.sep + 'volt.dat'):
+        if not check_if_needs_modeling(tomodir):
+            needs_inversion = False
+
+    # check if finished
+    inv_ctr_file = tomodir + os.sep + 'inv' + os.sep + 'inv.ctr'
+    if os.path.isfile(inv_ctr_file):
+        inv_lines = open(inv_ctr_file, 'r').readlines()
+        if inv_lines[-1].startswith('***finished***'):
+            needs_inversion = False
+
+    return needs_inversion
+
+
+def find_unfinished_tomodirs(directory):
+    needs_modeling = []
+    needs_inversion = []
+    for root, dirs, files in os.walk(directory):
+        dirs.sort()
+        if is_tomodir(dirs):
+            logging.info('found tomodir: {}'.format(root))
+            if check_if_needs_modeling(root):
+                needs_modeling.append(root)
+            if check_if_needs_inversion(root):
+                needs_inversion.append(root)
+
+    return sorted(needs_modeling), sorted(needs_inversion)
+
+
+def _register_tomodir_for_processing(
+        tomodir_raw, sim_type, global_settings, cmd_options):
+    """
+    sim_type: inv|mod
+    """
+    tomodir = os.path.abspath(tomodir_raw)
+    # should be read from the configuration file
+    username = 'mweigand'
+
+    crh_file = tomodir + '.crh'
+    if os.path.isfile(crh_file):
+        if cmd_options.force_registration:
+            logger.info('Checking existing .crh file: {}'.format(tomodir_raw))
+            # load crh file
+            with open(crh_file, 'r') as fid:
+                settings_tmp = json.load(fid)
+            if len(
+                    settings_tmp.keys()
+                    ) == 1 and 'datetime_init' in settings_tmp:
+                logger.info('Simulation was not fully registered.')
+                logger.info('Deleting .crh file and registering again')
+                os.unlink(crh_file)
+                # now proceed
+            else:
+                return
+        else:
+            # do nothing - assume another process is working with this tomodir
+            return
+
+    tomodir_id = username + '_' + str(uuid.uuid4())
+    archive_file = os.path.abspath(tomodir_id + '.tar.xz')
+
+    crh_settings = {
+        'datetime_init': '{}'.format(
+            datetime.datetime.now(tz=datetime.timezone.utc)
+         ),
+    }
+    # touch the crh file as a crude locking mechanism
+    with open(crh_file, 'w') as fid:
+        json.dump(crh_settings, fid)
+
+    # create archive
+    pwdx = os.getcwd()
+    os.chdir(os.path.dirname(tomodir))
+
+    data_archive = io.BytesIO()
+    with tarfile.open(fileobj=data_archive, mode='w:xz') as tar:
+        tar.add(os.path.basename(tomodir), recursive=True)
+    os.chdir(pwdx)
+
+    # prepare data for simulation registration
+    crh_settings['source_computer'] = platform.node()
+    crh_settings['sim_type'] = sim_type
+    crh_settings['crh_file'] = os.path.abspath(crh_file)
+    crh_settings['username'] = username
+
+    print(
+        'Connecting to: {}'.format(
+            global_settings['general']['db_credentials']
+        )
+    )
+    engine = create_engine(
+        global_settings['general']['db_credentials'],
+        echo=False,
+        pool_size=1,
+        pool_recycle=600,
+    )
+    connection = engine.connect()
+    # create hash of final data
+    m = hashlib.sha256()
+    data_archive.seek(0)
+    m.update(data_archive.read())
+    sha256 = m.hexdigest()
+
+    # upload archive to database
+    query = ' '.join((
+        'insert into binary_data (filename, hash, data) values',
+        '(:filename, :file_hash, :bin_data) returning index;'
+    ))
+
+    data_archive.seek(0)
+    result = connection.execute(
+        text(query),
+        parameters={
+            'filename': os.path.basename(archive_file),
+            'file_hash': sha256,
+            'bin_data': data_archive.read(),
+        },
+    )
+    connection.commit()
+    assert result.rowcount == 1
+    file_id = result.fetchone()[0]
+    crh_settings['tomodir_unfinished_file'] = file_id
+
+    query = ' '.join((
+        'insert into inversions (',
+        'username,',
+        'datetime_init,',
+        'tomodir_unfinished_file,',
+        'source_computer,',
+        'sim_type,',
+        'crh_file',
+        ') values (',
+        ':username,',
+        ':datetime_init,',
+        ':tomodir_unfinished_file,',
+        ':source_computer,',
+        ':sim_type,',
+        ':crh_file',
+        ')',
+        'returning index;'
+    ))
+    result = connection.execute(
+        text(query),
+        parameters=crh_settings
+    )
+    print('rowcount pre commit', result.rowcount)
+    connection.commit()
+    print('rowcount post commit', result.rowcount)
+    assert result.rowcount == 1
+    sim_id = result.fetchone()[0]
+
+    crh_settings['sim_id'] = sim_id
+    # update crh file
+    with open(crh_file, 'w') as fid:
+        json.dump(crh_settings, fid, sort_keys=True, indent=4)
+
+    # now we are ready for processing
+    connection.execute(
+        text(
+            'update inversions set '
+            'ready_for_processing=\'t\' where index=:id;'
+        ),
+        parameters={'id': sim_id},
+    )
+    connection.commit()
+    # delete tomodir
+    shutil.rmtree(tomodir)
+    logger.info('Added {} to queue'.format(os.path.relpath(tomodir)))
+    connection.close()
+    engine.dispose()
+
+
+def main():
+
+    global_settings = get_config()
+    cmd_options = handle_cmd_options()
+
+    needs_modeling, needs_inversion = find_unfinished_tomodirs('.')
+    print('-' * 20)
+    print('modeling:', needs_modeling)
+    print('inversion:', needs_inversion)
+    print('-' * 20)
+    for directory in needs_modeling:
+        _register_tomodir_for_processing(
+            directory, 'mod', global_settings, cmd_options)
+    for directory in needs_inversion:
+        _register_tomodir_for_processing(
+            directory, 'inv', global_settings, cmd_options)
+
+
+if __name__ == '__main__':
+    main()

cr_hydra-0.2.0/src/crh_get_file.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+import sys
+import json
+import logging
+
+from sqlalchemy import create_engine
+
+from cr_hydra.settings import get_config
+
+logging.basicConfig(
+    level=logging.INFO,
+    format='{asctime} - {name} - %{levelname} - {message}',
+    style='{',
+)
+
+
+def main():
+    # settings: should be read from config file/cmd
+    global_settings = get_config()
+
+    engine = create_engine(
+        global_settings['general']['db_credentials'],
+        echo=False, pool_size=1, pool_recycle=3600,
+    )
+
+    crh_file = sys.argv[1]
+
+    with open(crh_file, 'r') as fid:
+        settings = json.load(fid)
+    print(crh_file)
+    print(settings)
+
+    file_id = settings['tomodir_unfinished_file']
+
+    result = engine.execute(
+        'select data from binary_data where index=%(file_id)s;',
+        file_id=file_id
+    )
+    assert result.rowcount == 1
+    data = result.fetchone()[0]
+
+    outfile = crh_file + '.tar.xz'
+    with open(outfile, 'wb') as fid:
+        fid.write(data)
+
+    # check if an inversion is present
+    sim_id = settings['sim_id']
+    result = engine.execute(
+        ' '.join((
+            'select tomodir_finished_file from inversions',
+            'where index=%(sim_id)s and status=\'finished\'',
+            'and downloaded=\'f\'',
+            ';'
+        )),
+        sim_id=sim_id
+    )
+    assert result.rowcount == 1
+    tomodir_finished_file = result.fetchone()[0]
+
+    result = engine.execute(
+        'select hash, data from binary_data where index=%(data_id)s;',
+        data_id=tomodir_finished_file
+    )
+    assert result.rowcount == 1
+    file_hash, binary_data = result.fetchone()
+
+    outfile = crh_file + '_finished.tar.xz'
+    with open(outfile, 'wb') as fid:
+        fid.write(binary_data)

cr_hydra-0.2.0/src/crh_retrieve.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python
+"""
+check the database for inversions
+"""
+import logging
+import os
+import hashlib
+import io
+import tarfile
+import json
+
+from sqlalchemy import create_engine, text
+import IPython
+
+from cr_hydra.settings import get_config
+
+IPython
+
+logging.basicConfig(
+    level=logging.INFO,
+    format='{asctime} - {name} - %{levelname} - {message}',
+    style='{',
+)
+logger = logging.getLogger(__name__)
+
+global_settings = get_config()
+
+engine = create_engine(
+    global_settings['general']['db_credentials'],
+    echo=False, pool_size=10, pool_recycle=60,
+)
+
+
+def _is_finished(sim_id, conn):
+    """Check if the simulation has been processed.
+
+    Ignore any rows already locked by other processes (i.e., concurrent runs
+    of crh_retrieve)
+
+    """
+    result = conn.execute(
+        text(
+            ' '.join((
+                'select tomodir_finished_file from inversions',
+                'where index=:sim_id and status=\'finished\'',
+                'and downloaded=\'f\'',
+                'for update',
+                'skip locked',
+                ';'
+            ))
+        ),
+        parameters={
+            'sim_id': sim_id,
+        }
+    )
+    if result.rowcount == 1:
+        return result.fetchone()[0]
+    else:
+        result.close()
+        return None
+
+
+def _check_and_retrieve(filename):
+    """For a given .crh file, check if the inversion results are ready to be
+    downloaded and extract the results
+    """
+    logger.info('Checking: {}'.format(filename))
+    sim_settings = json.load(open(filename, 'r'))
+    # ignore any simulation not successfully uploade
+    if 'sim_id' not in sim_settings:
+        return False
+
+    conn = engine.connect()
+    # transaction = conn.begin_nested()
+
+    final_data_id = _is_finished(sim_settings['sim_id'], conn)
+
+    tomodir_name = os.path.basename(filename)[:-4]
+    basedir = os.path.abspath(os.path.dirname(filename))
+
+    pwd = os.getcwd()
+
+    if final_data_id is not None:
+        # we got data
+        result = conn.execute(
+            text(
+                'select hash, data from binary_data where index=:data_id;'
+            ),
+            parameters={
+                'data_id': final_data_id,
+            },
+        )
+        assert result.rowcount == 1
+        file_hash, binary_data = result.fetchone()
+
+        # check hash
+        m = hashlib.sha256()
+        m.update(binary_data)
+        assert file_hash == m.hexdigest()
+
+        logger.info('retrieving and unpacking')
+        os.chdir(basedir)
+
+        # unpack
+        fid = io.BytesIO(bytes(binary_data))
+        with tarfile.open(fileobj=fid, mode='r') as tar:
+            assert os.path.abspath(os.getcwd()) == os.path.abspath(basedir)
+
+            # make sure there are only files in the archive that go into the
+            # tomodir
+            for entry in tar.getnames():
+                if entry == '.':
+                    continue
+                # strip leading './'
+                if entry.startswith('./'):
+                    entry = entry[2:]
+                if not entry.startswith(tomodir_name):
+                    raise Exception('Content should go into tomodir')
+            # now extract
+            tar.extractall('.')
+        os.chdir(pwd)
+        mark_sim_as_downloaded(sim_settings['sim_id'], conn)
+        os.unlink(filename)
+        # IPython.embed()
+    # transaction.commit()
+    conn.close()
+    engine.dispose()
+
+
+def mark_sim_as_downloaded(sim_id, conn):
+    # mark the simulation as downloaded and delete the files
+    result = conn.execute(
+        text(
+            'select tomodir_unfinished_file, tomodir_finished_file from ' +
+            'inversions where index=:sim_id;'
+        ),
+        parameters={
+            'sim_id': sim_id,
+        }
+    )
+    assert result.rowcount == 1
+    file_ids = list(result.fetchone())
+    result = conn.execute(
+        text(
+            'update inversions set ' +
+            'tomodir_unfinished_file=NULL, ' +
+            'tomodir_finished_file=NULL, ' +
+            'downloaded=\'t\' where index=:sim_id;'
+        ),
+        parameters={
+            'sim_id': sim_id,
+        },
+    )
+    conn.commit()
+    assert result.rowcount == 1
+    result.close()
+    # delete
+    result = conn.execute(
+        text('delete from binary_data where index in (:id1, :id2);'),
+        parameters={
+            'id1': file_ids[0],
+            'id2': file_ids[1],
+        },
+    )
+    conn.commit()
+    result.close()
+
+
+def main():
+    for root, dirs, files in os.walk('.'):
+        dirs.sort()
+        files.sort()
+        for filename in files:
+            if filename.endswith('.crh'):
+                _check_and_retrieve(root + os.sep + filename)
+                print(engine.pool.status())
+
+
+if __name__ == '__main__':
+    main()

cr_hydra-0.2.0/src/crh_worker.py

@@ -0,0 +1,352 @@
+#!/usr/bin/env python
+"""This cr_hydra worker program - it queries the database and does the actual
+processing of simulations (modelings/inversions).
+
+"""
+import shutil
+import glob
+import time
+import datetime
+import hashlib
+import io
+from multiprocessing import Process
+import logging
+import tarfile
+import platform
+import os
+import tempfile
+import subprocess
+from optparse import OptionParser
+
+import pandas as pd
+from sqlalchemy import create_engine
+import IPython
+
+from cr_hydra.settings import get_config
+
+IPython
+
+logging.basicConfig(
+    level=logging.INFO,
+    format='{asctime} - {name} - %{levelname} - {message}',
+    style='{',
+)
+
+# settings: should be read from config file/cmd
+global_settings = get_config()
+# [s]
+query_interval = 15
+# settings end
+
+engine = create_engine(
+    global_settings['general']['db_credentials'],
+    echo=False, pool_size=1, pool_recycle=3600,
+)
+node_name = platform.node()
+logging.info('Identifying as node: {}'.format(node_name))
+results = engine.execute(
+    ' '.join((
+        'select nice_level, nr_cpus, nr_threads from node_settings',
+        'where node_name = %(node_name)s;',
+    )),
+    node_name=node_name
+)
+if results.rowcount == 1:
+    logging.info('Found node settings in database for node: {}'.format(
+        node_name
+    ))
+    (nice_level, nr_cpus, number_of_worker_threads) = results.fetchone()
+    number_of_workers = int(nr_cpus / number_of_worker_threads)
+else:
+    # hopefully sane defaults
+    number_of_workers = 2
+    number_of_worker_threads = 2
+    nice_level = 20
+    results.close()
+
+
+class hydra_worker(Process):
+
+    def __init__(self, name, cmd_opts, **kwargs):
+        Process.__init__(self)
+        self.name = name
+        self.cmd_opts = cmd_opts
+        self.logger = logging.getLogger('crh_worker-' + name)
+        self.engine = create_engine(
+            global_settings['general']['db_credentials'],
+            echo=False, pool_size=10, pool_recycle=3600,
+        )
+
+    def run(self):
+        """
+
+        """
+        while(True):
+            if self._check_node_active():
+                did_sim = self._query_db_and_run_sim()
+                if did_sim:
+                    # try to get the next inversion immediately
+                    interval = 0
+                else:
+                    if self.cmd_opts.quit_after_empty:
+                        self.logger.info(
+                            'thread quitting due to empty db queue ' +
+                            '(--quit option)'
+                        )
+                        return
+                    interval = query_interval
+            else:
+                # sleep 30 seconds
+                self.logger.info(
+                    'Node was disabled by db, sleeping 30 seconds')
+                interval = 30
+            time.sleep(interval)
+
+    def _check_node_active(self):
+        """Check if this node should be active
+        """
+        result = self.engine.execute(
+            'select active from node_settings where node_name=%(node_name)s;',
+            node_name=node_name
+        )
+        if result.rowcount == 1:
+            is_active = result.fetchone()[0]
+            return is_active
+        else:
+            result.close()
+        # by default we assume that this node is active
+        return True
+
+    def _query_db_and_run_sim(self):
+        """
+        Returns
+        -------
+        did_sim: bool
+            We want to know if there were simulations to run, e.g., to adapt
+            the sleeping interval between db queries
+        """
+        self.logger.info('Querying DB for new simulations')
+        # query database for open inversions
+        self.conn = self.engine.connect()
+        transaction = self.conn.begin_nested()
+        # get the next free, unfinished simulation
+        r = self.conn.execute(
+            ' '.join((
+                'select index from inversions where ',
+                'ready_for_processing=\'t\' and status <> \'finished\'',
+                'and error=0',
+                'order by ',
+                'index asc for update skip locked limit 1;'
+            ))
+        )
+        if r.rowcount == 0:
+            r.close()
+            transaction.commit()
+            self.conn.close()
+            return False
+        # now the row is locked for us
+        job_id = r.fetchone()[0]
+        self.logger.info('job id: {}'.format(job_id))
+        query = ' '.join((
+            'select',
+            'tomodir_unfinished_file, sim_type,',
+            'ready_for_processing, status',
+            'from inversions',
+            'where index=%(index)s;'
+        ))
+        job_data = pd.read_sql_query(
+            query, self.conn, params={'index': job_id})
+
+        self.logger.info('Processing job id: {}'.format(job_id))
+
+        self._run_sim(
+            job_id,
+            int(job_data['tomodir_unfinished_file'].values.take(0)),
+            # job_data['hydra_location'].values.take(0),
+            # job_data['archive_hash'].values.take(0),
+            job_data['sim_type'].values.take(0)
+        )
+
+        # this actually updates the database
+        transaction.commit()
+        self.conn.close()
+        return True
+
+    def _run_sim(self, job_id, file_id, sim_type):
+        tempdir = tempfile.mkdtemp('_crhydra')
+        self.logger.info('tempdir: {}'.format(tempdir))
+
+        # get unfinished data and unpack
+        result = self.conn.execute(
+            'select hash, data from binary_data where index=%(file_id)s;',
+            file_id=file_id
+        )
+        file_hash, binary_data = result.fetchone()
+
+        # check hash
+        m = hashlib.sha256()
+        m.update(binary_data)
+        assert file_hash == m.hexdigest()
+
+        # unpack to tempir
+        fid = io.BytesIO(bytes(binary_data))
+        with tarfile.open(fileobj=fid, mode='r') as tar:
+            tar.extractall(path=tempdir)
+
+        # call td run
+        old_pwd = os.getcwd()
+        os.chdir(tempdir)
+        self.logger.info('Running inversion')
+        dt_inv_started = datetime.datetime.now(tz=datetime.timezone.utc)
+
+        inv_cpu = subprocess.check_output(
+            'cat /proc/cpuinfo | grep "model name" | head -1',
+            shell=True
+        )[13:].decode('utf-8').strip()
+        # 0: everything ok
+        error_code = 0
+        try:
+            subprocess.check_output(
+                'nice -n {} td_run_all_local -n 1 -t {}'.format(
+                    nice_level,
+                    number_of_worker_threads
+                ),
+                stderr=subprocess.STDOUT,
+                shell=True
+            )
+        except subprocess.CalledProcessError as e:
+            self.logger.error('There was an error executing CRTomo')
+            self.logger.error(e)
+            # it seems something went wrong while running CRTomo...
+            query = ' '.join((
+                'update inversions set',
+                'error = %(error_code)s,',
+                'error_msg=%(error_msg)s,',
+                'datetime_inversion_started=%(dt_started)s,',
+                'datetime_finished=%(dt_finished)s, ',
+                'inv_computer=%(node_name)s,',
+                'inv_cpu=%(inv_cpu)s',
+                'where index=%(job_id)s;',
+            ))
+            r = self.conn.execute(
+                query,
+                {
+                    'job_id': job_id,
+                    'error_code': 2,
+                    'node_name': node_name,
+                    'inv_cpu': inv_cpu,
+                    'dt_started': dt_inv_started,
+                    'dt_finished': datetime.datetime.now(
+                        tz=datetime.timezone.utc),
+                    'error_msg': e.cmd + '_' + e.output.decode('utf-8'),
+                }
+            )
+            return
+
+        # special case: an error.dat file appeared, indicating that the
+        # inversion itself broke. This is seen as a successful simulation
+        # from the view of cr_hydra
+        tomodir_name = glob.glob(tempdir + '/*')[0]
+        error_file = os.sep.join((
+            tomodir_name, 'exe', 'error.dat'
+        ))
+        if os.path.isfile(error_file):
+            self.logger.info('found error.dat file - setting error code to 1')
+            error_code = 1
+
+        dt_inv_ended = datetime.datetime.now(tz=datetime.timezone.utc)
+
+        self.logger.info('finished')
+        # create in-memory archive
+        finished_data = io.BytesIO()
+        with tarfile.open(fileobj=finished_data, mode='w:xz') as tar_out:
+            tar_out.add('.')
+
+        # create hash of final data
+        m = hashlib.sha256()
+        finished_data.seek(0)
+        m.update(finished_data.read())
+        hash_final = m.hexdigest()
+
+        # upload
+        finished_data.seek(0)
+        result = self.conn.execute(
+            'insert into binary_data (hash, data) values' +
+            '(%(data_hash)s, %(bin_data)s) returning index;',
+            data_hash=hash_final,
+            bin_data=finished_data.read()
+        )
+        finished_data_index = result.fetchone()[0]
+
+        os.chdir(old_pwd)
+        self.logger.info('updating to finished')
+        # mark as finished
+        query = ' '.join((
+            'update inversions set',
+            'status=\'finished\',',
+            'error=%(error_code)s,',
+            'tomodir_finished_file=%(finished_data)s,',
+            'datetime_inversion_started=%(dt_started)s,',
+            'datetime_finished=%(dt_finished)s, ',
+            'inv_computer=%(node_name)s,',
+            'inv_cpu=%(inv_cpu)s',
+            'where index=%(job_id)s;',
+        ))
+        r = self.conn.execute(
+            query,
+            {
+                'job_id': job_id,
+                'finished_data': finished_data_index,
+                'node_name': node_name,
+                'inv_cpu': inv_cpu,
+                'error_code': error_code,
+                'dt_started': dt_inv_started,
+                'dt_finished': dt_inv_ended,
+            }
+        )
+        assert r.rowcount == 1
+
+        # delete temporary directory
+        shutil.rmtree(tempdir)
+
+    def _get_hash_sha256(self, filename):
+        sha256 = subprocess.check_output(
+            'sha256sum "{}"'.format(filename), shell=True).decode('utf-8')
+        sha256 = sha256[0:sha256.find(' ')]
+        return sha256
+
+
+def handle_cmd_options():
+    parser = OptionParser()
+    # parser.add_option(
+    #     "-n", "--number",
+    #     dest="number_processes",
+    #     help="How many CRMod/CRTomo instances to start in parallel. " +
+    #     "Default: number of detected CPUs/2",
+    #     type='int',
+    #     default=None,
+    # )
+
+    parser.add_option(
+        "-q", "--quit",
+        dest="quit_after_empty",
+        help="Exit the program when no more simulations are queries in the DB",
+        action='store_true',
+    )
+
+    (options, args) = parser.parse_args()
+    return options
+
+
+def main():
+    cmd_opts = handle_cmd_options()
+
+    workers = []
+    for i in range(number_of_workers):
+        worker = hydra_worker('thread_{:02}'.format(i), cmd_opts)
+        workers.append(worker)
+        worker.start()
+
+
+if __name__ == '__main__':
+    main()