pycistem 0.6.1__cp310-cp310-manylinux_2_28_x86_64.whl

pycistem/database/__init__.py

@@ -0,0 +1,301 @@
+import contextlib
+import sqlite3
+from pathlib import Path
+from selectors import EpollSelector
+#from pycistem.core import Project
+from datetime import datetime
+
+import mrcfile
+import pandas as pd
+import starfile
+import mdocfile
+
+from typing import Union, List, Optional
+
+def create_project(
+        project_name: str,
+        output_dir: Path):
+    from pycistem.core import Project
+    Path(output_dir, project_name).mkdir(parents=True, exist_ok=True)
+    project = Project()
+    success = project.CreateNewProject(
+        Path(output_dir, project_name, f"{project_name}.db").as_posix(),
+        Path(output_dir, project_name).as_posix(),
+        project_name,
+    )
+    return Path(output_dir, project_name, f"{project_name}.db")
+    
+def import_movies(project_path: Union[str, Path], movies: Union[str, Path, List[Union[str, Path]]], pixelsize: float, exposure_dose: float, pattern="*.tif", gain: Union[bool, str, Path] = True, import_metadata: bool = True, bin_to_pixelsize: float = 1.0):
+    from pycistem.core import Project
+    if isinstance(project_path, Path):
+        project_path = project_path.as_posix()
+    project = Project()
+    project.OpenProjectFromFile(project_path)
+    num_movies = project.database.ReturnSingleLongFromSelectCommand(
+        "SELECT COUNT(*) FROM MOVIE_ASSETS;"
+    )
+    if isinstance(movies, str) or isinstance(movies, Path):
+        movies = Path(movies)
+        if movies.is_dir():
+            movies = list(movies.glob(pattern))
+        else:
+            movies = [movies]
+    movies = [Path(m).as_posix() for m in movies]
+    
+    movie_filenames = sorted(
+        movies
+    )
+    if isinstance(gain, bool) and gain:
+        gain_filenames = [list(Path(movie).parent.glob("*.dm4"))[0] for movie in movie_filenames]
+    metadata_entries = []
+    for i, movie in enumerate(movie_filenames):
+        metadata = mdocfile.read(movie + ".mdoc")
+        metadata_entries.append(metadata.iloc[0])
+        # Insert data in MOVIE_ASSETS_METADATA using sqlite3
+        project.database.ExecuteSQL(
+            f"INSERT INTO MOVIE_ASSETS_METADATA "
+            f"(MOVIE_ASSET_ID,"
+            f"METADATA_SOURCE,"
+            f"CONTENT_JSON,"
+            f"TILT_ANGLE,"
+            f"STAGE_POSITION_X,"
+            f"STAGE_POSITION_Y,"
+            f"STAGE_POSITION_Z,"
+            f"IMAGE_SHIFT_X,"
+            f"IMAGE_SHIFT_Y,"
+            f"EXPOSURE_DOSE,"
+            f"ACQUISITION_TIME)"
+            f"VALUES ({i+1},"
+            f"'serialem_frames_mdoc',"
+            f"'{metadata.iloc[0].to_json(default_handler=str)}',"
+            f" {metadata.loc[0,'TiltAngle']},"
+            f" {metadata.loc[0,'StagePosition'][0]},"
+            f" {metadata.loc[0,'StagePosition'][1]},"
+            f" {metadata.loc[0,'StageZ']},"
+            f" {metadata.loc[0,'ImageShift'][0]},"
+            f" {metadata.loc[0,'ImageShift'][1]},"
+            f" {metadata.loc[0,'ExposureDose']},"
+            f" {datetime_to_msdos(datetime.strptime(metadata.loc[0,'DateTime'],'%d-%b-%Y %H:%M:%S'))});"
+        )
+    project.database.BeginMovieAssetInsert()
+    for i, movie in enumerate(movie_filenames):
+        project.database.AddNextMovieAsset(
+            i + 1,
+            Path(movie).name,
+            movie,
+            0,
+            11520,
+            8184,
+            34,
+            300,
+            pixelsize,
+            exposure_dose,
+            2.7,
+            gain_filenames[i].as_posix(),
+            "",
+            bin_to_pixelsize / pixelsize,
+            0,
+            0,
+            1.0,
+            1.0,
+            0,
+            25,
+            1,
+        )
+    project.database.EndMovieAssetInsert()
+
+    project.database.Close(True)
+    return(len(movie_filenames))
+
+
+
+
+def get_image_info_from_db(project,image_asset=None, get_ctf=True):
+    with contextlib.closing(sqlite3.connect(project)) as con:
+        if image_asset is None:
+            df1 = pd.read_sql_query("SELECT IMAGE_ASSET_ID,MOVIE_ASSET_ID,IMAGE_ASSETS.FILENAME, MOVIE_ASSETS.FILENAME as movie_filename, MOVIE_ASSETS.GAIN_FILENAME, CTF_ESTIMATION_ID , ALIGNMENT_ID, IMAGE_ASSETS.PIXEL_SIZE as image_pixel_size, IMAGE_ASSETS.VOLTAGE, IMAGE_ASSETS.SPHERICAL_ABERRATION, MOVIE_ASSETS.PIXEL_SIZE as movie_pixel_size, IMAGE_ASSETS.X_SIZE, IMAGE_ASSETS.Y_SIZE FROM IMAGE_ASSETS LEFT OUTER JOIN MOVIE_ASSETS ON MOVIE_ASSETS.MOVIE_ASSET_ID == IMAGE_ASSETS.PARENT_MOVIE_ID", con)
+        else:
+            df1 = pd.read_sql_query(f"SELECT IMAGE_ASSET_ID,MOVIE_ASSET_ID,IMAGE_ASSETS.FILENAME, MOVIE_ASSETS.FILENAME as movie_filename, CTF_ESTIMATION_ID , ALIGNMENT_ID, IMAGE_ASSETS.PIXEL_SIZE as image_pixel_size, IMAGE_ASSETS.VOLTAGE, IMAGE_ASSETS.SPHERICAL_ABERRATION, MOVIE_ASSETS.PIXEL_SIZE as movie_pixel_size, IMAGE_ASSETS.X_SIZE, IMAGE_ASSETS.Y_SIZE FROM IMAGE_ASSETS LEFT OUTER JOIN MOVIE_ASSETS ON MOVIE_ASSETS.MOVIE_ASSET_ID == IMAGE_ASSETS.PARENT_MOVIE_ID WHERE IMAGE_ASSETS.IMAGE_ASSET_ID = {image_asset} ", con)
+        if not get_ctf:
+            return(df1)
+        df2 = pd.read_sql_query("SELECT CTF_ESTIMATION_ID,DEFOCUS1,DEFOCUS2,DEFOCUS_ANGLE,OUTPUT_DIAGNOSTIC_FILE,SCORE, DETECTED_RING_RESOLUTION, AMPLITUDE_CONTRAST FROM ESTIMATED_CTF_PARAMETERS",con)
+        selected_micrographs = pd.merge(df1,df2,on="CTF_ESTIMATION_ID")
+    if image_asset is None:
+        return(selected_micrographs)
+    else:
+        if selected_micrographs.shape[0] > 0:
+            return(selected_micrographs.iloc[0])
+        else:
+            return(None)
+
+def get_movie_info_from_db(project):
+    with contextlib.closing(sqlite3.connect(project)) as con:
+        df1 = pd.read_sql_query("SELECT * FROM MOVIE_ASSETS", con)
+
+    return(df1)
+
+
+def get_tm_info_from_db(project,image_asset,tm_id=None):
+    with contextlib.closing(sqlite3.connect(project)) as con:
+        if tm_id is None:
+            df1 = pd.read_sql_query(f"SELECT * FROM TEMPLATE_MATCH_LIST WHERE IMAGE_ASSET_ID={image_asset}",con)
+        else:
+            df1 = pd.read_sql_query(f"SELECT * FROM TEMPLATE_MATCH_LIST WHERE IMAGE_ASSET_ID={image_asset} AND TEMPLATE_MATCH_JOB_ID={tm_id}",con)
+
+    if tm_id is None:
+        return(df1)
+    else:
+        if df1.shape[0] > 0:
+            return(df1.iloc[0])
+        else:
+            return(None)
+
+
+
+def ensure_template_is_a_volume_asset(project: str, template_filename: str, pixel_size: float) -> int:
+    with contextlib.closing(sqlite3.connect(project)) as con:
+        df1 = pd.read_sql_query(f"SELECT * FROM VOLUME_ASSETS WHERE FILENAME='{template_filename}'",con)
+        if df1.shape[0] > 0:
+            return(df1.iloc[0]["VOLUME_ASSET_ID"])
+        else:
+            # Open using mrcfile and get dimensions
+            with mrcfile.open(template_filename) as mrc:
+                x_size = mrc.header.nx
+                y_size = mrc.header.ny
+                z_size = mrc.header.nz
+            #Get highest VOLUME_ASSET_ID
+            df2 = pd.read_sql_query("SELECT MAX(VOLUME_ASSET_ID) as max_id FROM VOLUME_ASSETS",con)
+            max_id = df2.iloc[0]["max_id"]
+            if max_id is None:
+                vol_id = 1
+            else:
+                vol_id = max_id + 1
+            con.execute(f"INSERT INTO VOLUME_ASSETS (VOLUME_ASSET_ID,NAME,FILENAME,PIXEL_SIZE,X_SIZE,Y_SIZE,Z_SIZE) VALUES ('{vol_id}','{Path(template_filename).stem}','{template_filename}','{pixel_size}','{x_size}','{y_size}','{z_size}')")
+            con.commit()
+            return(vol_id)
+
+
+def insert_tmpackage_into_db(project, name, path):
+    with contextlib.closing(sqlite3.connect(project)) as con:
+        con.execute(f"INSERT INTO TEMPLATE_MATCHES_PACKAGE_ASSETS (NAME,STARFILE_FILENAME) VALUES ('{name}','{path}')")
+        con.commit()
+
+def write_match_template_to_starfile(project, match_template_job_id, filename,overwrite=True, switch_phi_psi=False):
+
+    result_peaks = pd.DataFrame({
+        "cisTEMOriginalImageFilename": pd.Series(dtype="object"),
+        "cisTEMReference3DFilename": pd.Series(dtype="object"),
+        "cisTEMMicroscopeVoltagekV": pd.Series(dtype="float"),
+        "cisTEMMicroscopeCsMM": pd.Series(dtype="float"),
+        "cisTEMAmplitudeContrast": pd.Series(dtype="float"),
+        "cisTEMPhaseShift": pd.Series(dtype="float"),
+        "cisTEMDefocus1": pd.Series(dtype="float"),
+        "cisTEMDefocus2": pd.Series(dtype="float"),
+        "cisTEMDefocusAngle": pd.Series(dtype="float"),
+        "cisTEMPositionInStack": pd.Series(dtype="int"),
+        "cisTEMOriginalXPosition": pd.Series(dtype="float"),
+        "cisTEMOriginalYPosition": pd.Series(dtype="float"),
+        "cisTEMAnglePsi": pd.Series(dtype="float"),
+        "cisTEMAngleTheta": pd.Series(dtype="float"),
+        "cisTEMAnglePhi": pd.Series(dtype="float"),
+        "cisTEMPixelSize": pd.Series(dtype="float"),
+        "cisTEMScore": pd.Series(dtype="float"),
+        })
+
+    with contextlib.closing(sqlite3.connect(project)) as con:
+        df1 = pd.read_sql_query(f"SELECT * FROM TEMPLATE_MATCH_LIST WHERE TEMPLATE_MATCH_JOB_ID={match_template_job_id}",con)
+        for _i, tmres in df1.iterrows():
+            image =  pd.read_sql_query(f"SELECT FILENAME FROM IMAGE_ASSETS WHERE IMAGE_ASSET_ID = {tmres['IMAGE_ASSET_ID']}",con)
+            volume = pd.read_sql_query(f"SELECT FILENAME FROM VOLUME_ASSETS WHERE VOLUME_ASSET_ID = {tmres['REFERENCE_VOLUME_ASSET_ID']}",con)
+            df2 = pd.read_sql_query(f"SELECT * FROM TEMPLATE_MATCH_PEAK_LIST_{tmres['TEMPLATE_MATCH_ID']}",con)
+            for _j, peakres in df2.iterrows():
+                new_peak_series = pd.Series([
+                    "'"+image["FILENAME"].iloc[0]+"'",
+                    "'"+volume["FILENAME"].iloc[0]+"'",
+                    tmres["USED_VOLTAGE"],
+                    tmres["USED_SPHERICAL_ABERRATION"],
+                    tmres["USED_AMPLITUDE_CONTRAST"],
+                    tmres["USED_PHASE_SHIFT"],
+                    tmres["USED_DEFOCUS1"] + peakres["DEFOCUS"],
+                    tmres["USED_DEFOCUS2"] + peakres["DEFOCUS"],
+                    tmres["USED_DEFOCUS_ANGLE"],
+                    peakres["PEAK_NUMBER"],
+                    peakres["X_POSITION"],
+                    peakres["Y_POSITION"],
+                    peakres["PSI"],
+                    peakres["THETA"],
+                    peakres["PHI"],
+                    tmres["USED_PIXEL_SIZE"],
+                    peakres["PEAK_HEIGHT"]
+                    ], index = result_peaks.columns)
+                result_peaks.loc[len(result_peaks.index)] = new_peak_series
+
+    # Due to a bug in cisTEM in earlier matches phi and psi are switched in the
+    # database
+    if(switch_phi_psi):
+        temp = result_peaks["cisTEMAnglePhi"]
+        result_peaks["cisTEMAnglePhi"] = result_peaks["cisTEMAnglePsi"]
+        result_peaks["cisTEMAnglePsi"] = temp
+    starfile.write(result_peaks, filename=filename, overwrite=overwrite)
+
+
+def datetime_to_msdos(now):
+    msdos_date = ((now.year - 1980) << 9) | (now.month << 5) | now.day
+    msdos_time = (now.hour << 11) | (now.minute << 5) | (now.second // 2)
+
+    msdos_datetime = (msdos_date << 16) | msdos_time
+
+    return msdos_datetime
+
+def create_peak_lists(con, id: int):
+    cur = con.cursor()
+    cur.execute(f"CREATE TABLE TEMPLATE_MATCH_PEAK_LIST_{id} (PEAK_NUMBER INTEGER PRIMARY KEY AUTOINCREMENT, X_POSITION REAL, Y_POSITION REAL, PSI REAL, THETA REAL, PHI REAL, DEFOCUS REAL, PIXEL_SIZE REAL, PEAK_HEIGHT REAL)")
+    con.commit()
+    cur.execute(f"CREATE TABLE TEMPLATE_MATCH_PEAK_CHANGE_LIST_{id} (PEAK_NUMBER INTEGER PRIMARY KEY AUTOINCREMENT, X_POSITION REAL, Y_POSITION REAL, PSI REAL, THETA REAL, PHI REAL, DEFOCUS REAL, PIXEL_SIZE REAL, PEAK_HEIGHT REAL, ORIGINAL_PEAK_NUMBER REAL, NEW_PEAK_NUMBER REAL)")
+    con.commit()
+
+def get_max_match_template_job_id(database):
+    with contextlib.closing(sqlite3.connect(database)) as con:
+        cur = con.cursor()
+        cur.execute("SELECT MAX(TEMPLATE_MATCH_JOB_ID) FROM TEMPLATE_MATCH_LIST")
+        max_match_template_job_id = cur.fetchone()[0]
+    return(max_match_template_job_id)
+
+def get_already_processed_images(database, match_template_job_id):
+    with contextlib.closing(sqlite3.connect(database)) as con:
+        already_processed_images = pd.read_sql_query(f"SELECT IMAGE_ASSET_ID FROM TEMPLATE_MATCH_LIST WHERE TEMPLATE_MATCH_JOB_ID = {match_template_job_id}",con)
+    return(already_processed_images)
+
+def get_num_already_processed_images(database, match_template_job_id):
+    with contextlib.closing(sqlite3.connect(database)) as con:
+        cur = con.cursor()
+        cur.execute(f"SELECT COUNT(1) FROM TEMPLATE_MATCH_LIST WHERE TEMPLATE_MATCH_JOB_ID = {match_template_job_id}")
+        num_already_processed_images = cur.fetchone()[0]
+    return(num_already_processed_images)
+
+def get_num_matches(database, match_template_job_id):
+    with contextlib.closing(sqlite3.connect(database)) as con:
+        cur = con.cursor()
+        cur.execute(f"SELECT TEMPLATE_MATCH_ID FROM TEMPLATE_MATCH_LIST WHERE TEMPLATE_MATCH_JOB_ID = {match_template_job_id}")
+        match_template_ids = cur.fetchall()
+        total = 0
+        for mti in match_template_ids:
+            cur.execute(f"SELECT MAX(RowId) FROM TEMPLATE_MATCH_PEAK_LIST_{mti[0]}")
+            num_matches = cur.fetchone()[0]
+            if num_matches is not None:
+                total += num_matches
+    return(total)
+
+def get_num_images(database):
+    with contextlib.closing(sqlite3.connect(database)) as con:
+        cur = con.cursor()
+        cur.execute("SELECT COUNT(1) FROM IMAGE_ASSETS WHERE CTF_ESTIMATION_ID IS NOT -1")
+        num_images = cur.fetchone()[0]
+    return(num_images)
+
+def get_num_movies(database):
+    with contextlib.closing(sqlite3.connect(database)) as con:
+        cur = con.cursor()
+        cur.execute("SELECT COUNT(1) FROM MOVIE_ASSETS")
+        num_movies = cur.fetchone()[0]
+    return(num_movies)

pycistem/programs/__init__.py

@@ -0,0 +1,10 @@
+from pycistem.programs.apply_ctf import *
+from pycistem.programs.cistem_program import *
+from pycistem.programs.refine_template import *
+from pycistem.programs.refine_template_niko import *
+from pycistem.programs.refine_template_dev import *
+from pycistem.programs.refine_ctf import *
+from pycistem.programs.estimate_beamtilt import *
+from pycistem.programs.run_profile import *
+from pycistem.programs.reconstruct3d import *
+from pycistem.programs.resample import *

pycistem/programs/_cistem_constants.py

@@ -0,0 +1,23 @@
+
+socket_please_identify = b"JcFG>&P.RuC9,>za"
+socket_sending_identification  = b"gC2CeZWNb2GPv5qh"
+socket_you_are_connected  = b"J82zjSwYY^-!bF>4"
+socket_send_job_details  = b"gr<V>ThBp6w9fzLg"
+socket_sending_job_package  = b"'8ujA!Lup%PR*!hG"
+socket_you_are_the_master  = b"eVmYc.3!g}}cZZs"
+socket_you_are_a_worker  = b"U6u*:z6}W+7nV2g'"
+socket_send_next_job  = b"z7PnJh=x;[b#f/6L"
+socket_time_to_die  = b")[czL7$#Sg/d4-*K"
+socket_ready_to_send_single_job  = b"-TDv(X*kY.:d`D5:"
+socket_i_have_an_error  = b"8TU.cDc3jr,rb[SN"
+socket_i_have_info  = b"+5nxvY@zt.!_R#Vn"
+socket_job_finished  = b"jNA[3!VdLdkb$LwM"
+socket_number_of_connections  = b"Uu6tsQ,z}M''T`7f"
+socket_all_jobs_finished  = b"aL)yaH[$3s;9Ymk6"
+socket_job_result  = b"3F6E_.``L6YC^q[U"
+socket_job_result_queue  = b"^}`@pF9m;{m9k=$F"
+socket_result_with_image_to_write  = b"=z4-Y8Ge?vEjh`H^"
+socket_program_defined_result  = b"e}w<S9hm<3L6Dr+V"
+socket_send_thread_timing  = b"Kq04etrq1fO2QV4d"
+socket_template_match_result_ready  = b"EP927e$*cQ^egWq'"
+socket_remote_control              = b"9^$[9^$[9^$[9^$["

pycistem/programs/apply_ctf.py

@@ -0,0 +1,59 @@
+import asyncio
+from dataclasses import dataclass
+from typing import Union
+
+from pycistem.database import get_image_info_from_db
+from pycistem.programs import cistem_program
+from pycistem.programs._cistem_constants import socket_send_next_job
+
+
+@dataclass
+class ApplyCtfParameters:
+    input_filename: str
+    output_filename: str
+    pixel_size: float = 1.0
+    acceleration_voltage: float = 300.0
+    spherical_aberration: float = 2.7
+    amplitude_contrast: float = 0.07
+    defocus_1: float = 20000
+    defocus_2: float = 20000
+    astigmatism_angle: float = 0.0
+    additional_phase_shift: float = 0.0
+    input_ctf_values_from_text_file: bool = False
+    text_filename: str = ""
+    phase_flip_only: bool = False
+    apply_wiener_filter: bool = False
+    wiener_filter_falloff_frequency: float = 100.0
+    wiener_filter_falloff_fudge_factor: float = 1.0
+    wiener_filter_scale_fudge_factor: float = 1.0
+    wiener_filter_high_pass_radius: float = 200.0
+    maintain_image_contrast: bool = True
+
+def parameters_from_database(database, image_asset_id, output_filename, **kwargs):
+    image_info = get_image_info_from_db(database, image_asset=image_asset_id)
+    par = ApplyCtfParameters(input_filename=image_info["FILENAME"],
+                             output_filename=output_filename,
+                             pixel_size=image_info["image_pixel_size"],
+                             acceleration_voltage=image_info["VOLTAGE"],
+                             spherical_aberration=image_info["SPHERICAL_ABERRATION"],
+                             amplitude_contrast=image_info["AMPLITUDE_CONTRAST"],
+                             defocus_1=image_info["DEFOCUS1"],
+                             defocus_2=image_info["DEFOCUS2"],
+                             astigmatism_angle=image_info["DEFOCUS_ANGLE"],
+                             **kwargs)
+    return(par)
+
+def run(parameters: Union[ApplyCtfParameters,list[ApplyCtfParameters]],**kwargs):
+    if not isinstance(parameters, list):
+        parameters = [parameters]
+
+
+    asyncio.run(cistem_program.run("applyctf",parameters,**kwargs))
+
+async def run_async(parameters: Union[ApplyCtfParameters,list[ApplyCtfParameters]],**kwargs):
+    if not isinstance(parameters, list):
+        parameters = [parameters]
+
+
+    await cistem_program.run("applyctf",parameters,**kwargs)
+

pycistem/programs/cistem_program.py

@@ -0,0 +1,277 @@
+import asyncio
+import logging
+import secrets
+import socket
+import string
+import struct
+import subprocess
+import time
+from dataclasses import astuple, fields
+from pathlib import Path
+from time import sleep
+
+import psutil
+
+
+def get_ip_addresses(family):
+    for interface, snics in psutil.net_if_addrs().items():
+        for snic in snics:
+            if snic.family == family:
+                yield (interface, snic.address)
+
+HOST = ",".join([a[1] for a in get_ip_addresses(socket.AF_INET)])
+
+
+from pycistem.config import config
+
+log = logging.getLogger(__name__)
+log.addHandler(logging.NullHandler())
+
+
+from pycistem.programs._cistem_constants import *
+
+
+def _encode_parameters(parameters):
+    # creates a cisTEM compatible buffer from the parameters class
+    buffer = b""
+    buffer += int(1).to_bytes(4,"little")
+    buffer += len(fields(parameters)).to_bytes(4,"little")
+    parameterstuple = astuple(parameters)
+    for i,argument in enumerate(fields(parameters)):
+        if argument.type == float:
+            buffer += int(3).to_bytes(1,"little")
+            buffer += struct.pack("<f", parameterstuple[i])
+        if argument.type == bool:
+            buffer += int(4).to_bytes(1,"little")
+            buffer += parameterstuple[i].to_bytes(1,"little")
+        if argument.type == str:
+            buffer += int(1).to_bytes(1,"little")
+            bb = parameterstuple[i].encode("utf-8")
+            buffer += len(bb).to_bytes(4,"little")
+            buffer += bb
+        if argument.type == int:
+            buffer += int(2).to_bytes(1,"little")
+            buffer += struct.pack("<i", parameterstuple[i])
+    return buffer
+
+
+async def handle_manager(reader, writer, identity, port):
+    # Handles initial connection from the executable and directs them to the leader
+    addr = writer.get_extra_info("peername")
+    #logger.info(f"{addr} connected to manager")
+    writer.write(socket_please_identify)
+    await writer.drain()
+    data = await reader.readexactly(16)
+    if data != socket_sending_identification:
+        log.error(f"{addr!r} {data} is not {socket_sending_identification}")
+        writer.close()
+        return
+    data = await reader.readexactly(16)
+    message= data.decode()
+    if message != identity:
+        log.error(f"{addr!r} {message} is not {identity}: wrong process connected")
+        writer.close()
+        return
+    writer.write(socket_you_are_a_worker)
+    host = HOST.encode("utf-8")
+    writer.write(len(host).to_bytes(4,"little"))
+    writer.write(host)
+    port = str(port).encode("utf-8")
+    writer.write(len(port).to_bytes(4,"little"))
+    writer.write(port)
+    await writer.drain()
+    time.sleep(1)
+    #logger.info(f"{addr} connected")
+    writer.close()
+
+async def handle_leader(reader, writer, buffers, signal_handlers,results):
+    log_str = ""
+    # Handles connections from the executable asking for work
+
+    addr = writer.get_extra_info("peername")
+
+    writer.write(socket_you_are_connected)
+    await writer.drain()
+    data = await reader.readexactly(16)
+    if data != socket_send_next_job:
+        log.error(f"{addr!r} did not request next job, instead sent {data}")
+        writer.close()
+        return
+    data = await reader.readexactly(8)
+    #logger.info(f"{addr} sent {data} as dummy result")
+    while len(buffers) > 0:
+        parameter_index, buffer = buffers.pop(0)
+        log.debug(f"Working on parameter set {parameter_index}")
+        writer.write(socket_ready_to_send_single_job)
+        writer.write(len(buffer).to_bytes(8,"little"))
+        writer.write(buffer)
+        await writer.drain()
+
+        # check length of signal_handlers
+        if len(signal_handlers) > 0:
+            cont = True
+            while cont:
+                data = await reader.readexactly(16)
+                # log.debug(f"In no sig {addr} sent {data}")
+                result = None
+                if data != socket_job_result_queue and data != socket_i_have_info:
+                    cont = False
+                else:
+                    await signal_handlers[data](reader,writer,log)
+                    log_str += "J"
+                    continue
+                if data in signal_handlers:
+                    #logger.info(f"{addr} sent {data} and I know what to do with it")
+                    result = await signal_handlers[data](reader,writer,log)
+                    results.append((parameter_index,result))
+                    log_str += "R"
+                else:
+                    log.error(f"{addr} sent {data} and I don't know what to do with it, really {log_str}")
+
+                    #logger.error(f"{buffer}")
+                    break
+        cont = True
+        while cont:
+            if socket_send_next_job not in signal_handlers:
+                data = await reader.readexactly(16)
+                # log.debug(f"In cont {addr} sent {data}")
+                if data != socket_send_next_job:
+                    if data == socket_job_result_queue:
+                        res = await signal_handlers[data](reader,writer,log)
+                        log_str += "j"
+                        continue
+                    else:
+                        log.error(f"{addr!r} did not request next job, instead sent {data}")
+                data = await reader.readexactly(8)
+                log_str += "N"
+                cont = False
+                #logger.info(f"{addr} sent {data} after requesting next results")
+            else:
+                cont = False
+    log.debug(f"{addr} finished, sending time to die")
+    writer.write(socket_time_to_die)
+    await writer.drain()
+    data = await reader.read(16)
+    #logger.info(f"{addr} sent {data}")
+    data = await reader.read(16)
+    #logger.info(f"{addr} sent {data}")
+    data = await reader.read(16)
+    #logger.info(f"{addr} sent {data}")
+    
+    writer.close()
+
+
+async def run(executable: str,parameters,signal_handlers={},num_procs=1,num_threads=1, cmd_prefix="", cmd_suffix="", save_output=False, save_output_path="",sleep_time=0.1):
+    results = []
+    buffers = []
+    # Set HOST to curretn ip address
+    #'127.0.0.1'  # Standard loopback interface address (localhost)
+
+    # Create secret to identify workers
+    alphabet = string.ascii_letters + string.digits
+    identity = "".join(secrets.choice(alphabet) for i in range(16))
+    buffers = [(i,_encode_parameters(parameter)) for i, parameter in enumerate(parameters)]
+    log.debug(f"Secret is {identity}")
+
+    # Start the leader
+    start_port = 3000
+    leader_started = False
+    while not leader_started:
+        try:
+            server_leader = await asyncio.start_server(
+                lambda r,w : handle_leader(r,w, buffers,signal_handlers,results), "", start_port, family=socket.AF_INET)
+            leader_started = True
+            start_port += 1
+        except OSError:
+            log.debug(f"Port {start_port} already in use, trying next port")
+            start_port += 1
+            if start_port > 4000:
+                msg = "No ports available"
+                raise OSError(msg)
+    port_leader = server_leader.sockets[0].getsockname()[1]
+    addrs = ", ".join(str(sock.getsockname()) for sock in server_leader.sockets)
+    log.debug(f"Serving leader on {addrs}")
+
+    # Start the manager
+    manager_started = False
+    while not manager_started:
+        try:
+            server_manager = await asyncio.start_server(
+                lambda r,w : handle_manager(r,w,identity,port_leader), "",start_port, family=socket.AF_INET)
+            manager_started = True
+        except OSError:
+            log.debug(f"Port {start_port} already in use, trying next port")
+            start_port += 1
+            if start_port > 4000:
+                msg = "No ports available"
+                raise OSError(msg)
+    port_manager = server_manager.sockets[0].getsockname()[1]
+    addrs = ", ".join(str(sock.getsockname()) for sock in server_manager.sockets)
+    log.debug(f"Serving manager on {addrs}")
+
+
+    # Starting workers
+    cmd = str(Path(config["CISTEM_PATH"]) / executable)
+    cmd += f" {HOST} {port_manager} {identity} {num_threads}"
+
+    # Test if cmd_prefix is iterable
+    if type(num_procs) == int and type(cmd_prefix) == str:
+        cmd_prefix = [cmd_prefix for i in range(num_procs)]
+
+    if type(num_procs) == int and type(cmd_suffix) == str:
+        cmd_suffix = [cmd_suffix for i in range(num_procs)]
+
+
+
+    launch_futures = []
+    if type(num_procs) == int:
+        tasks = [cmd_prefix[i] + cmd +cmd_suffix[i] for i in range(num_procs)]
+    elif type(num_procs) == RunProfile:
+        tasks = []
+        num_procs.SubstituteExecutableName(executable)
+        for rc in num_procs.run_commands:
+            for _i in range(rc.number_of_copies):
+                tasks.append(rc.command_to_run)
+
+    for task in tasks:
+        launch_futures.append(await asyncio.create_subprocess_shell(
+        task,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.STDOUT))
+        sleep(sleep_time)
+
+    logging.debug(f"Launched {num_procs} processes")
+
+
+    #if save_output:
+    #    while True:
+    #        for i, future in enumerate(launch_futures):
+    #            line = await future.stdout.readline()
+    #            if line:
+    #                print(f"Pro {i}:")
+    #                print(line)
+
+
+    result_futures = [
+            future.communicate()
+            for future in launch_futures
+        ]
+    
+
+    try:
+        proc_results = await asyncio.gather(*result_futures, return_exceptions=False)
+    except Exception as ex:
+        print("Caught error executing task", ex)
+        raise
+    finally:
+        if save_output:
+            for i, result in enumerate(proc_results):
+                with open(save_output_path + f"_{i}.txt", "w") as f:
+                    f.write(result[0].decode("utf-8"))
+                if result[1] and len(result[1]) > 0:
+                    with open(save_output_path + f"_{i}_error.txt", "w") as f:
+                        f.write(result[1].decode("utf-8"))
+    server_leader.close()
+    server_manager.close()
+    return(results)
+