pycistem 0.6.1__cp310-cp310-manylinux_2_28_x86_64.whl

pycistem/programs/ctffind.py

@@ -0,0 +1,254 @@
+import asyncio
+import datetime
+import sqlite3
+import struct
+from dataclasses import dataclass
+from pathlib import Path
+from typing import List, Union
+
+import mrcfile
+import pandas as pd
+
+from pycistem.database import datetime_to_msdos, get_image_info_from_db
+from pycistem.programs import cistem_program
+from pycistem.programs._cistem_constants import socket_send_next_job
+
+
+@dataclass
+class CtffindParameters:
+    input_filename : str
+    input_is_a_movie: bool = True
+    number_of_frames_to_average : int = 3
+    output_diagnostic_filename : str = "diagnostic.mrc"
+    pixel_size_of_input_image : float = 1.0
+    acceleration_voltage : float = 300.0
+    spherical_aberration : float = 2.7
+    amplitude_contrast: float = 0.07
+    box_size: int = 512
+    minimum_resolution: float = 30.0
+    maximum_resolution: float = 5.0
+    minimum_defocus: float = 3000.0
+    maximum_defocus: float = 30000.0
+    defocus_search_step: float = 100.0
+    slower_search: bool = False
+    astigmatism_tolerance: float = -100.0
+    find_additional_phase_shift: bool = False
+    minimum_additional_phase_shift: float = 0.0
+    maximum_additional_phase_shift: float = 0.0
+    additional_phase_shift_search_step: float = 0.0
+    astigmatism_is_known: bool = False
+    known_astigmatism: float = 0.0
+    known_astigmatism_angle: float = 0.0
+    resample_if_pixel_too_small: bool = True
+    movie_is_gain_corrected: bool = False
+    gain_filename: str = "gain.mrc"
+    movie_is_dark_corrected: bool = True
+    dark_filename: str = "dark.mrc"
+    correct_movie_mag_distortion: bool = False
+    movie_mag_distortion_angle: float = 0.0
+    movie_mag_distortion_major_scale: float = 1.0
+    movie_mag_distortion_minor_scale: float = 1.0
+    defocus_is_known: bool = False
+    known_defocus_1: float = 0.0
+    known_defocus_2: float = 0.0
+    known_phase_shift: float = 0.0
+    determine_tilt: bool = False
+    desired_number_of_threads: int = 1
+    eer_frames_per_image: int = 0
+    eer_super_res_factor: int = 1
+    filter_lowres_signal: bool = True
+    fit_nodes: bool = False
+    fit_nodes_1D_brute_force: bool = True
+    fit_nodes_2D_refine: bool = True
+    fit_nodes_low_resolution_limit: float = 30.0
+    fit_nodes_high_resolution_limit: float = 4.0
+    target_pixel_size_after_resampling: float = 1.4
+    fit_nodes_use_rounded_square: bool = False
+    fit_nodes_downweight_nodes: bool = False
+
+def parameters_from_database(database, decolace=False, **kwargs):
+    image_info = get_image_info_from_db(database,get_ctf=False)
+    ProjectDirectory = Path(database).parent
+    par = [CtffindParameters(
+        input_filename = image["movie_filename"],
+        pixel_size_of_input_image = image["movie_pixel_size"],
+        gain_filename=image["GAIN_FILENAME"],
+        output_diagnostic_filename=(ProjectDirectory / "Assets" / "CTF" / f"{Path(image['FILENAME']).stem}_{image['MOVIE_ASSET_ID']}_auto.mrc").as_posix(),
+        fit_nodes=decolace
+    ) for i,image in image_info.iterrows()]
+    return((par, image_info))
+
+
+def write_results_to_database(database,  parameters: list[CtffindParameters], results, image_info):
+    conn = sqlite3.connect(database, isolation_level=None)
+    cur = conn.cursor()
+    results = sorted(results, key=lambda x: x["parameter_index"])
+    ESTIMATED_CTF_PARAMETERS_LIST = []
+
+    max_ctf_estimation_id= cur.execute("SELECT MAX(CTF_ESTIMATION_ID) FROM ESTIMATED_CTF_PARAMETERS").fetchone()[0]
+    if max_ctf_estimation_id is None:
+        max_ctf_estimation_id = 0
+    ctf_estimation_job_id= cur.execute("SELECT MAX(CTF_ESTIMATION_JOB_ID) FROM ESTIMATED_CTF_PARAMETERS").fetchone()[0]
+    if ctf_estimation_job_id is None:
+        ctf_estimation_job_id = 1
+    else:
+        ctf_estimation_job_id += 1
+
+    for result in results:
+
+        ESTIMATED_CTF_PARAMETERS_LIST.append({
+            "CTF_ESTIMATION_ID": max_ctf_estimation_id + 1,
+            "CTF_ESTIMATION_JOB_ID": ctf_estimation_job_id,
+            "DATETIME_OF_RUN": datetime_to_msdos(datetime.datetime.now()),
+            "IMAGE_ASSET_ID": image_info.loc[result["parameter_index"]]["IMAGE_ASSET_ID"],
+            "ESTIMATED_ON_MOVIE_FRAMES": True,
+            "VOLTAGE": parameters[result["parameter_index"]].acceleration_voltage,
+            "SPHERICAL_ABERRATION": parameters[result["parameter_index"]].spherical_aberration,
+            "PIXEL_SIZE": parameters[result["parameter_index"]].pixel_size_of_input_image,
+            "AMPLITUDE_CONTRAST": parameters[result["parameter_index"]].amplitude_contrast,
+            "BOX_SIZE": parameters[result["parameter_index"]].box_size,
+            "MIN_RESOLUTION": parameters[result["parameter_index"]].minimum_resolution,
+            "MAX_RESOLUTION": parameters[result["parameter_index"]].maximum_resolution,
+            "MIN_DEFOCUS": parameters[result["parameter_index"]].minimum_defocus,
+            "MAX_DEFOCUS": parameters[result["parameter_index"]].maximum_defocus,
+            "DEFOCUS_STEP": parameters[result["parameter_index"]].defocus_search_step,
+            "RESTRAIN_ASTIGMATISM": parameters[result["parameter_index"]].astigmatism_tolerance > 0,
+            "TOLERATED_ASTIGMATISM": parameters[result["parameter_index"]].astigmatism_tolerance,
+            "FIND_ADDITIONAL_PHASE_SHIFT": parameters[result["parameter_index"]].find_additional_phase_shift,
+            "MIN_PHASE_SHIFT": parameters[result["parameter_index"]].minimum_additional_phase_shift,
+            "MAX_PHASE_SHIFT": parameters[result["parameter_index"]].maximum_additional_phase_shift,
+            "PHASE_SHIFT_STEP": parameters[result["parameter_index"]].additional_phase_shift_search_step,
+            "DEFOCUS1": result["defocus1"],
+            "DEFOCUS2": result["defocus2"],
+            "DEFOCUS_ANGLE": result["astigmatism_angle"],
+            "ADDITIONAL_PHASE_SHIFT": result["phase_shift"],
+            "SCORE": result["score"],
+            "DETECTED_RING_RESOLUTION": result["fit_resolution"],
+            "DETECTED_ALIAS_RESOLUTION": result["aliasing_resolution"],
+            "OUTPUT_DIAGNOSTIC_FILE": parameters[result["parameter_index"]].output_diagnostic_filename,
+            "NUMBER_OF_FRAMES_AVERAGED": parameters[result["parameter_index"]].number_of_frames_to_average,
+            "LARGE_ASTIGMATISM_EXPECTED": parameters[result["parameter_index"]].slower_search,
+            "ICINESS": result["iciness"],
+            "TILT_ANGLE": result["tilt_angle"],
+            "TILT_AXIS": result["tilt_axis"],
+            "SAMPLE_THICKNESS": result["sample_thickness"],
+            "SAMPLE_THICKNESS_JSON": "",
+            "DETERMINE_TILT": parameters[result["parameter_index"]].determine_tilt,
+            "FIT_NODES": parameters[result["parameter_index"]].fit_nodes,
+            "FIT_NODES_1D": parameters[result["parameter_index"]].fit_nodes_1D_brute_force,
+            "FIT_NODES_2D": parameters[result["parameter_index"]].fit_nodes_2D_refine,
+            "FIT_NODES_LOW_LIMIT": parameters[result["parameter_index"]].fit_nodes_low_resolution_limit,
+            "FIT_NODES_HIGH_LIMIT": parameters[result["parameter_index"]].fit_nodes_high_resolution_limit,
+             "FIT_NODES_ROUNDED_SQUARE": parameters[result["parameter_index"]].fit_nodes_use_rounded_square,
+             "FIT_NODES_DOWNWEIGHT_NODES": parameters[result["parameter_index"]].fit_nodes_downweight_nodes,
+             "RESAMPLE_IF_NESCESSARY": parameters[result["parameter_index"]].resample_if_pixel_too_small,
+             "TARGET_PIXEL_SIZE": parameters[result["parameter_index"]].target_pixel_size_after_resampling
+        })
+        #print("UPDATE IMAGE_ASSETS SET CTF_ESTIMATION_ID = ? WHERE IMAGE_ASSET_ID = ?",  (max_ctf_estimation_id + 1, image_info.loc[result["parameter_index"]]["IMAGE_ASSET_ID"]))
+        #cur.execute("UPDATE IMAGE_ASSETS SET CTF_ESTIMATION_ID = ? WHERE IMAGE_ASSET_ID = ?",  (max_ctf_estimation_id + 1, image_info.loc[result["parameter_index"]]["IMAGE_ASSET_ID"]))
+        max_ctf_estimation_id += 1
+
+    ESTIMATED_CTF_PARAMETERS_LIST = pd.DataFrame(ESTIMATED_CTF_PARAMETERS_LIST)
+    ESTIMATED_CTF_PARAMETERS_LIST.to_sql("ESTIMATED_CTF_PARAMETERS", conn, if_exists="append", index=False)
+    
+    cur = conn.cursor()
+    cur.executemany("UPDATE IMAGE_ASSETS SET CTF_ESTIMATION_ID = ? WHERE IMAGE_ASSET_ID = ?",[
+        (row["CTF_ESTIMATION_ID"], row["IMAGE_ASSET_ID"])
+        for i, row in ESTIMATED_CTF_PARAMETERS_LIST.iterrows()
+    ])
+    conn.commit()
+    # Update CTF_ESTIMATION_ID in IMAGE_ASSETS table
+
+    conn.close()
+
+async def handle_results(reader, writer, logger):
+    #logger.info("Handling results")
+    await reader.read(4)
+    length = await reader.readexactly(4)
+    print(f"CTFFIND results length {length}")
+    number_of_bytes = int.from_bytes(length, byteorder="little")
+    results = await reader.readexactly(number_of_bytes*4)
+    print(f"CTFFIND results {results}")
+    return(results)
+
+signal_handlers = {
+    socket_send_next_job : handle_results
+}
+
+def run(parameters: Union[CtffindParameters,list[CtffindParameters]],**kwargs):
+
+    if not isinstance(parameters, list):
+        parameters = [parameters]
+
+    byte_results = asyncio.run(cistem_program.run("ctffind", parameters, signal_handlers=signal_handlers,**kwargs))
+    result_ctf = []
+
+    for parameter_index,byte_result in byte_results:
+        defocus1 = struct.unpack("<f",byte_result[0:4])[0]
+        defocus2 = struct.unpack("<f",byte_result[4:8])[0]
+        astigmatism_angle = struct.unpack("<f",byte_result[8:12])[0]
+        phase_shift = struct.unpack("<f",byte_result[12:16])[0]
+        score = struct.unpack("<f",byte_result[16:20])[0]
+        fit_resolution = struct.unpack("<f",byte_result[20:24])[0]
+        aliasing_resolution = struct.unpack("<f",byte_result[24:28])[0]
+        iciness = struct.unpack("<f",byte_result[28:32])[0]
+        tilt_angle = struct.unpack("<f",byte_result[32:36])[0]
+        tilt_axis = struct.unpack("<f",byte_result[36:40])[0]
+        sample_thickness = struct.unpack("<f",byte_result[40:44])[0]
+
+        result_ctf.append({
+            "parameter_index" : parameter_index,
+            "defocus1" : defocus1,
+            "defocus2" : defocus2,
+            "astigmatism_angle" : astigmatism_angle,
+            "phase_shift" : phase_shift,
+            "score" : score,
+            "fit_resolution" : fit_resolution,
+            "aliasing_resolution" : aliasing_resolution,
+            "iciness" : iciness,
+            "tilt_angle" : tilt_angle,
+            "tilt_axis" : tilt_axis,
+            "sample_thickness" : sample_thickness
+        })
+
+
+    return(result_ctf)
+
+async def run_async(parameters: Union[CtffindParameters,list[CtffindParameters]],**kwargs):
+
+    if not isinstance(parameters, list):
+        parameters = [parameters]
+
+    byte_results = await cistem_program.run("ctffind", parameters, signal_handlers=signal_handlers,**kwargs)
+    result_ctf = []
+
+    for parameter_index,byte_result in byte_results:
+        defocus1 = struct.unpack("<f",byte_result[0:4])[0]
+        defocus2 = struct.unpack("<f",byte_result[4:8])[0]
+        astigmatism_angle = struct.unpack("<f",byte_result[8:12])[0]
+        phase_shift = struct.unpack("<f",byte_result[12:16])[0]
+        score = struct.unpack("<f",byte_result[16:20])[0]
+        fit_resolution = struct.unpack("<f",byte_result[20:24])[0]
+        aliasing_resolution = struct.unpack("<f",byte_result[24:28])[0]
+        iciness = struct.unpack("<f",byte_result[28:32])[0]
+        tilt_angle = struct.unpack("<f",byte_result[32:36])[0]
+        tilt_axis = struct.unpack("<f",byte_result[36:40])[0]
+        sample_thickness = struct.unpack("<f",byte_result[40:44])[0]
+
+        result_ctf.append({
+            "parameter_index" : parameter_index,
+            "defocus1" : defocus1,
+            "defocus2" : defocus2,
+            "astigmatism_angle" : astigmatism_angle,
+            "phase_shift" : phase_shift,
+            "score" : score,
+            "fit_resolution" : fit_resolution,
+            "aliasing_resolution" : aliasing_resolution,
+            "iciness" : iciness,
+            "tilt_angle" : tilt_angle,
+            "tilt_axis" : tilt_axis,
+            "sample_thickness" : sample_thickness
+        })
+
+
+    return(result_ctf)

pycistem/programs/estimate_beamtilt.py

@@ -0,0 +1,60 @@
+import asyncio
+from dataclasses import dataclass
+from typing import Union
+import numpy as np
+import struct
+import mrcfile
+from functools import partial
+import pandas as pd
+
+
+from pycistem.programs import cistem_program
+from pycistem.programs._cistem_constants import socket_send_next_job, socket_job_result_queue
+#from pycistem.core import Image
+
+@dataclass
+class EstimateBeamtiltParameters:
+    input_phase_difference_image: str
+    pixel_size: float = 1.0
+    voltage_kV: float = 300.0      
+    spherical_aberration_mm: float  = 2.7
+    first_position_to_search: int = 0  
+    last_position_to_search: int = 0
+
+async def handle_results(reader, writer, logger):
+    #logger.info("Handling results")
+    await reader.read(4)
+    length = await reader.read(4)
+    number_of_bytes = int.from_bytes(length, byteorder="little")
+    results = await reader.read(number_of_bytes*4)
+    score = struct.unpack("<f",results[0:4])[0]
+    beam_tilt_x = struct.unpack("<f",results[4:8])[0]
+    beam_tilt_y = struct.unpack("<f",results[8:12])[0]
+    particle_shift_x = struct.unpack("<f",results[12:16])[0]
+    particle_shift_y = struct.unpack("<f",results[16:20])[0]
+    return(score,beam_tilt_x,beam_tilt_y,particle_shift_x,particle_shift_y)
+
+async def handle_job_result_queue(reader, writer, logger):
+
+    length = await reader.readexactly(4)
+    number_of_bytes = int.from_bytes(length, byteorder="little")
+    results = await reader.readexactly(number_of_bytes)
+    return(results)
+
+def run(parameters: Union[EstimateBeamtiltParameters,list[EstimateBeamtiltParameters]],**kwargs) -> pd.DataFrame:
+
+    if not isinstance(parameters, list):
+        parameters = [parameters]
+    signal_handlers = {
+        socket_send_next_job : handle_results,
+        socket_job_result_queue : handle_job_result_queue,
+
+    }   
+    result = asyncio.run(cistem_program.run("estimate_beamtilt", parameters, signal_handlers=signal_handlers,**kwargs))
+    result = pd.DataFrame([a[1] for a in result],
+                            index = [a[0] for a in result],
+                            columns=["score","beam_tilt_x","beam_tilt_y","particle_shift_x","particle_shift_y"])
+    return(result)
+        
+        
+        

pycistem/programs/match_template.py

@@ -0,0 +1,325 @@
+import asyncio
+import sqlite3
+import struct
+from dataclasses import dataclass
+from functools import partial
+from pathlib import Path
+from typing import List, Tuple, Union, Optional
+
+import mrcfile
+import numpy as np
+import pandas as pd
+from skimage.feature import peak_local_max
+import datetime
+
+from pycistem.core import EulerSearch, ParameterMap
+from pycistem.database import datetime_to_msdos, ensure_template_is_a_volume_asset, get_image_info_from_db, create_peak_lists, get_max_match_template_job_id
+from pycistem.programs import cistem_program
+from pycistem.programs._cistem_constants import socket_job_result_queue, socket_program_defined_result, socket_i_have_info
+
+
+@dataclass
+class MatchTemplateParameters:
+    input_search_images_filename: str #0
+    input_reconstruction_filename: str #1
+    pixel_size: float = 1.0 #2
+    voltage_kV: float = 300.0 #3
+    spherical_aberration_mm: float = 2.7
+    amplitude_contrast: float = 0.07
+    defocus1: float = 10000.0
+    defocus2: float = 10000.0
+    defocus_angle: float = 0.0
+    low_resolution_limit: float = 30.0
+    high_resolution_limit_search: float = 3.0
+    angular_step: float = 3.0 #10
+    best_parameters_to_keep: int = 1
+    defocus_search_range: float = 1000.0
+    defocus_step: float = 200.0
+    pixel_size_search_range: float = 0.0
+    pixel_size_step: float = 0.0
+    padding: float = 1.0
+    ctf_refinement: bool = False
+    particle_radius_angstroms: float = 0.0
+    phase_shift: float = 0.0
+    mip_output_file: str = "/dev/null"
+    best_psi_output_file: str = "best_psi.mrc"
+    best_theta_output_file: str = "best_theta.mrc"
+    best_phi_output_file: str = "best_phi.mrc"
+    best_defocus_output_file: str = "best_defocus.mrc"
+    best_pixel_size_output_file: str = "best_pixel_size.mrc"
+    scaled_mip_output_file: str = "scaled_mip.mrc"
+    correlation_avg_output_file : str = "correlation_avg.mrc"
+    my_symmetry: str = "C1"
+    in_plane_angular_step: float = 2.0
+    output_histogram_file: str = "histogram.txt"
+    first_search_position: int = 0
+    last_search_position: int = 100
+    image_number_for_gui: int = 0
+    number_of_jobs_per_image_in_gui: int = 1
+    correlation_std_output_file: str = "correlation_std.mrc"
+    directory_for_results: str = "/dev/null"
+    result_output_filename: str = "/dev/null"
+    min_peak_radius: float = 10.0
+    use_gpu: bool = True
+    max_threads: int = 4
+
+# TODO
+# 1. Write correct threshold into db
+# 2. Get correct border_exclude value
+# 3. Maybe do projection?
+async def handle_socket_i_have_info(reader, writer, logger):
+    data = await reader.readexactly(4)
+    length = int.from_bytes(data, byteorder="little")
+    data = await reader.readexactly(length)
+    print(f"Info: {data.decode('utf-8')}")
+
+def get_np_arrays(bytes,o,i,x,y,numpix):
+    array = np.frombuffer(bytes,offset=o+i*numpix*4, count=numpix,dtype=np.float32).copy()
+    array = array.reshape((y,-1))
+    array = array[:,:x]
+    return array
+
+def parameters_from_database(database, template_filename: str, match_template_job_id: Optional[int] = None,**kwargs) :
+    image_info = get_image_info_from_db(database,get_ctf=True)
+    if image_info is None:
+        return []
+    if match_template_job_id is None:
+        match_template_job_id = get_max_match_template_job_id(database) + 1
+    ProjectDirectory = Path(database).parent
+    par = [MatchTemplateParameters(
+        input_search_images_filename = image["FILENAME"],
+        input_reconstruction_filename = template_filename,
+        pixel_size = image["image_pixel_size"],
+        defocus1=image["DEFOCUS1"],
+        defocus2=image["DEFOCUS2"],
+        defocus_angle=image["DEFOCUS_ANGLE"],
+        scaled_mip_output_file=(ProjectDirectory / "Assets" / "TemplateMatching" / f"{Path(image['FILENAME']).stem}_auto_{match_template_job_id}_scaled_mip.mrc").as_posix(),
+        output_histogram_file=(ProjectDirectory / "Assets" / "TemplateMatching" / f"{Path(image['FILENAME']).stem}_auto_{match_template_job_id}_histogram.txt").as_posix(),
+    ) for i,image in image_info.iterrows()]
+    image_info["MATCH_TEMPLATE_JOB_ID"] = match_template_job_id
+    image_info["PARAMETERS"] = par
+    image_info["DATABASE"] = database
+    image_info["THRESHOLD"] = 0.0
+    return(image_info)
+
+def write_results_to_database(database,  parameters: list[MatchTemplateParameters], results: list[tuple[int, pd.DataFrame]], image_info):
+    # Ensure Volume assets
+    template_vol_ids = {}
+    for par in parameters:
+        template_vol_ids[par.input_reconstruction_filename] = -1
+    for template in template_vol_ids.keys():
+        template_vol_ids[template] = ensure_template_is_a_volume_asset(database, template, parameters[0].pixel_size)
+
+    conn = sqlite3.connect(database, isolation_level=None)
+    cur = conn.cursor()
+    results = sorted(results, key=lambda x: x[0])
+    # Add results to TEMPLATE_MATCH_LIST
+    template_match_result_list = []
+
+    max_template_match_id = cur.execute("SELECT MAX(TEMPLATE_MATCH_ID) FROM TEMPLATE_MATCH_LIST").fetchone()[0]
+    if max_template_match_id is None:
+        max_template_match_id = 0
+    template_match_id = max_template_match_id + 1
+    #template_match_job_id = cur.execute("SELECT MAX(TEMPLATE_MATCH_JOB_ID) FROM TEMPLATE_MATCH_LIST").fetchone()[0]
+    #if template_match_job_id is None:
+    #    template_match_job_id = 1
+    #else:
+    #    template_match_job_id += 1
+
+    for result in results:
+        template_match_job_id = image_info.iloc[result[0]]["MATCH_TEMPLATE_JOB_ID"]
+        # CHeck if THRESHOLD column exists in image_info
+        if "THRESHOLD" in image_info.columns:
+            threshold = image_info.iloc[result[0]]["THRESHOLD"]
+        else:
+            threshold = 7.0
+        template_match_result_list.append({
+            "TEMPLATE_MATCH_ID": template_match_id,
+            "JOB_NAME": f"auto_{template_match_job_id}_{Path(parameters[result[0]].input_reconstruction_filename).stem}",
+            "DATETIME_OF_RUN": datetime_to_msdos(datetime.datetime.now()),
+            "TEMPLATE_MATCH_JOB_ID": template_match_job_id,
+            "JOB_TYPE_CODE": 0,
+            "INPUT_TEMPLATE_MATCH_ID": 0,
+            "IMAGE_ASSET_ID": image_info.iloc[result[0]]["IMAGE_ASSET_ID"],
+            "REFERENCE_VOLUME_ASSET_ID": template_vol_ids[parameters[result[0]].input_reconstruction_filename],
+            "IS_ACTIVE": 1,
+            "USED_SYMMETRY": parameters[result[0]].my_symmetry,
+            "USED_PIXEL_SIZE": parameters[result[0]].pixel_size,
+            "USED_VOLTAGE": parameters[result[0]].voltage_kV,
+            "USED_SPHERICAL_ABERRATION": parameters[result[0]].spherical_aberration_mm,
+            "USED_AMPLITUDE_CONTRAST": parameters[result[0]].amplitude_contrast,
+            "USED_DEFOCUS1": parameters[result[0]].defocus1,
+            "USED_DEFOCUS2": parameters[result[0]].defocus2,
+            "USED_DEFOCUS_ANGLE": parameters[result[0]].defocus_angle,
+            "USED_PHASE_SHIFT": parameters[result[0]].phase_shift,
+            "LOW_RESOLUTION_LIMIT": parameters[result[0]].low_resolution_limit,
+            "HIGH_RESOLUTION_LIMIT": parameters[result[0]].high_resolution_limit_search,
+            "OUT_OF_PLANE_ANGULAR_STEP": parameters[result[0]].angular_step,
+            "IN_PLANE_ANGULAR_STEP": parameters[result[0]].in_plane_angular_step,
+            "DEFOCUS_SEARCH_RANGE": parameters[result[0]].defocus_search_range,
+            "DEFOCUS_STEP": parameters[result[0]].defocus_step,
+            "PIXEL_SIZE_SEARCH_RANGE": parameters[result[0]].pixel_size_search_range,
+            "PIXEL_SIZE_STEP": parameters[result[0]].pixel_size_step,
+            "REFINEMENT_THRESHOLD": 0.0,
+            "USED_THRESHOLD": threshold,
+            "REF_BOX_SIZE_IN_ANGSTROMS": 0,
+            "MASK_RADIUS": 0.0,
+            "MIN_PEAK_RADIUS": parameters[result[0]].min_peak_radius,
+            "XY_CHANGE_THRESHOLD": 0.0,
+            "EXCLUDE_ABOVE_XY_THRESHOLD": False,
+            "MIP_OUTPUT_FILE": "/dev/null",
+            "SCALED_MIP_OUTPUT_FILE": parameters[result[0]].scaled_mip_output_file,
+            "AVG_OUTPUT_FILE": "/dev/null",
+            "STD_OUTPUT_FILE": "/dev/null",
+            "PSI_OUTPUT_FILE": "/dev/null",
+            "THETA_OUTPUT_FILE": "/dev/null",
+            "PHI_OUTPUT_FILE": "/dev/null",
+            "DEFOCUS_OUTPUT_FILE": "/dev/null",
+            "PIXEL_SIZE_OUTPUT_FILE": "/dev/null",
+            "HISTOGRAM_OUTPUT_FILE": parameters[result[0]].output_histogram_file,
+            "PROJECTION_RESULT_OUTPUT_FILE": parameters[result[0]].scaled_mip_output_file,
+        })
+        create_peak_lists(conn, template_match_id)
+        result[1].to_sql(f"TEMPLATE_MATCH_PEAK_LIST_{template_match_id}", conn, if_exists="append", index=False)
+        conn.commit()
+        result[1]['ORIGINAL_PEAK_NUMBER'] = 0
+        result[1]['NEW_PEAK_NUMBER'] = 0
+        print(result[1])
+        result[1].to_sql(f"TEMPLATE_MATCH_PEAK_CHANGE_LIST_{template_match_id}", conn, if_exists="append", index=False)
+        template_match_id += 1
+    template_match_result_list = pd.DataFrame(template_match_result_list)
+    template_match_result_list.to_sql("TEMPLATE_MATCH_LIST", conn, if_exists="append", index=False)
+
+ 
+    conn.close()
+
+
+async def handle_results(reader, writer, logger, parameters, write_directly_to_db, image_info):
+    logger.debug("Handling results")
+    size_of_array= await reader.readexactly(4)
+    result_number= await reader.readexactly(4)
+    number_of_expected_results= await reader.readexactly(4)
+    number_of_floats = int.from_bytes(size_of_array, byteorder="little")
+    result_number = int.from_bytes(result_number, byteorder="little")
+    number_of_expected_results = int.from_bytes(number_of_expected_results, byteorder="little")
+    results = await reader.readexactly(number_of_floats*4)
+    x_dim = int(struct.unpack("<f",results[0:4])[0])
+    y_dim = int(struct.unpack("<f",results[4:8])[0])
+    num_pixels = int(struct.unpack("<f",results[8:12])[0])
+    num_histogram_points = int(struct.unpack("<f",results[16:20])[0])
+    print(f"Got number of histogram points: {num_histogram_points}, but replacing with hardcoded 512")
+    num_histogram_points = 512
+    num_ccs = struct.unpack("<f",results[12:16])[0]
+    struct.unpack("<f",results[20:24])[0]
+    struct.unpack("<f",results[24:28])[0]
+    print(f"Result number: {result_number} Number of expected results: {number_of_expected_results} Number of pixels: {num_pixels} X dim: {x_dim} Y dim: {y_dim} Num histogram points: {num_histogram_points} Num ccs: {num_ccs} Num float {number_of_floats}")
+    mip = get_np_arrays(results,28,0,x_dim,y_dim,num_pixels)
+    psi = get_np_arrays(results,28,1,x_dim,y_dim,num_pixels)
+    theta = get_np_arrays(results,28,2,x_dim,y_dim,num_pixels)
+    phi = get_np_arrays(results,28,3,x_dim,y_dim,num_pixels)
+    defocus = get_np_arrays(results,28,4,x_dim,y_dim,num_pixels)
+    get_np_arrays(results,28,5,x_dim,y_dim,num_pixels)
+    sum = get_np_arrays(results,28,6,x_dim,y_dim,num_pixels)
+
+    sum = sum / num_ccs
+    sum_squares = get_np_arrays(results,28,7,x_dim,y_dim,num_pixels)
+    sum_squares = np.sqrt(sum_squares/num_ccs - sum**2)
+    scaled_mip = np.divide(mip - sum, sum_squares, out=np.zeros_like(mip), where=sum_squares!=0)
+    par = parameters[result_number]
+    histogram = np.frombuffer(results,offset=28+8*num_pixels*4, count=num_histogram_points,dtype=np.int64).copy()
+    survival_histogram = np.zeros(num_histogram_points, dtype=np.float32)
+    survival_histogram[-1] = histogram[-1]
+    for line_counter in range(num_histogram_points - 2, -1, -1):
+        survival_histogram[line_counter] = survival_histogram[line_counter + 1] + histogram[line_counter]
+    
+   
+    # Calculate expected threshold
+    from scipy.special import erfcinv,erfc
+    expected_threshold = np.sqrt(2.0) * erfcinv(2.0/(x_dim*y_dim*num_ccs)) * 1
+
+    histogram_min              = -12.5
+    histogram_max              = 22.5
+    histogram_step        = (histogram_max - histogram_min) / num_histogram_points
+    temp_float = histogram_min + (histogram_step / 2.0)
+
+    expected_survival_histogram = np.zeros(num_histogram_points, dtype=np.float32)
+    for line_counter in range(num_histogram_points):
+        expected_survival_histogram[line_counter] = (erfc((temp_float + histogram_step * float(line_counter)) / np.sqrt(2.0)) / 2.0) * (x_dim*y_dim*num_ccs)
+    
+    survival_histogram_float  = survival_histogram * (expected_survival_histogram.sum() / survival_histogram.sum())
+    with open(par.output_histogram_file, 'w') as f:
+        f.write(f"# Expected threshold = {expected_threshold:.2f}\n")
+        f.write("#  histogram, expected histogram, survival histogram, expected survival histogram\n")
+        for line_counter in range(num_histogram_points):
+            temp_double_array = [
+                temp_float + histogram_step * float(line_counter),
+                histogram[line_counter],
+                survival_histogram[line_counter],
+                expected_survival_histogram[line_counter]
+            ]
+            f.write(" ".join(str(x) for x in temp_double_array) + "\n")
+    mrcfile.write(par.scaled_mip_output_file, scaled_mip.astype(np.float32), overwrite=True)
+    if par.mip_output_file != "/dev/null":
+        mrcfile.write(par.mip_output_file, mip.astype(np.float32), overwrite=True)
+    peak_coordinates = peak_local_max(scaled_mip, min_distance=int(par.min_peak_radius), exclude_border=50, threshold_abs=expected_threshold)
+    result = pd.DataFrame({
+        "X_POSITION": peak_coordinates[:,1] * par.pixel_size,
+        "Y_POSITION": peak_coordinates[:,0] * par.pixel_size,
+        "PSI": psi[tuple(peak_coordinates.T)],
+        "THETA": theta[tuple(peak_coordinates.T)],
+        "PHI": phi[tuple(peak_coordinates.T)],
+        "DEFOCUS": defocus[tuple(peak_coordinates.T)],
+        "PEAK_HEIGHT": scaled_mip[tuple(peak_coordinates.T)]
+    })
+    if(write_directly_to_db):
+        image_info["THRESHOLD"].iat[result_number] = expected_threshold
+        write_results_to_database(image_info.iloc[result_number]["DATABASE"], parameters, [(result_number, result)], image_info)
+        print("Wrote results to database")
+    print(f"{par.input_search_images_filename}: {len(result)} peaks found. Median {result['PEAK_HEIGHT'].median()} Max {result['PEAK_HEIGHT'].max()} Threshold {expected_threshold}")
+    return(result)
+
+async def handle_job_result_queue(reader, writer, logger):
+    #logger.info("Handling results")
+    #await reader.read(4)
+    length = await reader.readexactly(4)
+    number_of_bytes = int.from_bytes(length, byteorder="little")
+    results = await reader.readexactly(number_of_bytes)
+    number_of_jobs = int.from_bytes(results[0:4], byteorder="little")
+    # print(f"Number of bytes: {number_of_bytes} Number of jobs: {number_of_jobs}")
+    #for i in range(number_of_jobs):
+    #    job_number = int.from_bytes(results[4+i*4:8+i*4], byteorder="little")
+    #    result_size = int.from_bytes(results[8+i*4:12+i*4], byteorder="little")
+        
+    #    logger.info(f"Job {job_number} finished with result {result_number}")
+    return(results)
+
+
+
+def run(parameters: Union[MatchTemplateParameters,list[MatchTemplateParameters],pd.DataFrame],write_directly_to_db=False,image_info=None,**kwargs):
+
+    if isinstance(parameters, pd.DataFrame):
+        image_info = parameters
+        parameters = image_info["PARAMETERS"].tolist()
+    if not isinstance(parameters, list):
+        parameters = [parameters]
+
+    signal_handlers = {
+        socket_program_defined_result : partial(handle_results, parameters = parameters, write_directly_to_db=write_directly_to_db,image_info=image_info),
+        socket_job_result_queue : handle_job_result_queue,
+        socket_i_have_info: handle_socket_i_have_info,
+    }
+    for i, par in enumerate(parameters):
+        par.image_number_for_gui = i
+        global_euler_search = EulerSearch()
+        parameter_map = ParameterMap()
+        parameter_map.SetAllTrue( )
+        global_euler_search.InitGrid(par.my_symmetry,par.angular_step, 0.0, 0.0, 360.0, par.in_plane_angular_step, 0.0, 1.0 / 2.0, parameter_map, 10)
+        if par.my_symmetry.startswith("C") and global_euler_search.test_mirror:
+            global_euler_search.theta_max = 180.0
+        global_euler_search.CalculateGridSearchPositions(False)
+        par.first_search_position = 0
+        par.last_search_position = global_euler_search.number_of_search_positions - 1
+
+    results = asyncio.run(cistem_program.run("match_template_gpu", parameters, signal_handlers=signal_handlers,num_threads=parameters[0].max_threads,**kwargs))
+
+    return(results)