pycistem 0.6.1__cp310-cp310-manylinux_2_28_x86_64.whl

pycistem/programs/reconstruct3d.py

@@ -0,0 +1,79 @@
+import asyncio
+
+from dataclasses import dataclass
+from typing import Union, List
+
+
+from pycistem.programs import cistem_program
+from pycistem.programs._cistem_constants import socket_i_have_info, socket_job_result_queue, socket_send_next_job
+
+@dataclass
+class Reconstruct3dParameters:
+    input_particle_stack: str 
+    input_star_filename: str           
+    input_reconstruction: str         = "/dev/null"
+    output_reconstruction_1: str =  "my_reconstruction1.mrc"
+    output_reconstruction_2: str = "my_reconstruction2.mrc"
+    output_reconstruction_filtered: str = "my_reconstruction_filtered.mrc"
+    output_resolution_statistics: str = "my_resolution_statistics.txt"
+    my_symmetry: str = "C1"
+    first_particle: int = 1
+    last_particle: int = 0
+    pixel_size: float = 1.0
+    molecular_mass_kDa: float = 300.0
+    inner_mask_radius: float = 0.0
+    outer_mask_radius: float = 150.0
+    resolution_limit_rec: float = 0.0
+    resolution_limit_ref: float = 0.0
+    score_weight_conversion: float = 5.0
+    score_threshold: float = 1.0
+    smoothing_factor: float = 1.0
+    padding : float = 1.0
+    normalize_particles: bool = True
+    adjust_scores: bool = True
+    invert_contrast: bool = False
+    exclude_blank_edges: bool = False
+    crop_images: bool = False
+    split_even_odd: bool = True
+    center_mass: bool = False
+    use_input_reconstruction: bool = False
+    threshold_input_3d: bool = True
+    dump_arrays: bool = False
+    dump_file_1: str = "my_dump_file1.dat"
+    dump_file_2: str = "my_dump_file2.dat"
+    correct_ewald_sphere: int = 0
+    max_threads: int = 1
+
+
+async def handle_job_result_queue(reader, writer, logger):
+
+    length = await reader.readexactly(4)
+    number_of_bytes = int.from_bytes(length, byteorder="little")
+    results = await reader.readexactly(number_of_bytes)
+    return(results)
+
+async def handle_socket_i_have_info(reader, writer, logger):
+    data = await reader.readexactly(4)
+    length = int.from_bytes(data, byteorder="little")
+    data = await reader.readexactly(length)
+    print(f"Info: {data.decode('utf-8')}")
+
+async def handle_results(reader, writer, logger):
+    #logger.info("Handling results")
+    await reader.readexactly(4)
+    length = await reader.readexactly(4)
+    number_of_bytes = int.from_bytes(length, byteorder="little")
+    data = await reader.readexactly(number_of_bytes*4)
+
+def run(parameters: Union[Reconstruct3dParameters,list[Reconstruct3dParameters]],**kwargs):
+
+    if not isinstance(parameters, list):
+        parameters = [parameters]
+    signal_handlers = {
+        socket_job_result_queue : handle_job_result_queue,
+        socket_i_have_info: handle_socket_i_have_info,
+        socket_send_next_job: handle_results
+    }   
+    asyncio.run(cistem_program.run("reconstruct3d", parameters, signal_handlers=signal_handlers,**kwargs))
+    
+        

pycistem/programs/refine_ctf.py

@@ -0,0 +1,106 @@
+import asyncio
+from dataclasses import dataclass
+from typing import Union
+import numpy as np
+import struct
+import mrcfile
+from functools import partial
+
+
+from pycistem.programs import cistem_program
+from pycistem.programs._cistem_constants import socket_program_defined_result, socket_job_result_queue, socket_i_have_an_error
+#from pycistem.core import Image
+
+@dataclass
+class RefineCtfParameters:
+    input_particle_images: str 
+    input_star_filename: str
+    input_reconstruction: str
+    input_reconstruction_statistics: str = "my_statistics.txt"
+    use_statistics: bool = False
+    ouput_star_filename: str = "/dev/null"
+    ouput_shift_filename: str = "/dev/null"
+    ouput_phase_difference_image: str = "/dev/null"
+    ouput_beamtilt_image: str = "/dev/null"
+    ouput_difference_image: str = "/dev/null"
+    first_particle: int = 0
+    last_particle: int = 0
+    pixel_size: float = 1.0
+    molecular_mass_kDa: float = 300.0
+    inner_mask_radius: float = 0.0
+    outer_mask_radius: float = 150.0
+    low_resolution_limit: float = 300.0
+    high_resolution_limit: float = 3.0
+    defocus_search_range: float = 500.0
+    defocus_step: float = 50.0
+    padding: float = 1.0
+    ctf_refinement: bool = False
+    beamtilt_refinement: bool = False
+    normalize_particles: bool = True
+    invert_contrast: bool = False
+    exclude_blank_edges: bool = True
+    normalize_input_3d: bool = True
+    threshold_input_3d: bool = False
+    job_number_from_gui: int = 0 
+    expected_number_of_results_from_gui: int = 0
+    max_threads: int = 1
+
+def get_np_arrays(bytes,o,i,x,y,numpix):
+    array = np.frombuffer(bytes,offset=o+i*numpix*4, count=numpix,dtype=np.float32).copy()
+    array = array.reshape((y,-1))
+    array = array[:,:x]
+    return array
+
+async def handle_results(reader, writer, logger, parameters):
+    #logger.info("Handling results")
+    size_of_array= await reader.readexactly(4)
+    result_number= await reader.readexactly(4)
+    number_of_expected_results= await reader.readexactly(4)
+    number_of_floats = int.from_bytes(size_of_array, byteorder="little")
+    result_number = int.from_bytes(result_number, byteorder="little")
+    number_of_expected_results = int.from_bytes(number_of_expected_results, byteorder="little")
+    results = await reader.readexactly(number_of_floats*4)
+    x_dim = int(struct.unpack("<f",results[0:4])[0])
+    y_dim = int(struct.unpack("<f",results[4:8])[0])
+    num_pixels = int(struct.unpack("<f",results[8:12])[0])
+    images_to_process = int(struct.unpack("<f",results[12:16])[0])
+    voltage_kV = struct.unpack("<f",results[16:20])[0]
+    spherical_aberration_mm = struct.unpack("<f",results[20:24])[0]
+    phase_difference_image = get_np_arrays(results,24,0,x_dim,y_dim,num_pixels)
+    phase_difference_image_cistem = Image()
+
+    phase_difference_image_cistem.Allocate(x_dim,y_dim,1,False,True)
+    np.copyto(phase_difference_image_cistem.real_values , phase_difference_image)
+    phase_difference_image_cistem.DivideByConstant(images_to_process)
+    phase_difference_image_cistem.CosineMask(0.45, parameters[result_number].pixel_size / 20.0, False, False, 0.0)
+    phase_difference_image_cistem.QuickAndDirtyWriteSlice(parameters[result_number].ouput_phase_difference_image,1,True,0.0)
+    #mrcfile.write("test.mrc", phase_difference_image.astype(np.float32), overwrite=True)
+    return(results)
+
+async def handle_job_result_queue(reader, writer, logger):
+
+    length = await reader.readexactly(4)
+    number_of_bytes = int.from_bytes(length, byteorder="little")
+    results = await reader.readexactly(number_of_bytes)
+    return(results)
+
+async def handle_i_have_an_error(reader, writer, logger):
+    print("I have an error")
+    #number_of_bytes = int.from_bytes(length, byteorder="little")
+    number_of_bytes = 40
+    results = await reader.read(number_of_bytes)
+    print(results)
+    return(results)
+
+def run(parameters: Union[RefineCtfParameters,list[RefineCtfParameters]],**kwargs):
+
+    if not isinstance(parameters, list):
+        parameters = [parameters]
+    for i, par in enumerate(parameters):
+        par.image_number_for_gui = i
+    signal_handlers = {
+        socket_program_defined_result : partial(handle_results,parameters=parameters),
+        socket_job_result_queue : handle_job_result_queue,
+        socket_i_have_an_error: handle_i_have_an_error
+    }   
+    byte_results = asyncio.run(cistem_program.run("refine_ctf", parameters, signal_handlers=signal_handlers,**kwargs))

pycistem/programs/refine_template.py

@@ -0,0 +1,157 @@
+import asyncio
+import re
+import struct
+from dataclasses import dataclass
+from pickle import FALSE
+from typing import Union
+
+import pandas as pd
+import starfile
+
+from pycistem.database import get_image_info_from_db, get_tm_info_from_db
+from pycistem.programs import cistem_program
+from pycistem.programs._cistem_constants import socket_template_match_result_ready
+
+
+@dataclass
+class RefineTemplateParameters:
+    input_search_image: str
+    input_reconstruction: str = "template.mrc"
+    pixel_size: float = 1.0
+    voltate_kV: float = 300.0
+    spherical_aberration_mm: float = 2.7
+    amplitude_contrast: float = 0.07
+    defocus1: float = 10000
+    defocus2: float = 10000
+    defocus_angle: float = 0.0
+    angular_step: float = 0.2
+    defocus_search_range: float = 200.0
+    defocus_search_step: float = 10.0
+    padding: float = 1.0
+    mask_radius: float = 0.0
+    phase_shift: float = 0.0
+    mip_input_filename: str = "input_mip.mrc"
+    scaled_mip_input_filename: str = "input_scaled_mip.mrc"
+    best_psi_input_filename: str = "input_psi.mrc"
+    best_theta_input_filename: str = "input_theta.mrc"
+    best_phi_input_filename: str = "input_phi.mrc"
+    best_defocus_input_filename: str = "input_defocus.mrc"
+    best_psi_output_file: str = "/tmp/psi.mrc"
+    best_theta_output_file: str = "/tmp/theta.mrc"
+    best_phi_output_file: str = "/tmp/phi.mrc"
+    best_defocus_output_file: str = "/tmp/defocus.mrc"
+    mip_output_file: str = "/tmp/mip.mrc"
+    scaled_mip_output_file: str = "/tmp/scaled_mip.mrc"
+    wanted_threshold: float = 7.5
+    min_peak_radius: float = 10.0
+    xy_change_threshold: float = 10.0
+    exclude_above_xy_threshold: bool = False
+    in_plane_angular_step: float = 0.1
+    image_number_for_gui: int = 0
+    number_of_jobs_per_image_in_gui: int = 0
+    result_number: int = 1
+    max_threads: int = 1
+    directory_for_results: str = "/tmp"
+    threshold_for_result_plotting: float = 8.0
+    filename_for_gui_result_image: str = "/tmp/gui_result.mrc"
+    xyz_coords_filename: str ="/tmp/test.txt"
+    read_coordinates: bool = False
+
+async def handle_results(reader, writer, logger):
+    #logger.info("Handling results")
+    data = await reader.read(4)
+    number_of_bytes = int.from_bytes(data[0:3], byteorder="little")
+    results = await reader.read(number_of_bytes)
+    return(results)
+
+signal_handlers = {
+    socket_template_match_result_ready : handle_results
+}
+
+def parameters_from_database(database, image_asset_id, template_match_id, **kwargs):
+    image_info = get_image_info_from_db(database, image_asset=image_asset_id)
+    if image_info is None:
+        return None
+    tm_info = get_tm_info_from_db(database,image_asset_id, template_match_id)
+    if tm_info is None:
+        return(None)
+    par = RefineTemplateParameters(input_search_image=image_info["FILENAME"],
+                             pixel_size=image_info["image_pixel_size"],
+                             voltate_kV=image_info["VOLTAGE"],
+                             spherical_aberration_mm=image_info["SPHERICAL_ABERRATION"],
+                             amplitude_contrast=image_info["AMPLITUDE_CONTRAST"],
+                             defocus1=image_info["DEFOCUS1"],
+                             defocus2=image_info["DEFOCUS2"],
+                             defocus_angle=image_info["DEFOCUS_ANGLE"],
+                             mip_input_filename=tm_info["MIP_OUTPUT_FILE"],
+                             scaled_mip_input_filename=tm_info["SCALED_MIP_OUTPUT_FILE"],
+                             best_defocus_input_filename=tm_info["DEFOCUS_OUTPUT_FILE"],
+                             best_psi_input_filename=tm_info["PSI_OUTPUT_FILE"],
+                             best_theta_input_filename=tm_info["THETA_OUTPUT_FILE"],
+                             best_phi_input_filename=tm_info["PHI_OUTPUT_FILE"],
+                             **kwargs)
+    return(par)
+
+def run(parameters: Union[RefineTemplateParameters,list[RefineTemplateParameters]],**kwargs):
+
+    if not isinstance(parameters, list):
+        parameters = [parameters]
+
+    byte_results = asyncio.run(cistem_program.run("refine_template", parameters, signal_handlers=signal_handlers,**kwargs))
+    #File names of original image file name, 3D template file name, energy, Cs, amp. contrast, phase shift, X, Y position, Euler angles, defocus 1 & 2 & angle, pixel size, CC average, CC STD, SNR, scaled SNR
+    result_peaks = pd.DataFrame({
+        "image_filename": pd.Series(dtype="object"),
+        "template_filename": pd.Series(dtype="object"),
+        "energy": pd.Series(dtype="float"),
+        "Cs": pd.Series(dtype="float"),
+        "amplitude_contrast": pd.Series(dtype="float"),
+        "phase_shift": pd.Series(dtype="float"),
+        "defocus1": pd.Series(dtype="float"),
+        "defocus2": pd.Series(dtype="float"),
+        "defocus_angle": pd.Series(dtype="float"),
+        "peak_number": pd.Series(dtype="int"),
+        "x": pd.Series(dtype="float"),
+        "y": pd.Series(dtype="float"),
+        "psi": pd.Series(dtype="float"),
+        "theta": pd.Series(dtype="float"),
+        "phi": pd.Series(dtype="float"),
+        "defocus": pd.Series(dtype="float"),
+        "pixel_size": pd.Series(dtype="float"),
+        "peak_value": pd.Series(dtype="float")
+        })
+
+    for parameter_index,byte_result in byte_results:
+        struct.unpack_from("<i",byte_result,offset=0)[0]
+        peak_numbers = struct.unpack_from("<i",byte_result,offset=4)[0]
+        struct.unpack_from("<i",byte_result,offset=8)[0]
+        struct.unpack_from("<f",byte_result,offset=12)[0]
+
+
+        for peak_number in range(peak_numbers):
+            (x, y, psi, theta, phi, defocus, pixel_size, peak_height) = struct.unpack_from("<ffffffff",byte_result,offset=16+peak_number*32)
+            new_peak_series = pd.Series([
+                parameters[parameter_index].input_search_image,
+                parameters[parameter_index].input_reconstruction,
+                parameters[parameter_index].voltate_kV,
+                parameters[parameter_index].spherical_aberration_mm,
+                parameters[parameter_index].amplitude_contrast,
+                parameters[parameter_index].phase_shift,
+                parameters[parameter_index].defocus1,
+                parameters[parameter_index].defocus2,
+                parameters[parameter_index].defocus_angle,
+                int(peak_number),
+                x,
+                y,
+                psi,
+                theta,
+                phi,
+                defocus,
+                pixel_size,
+                peak_height], index = result_peaks.columns)
+            result_peaks.loc[len(result_peaks.index)] = new_peak_series
+
+    return(result_peaks)
+
+def write_starfile(results,filename, overwrite=True):
+    # Write the results dataframe to a star file
+    starfile.write(results, filename=filename, overwrite=overwrite)

pycistem/programs/refine_template_dev.py

@@ -0,0 +1,35 @@
+from dataclasses import dataclass
+import asyncio
+import pycistem.programs.cistem_program as cistem_program
+from typing import Union
+
+@dataclass
+class RefineTemplateDevParameters:
+    input_starfile: str = "input.star"
+    output_starfile: str = "output.star"
+    input_template: str = "input.mrc"
+    start_position:int = 0
+    end_position:int = -1
+    num_threads:int = 1
+
+def run(parameters: Union[RefineTemplateDevParameters,list[RefineTemplateDevParameters]],write_directly_to_db=True,**kwargs):
+
+    if not isinstance(parameters, list):
+        parameters = [parameters]
+
+    signal_handlers = {
+       
+    }
+    
+    results = asyncio.run(cistem_program.run("refine_template_dev", parameters, signal_handlers=signal_handlers,num_threads=parameters[0].num_threads,**kwargs))
+
+    return(results)
+
+async def run_async(parameters: Union[RefineTemplateDevParameters,list[RefineTemplateDevParameters]],**kwargs):
+
+    if not isinstance(parameters, list):
+        parameters = [parameters]
+
+    result = await cistem_program.run("refine_template_dev", parameters,num_threads=parameters[0].num_threads,**kwargs)
+
+    return(result)

pycistem/programs/refine_template_niko.py

@@ -0,0 +1,124 @@
+import asyncio
+import struct
+from dataclasses import dataclass
+from pickle import FALSE
+
+import pandas as pd
+
+from pycistem.database import get_image_info_from_db, get_tm_info_from_db
+from pycistem.programs import cistem_program
+from pycistem.programs._cistem_constants import socket_template_match_result_ready
+
+
+@dataclass
+class RefineTemplateParameters:
+    input_search_image: str
+    input_reconstruction: str = "template.mrc"
+    pixel_size: float = 1.0
+    voltate_kV: float = 300.0
+    spherical_aberration_mm: float = 2.7
+    amplitude_contrast: float = 0.07
+    defocus1: float = 10000
+    defocus2: float = 10000
+    defocus_angle: float = 0.0
+    low_resolution_limit: float = 300.0
+    high_resolution_limit: float = 3.0
+    angular_range: float = 0.2
+    angular_step: float = 0.2
+    best_parameters_to_keep: int = 20
+    defocus_search_range: float = 200.0
+    defocus_search_step: float = 10.0
+    pixel_size_search_range: float = 0.0
+    pixel_size_step: float = 0.01
+    padding: float = 1.0
+    ctf_refinement: bool = True
+    mask_radius: float = 0.0
+    phase_shift: float = 0.0
+    mip_input_filename: str = "input_mip.mrc"
+    scaled_mip_input_filename: str = "input_scaled_mip.mrc"
+    best_psi_input_filename: str = "input_psi.mrc"
+    best_theta_input_filename: str = "input_theta.mrc"
+    best_phi_input_filename: str = "input_phi.mrc"
+    best_defocus_input_filename: str = "input_defocus.mrc"
+    best_pixel_size_input_filename: str = "input_pixel_size.mrc"
+    best_psi_output_file: str = "/tmp/psi.mrc"
+    best_theta_output_file: str = "/tmp/theta.mrc"
+    best_phi_output_file: str = "/tmp/phi.mrc"
+    best_defocus_output_file: str = "/tmp/defocus.mrc"
+    best_pixel_size_output_file: str = "/tmp/pixel_size.mrc"
+    mip_output_file: str = "/tmp/mip.mrc"
+    scaled_mip_output_file: str = "/tmp/scaled_mip.mrc"
+    wanted_threshold: float = 7.5
+    min_peak_radius: float = 10.0
+    xy_change_threshold: float = 10.0
+    exclude_above_xy_threshold: bool = False
+    my_symmetry: str = "C1"
+    in_plane_angular_step: float = 0.1
+    first_search_position: int = -1
+    last_search_position: int = -1
+    image_number_for_gui: int = 0
+    number_of_jobs_per_image_in_gui: int = 0
+    result_number: int = 1
+    max_threads: int = 1
+    directory_for_results: str = "/tmp"
+    threshold_for_result_plotting: float = 8.0
+    filename_for_gui_result_image: str = "/tmp/gui_result.mrc"
+    xyz_coords_filename: str ="/tmp/test.txt"
+    read_coordinates: bool = False
+
+async def handle_results(reader, writer, logger):
+    logger.info("Handling results")
+    data = await reader.read(4)
+    number_of_bytes = int.from_bytes(data[0:3], byteorder="little")
+    results = await reader.read(number_of_bytes)
+    return(results)
+
+signal_handlers = {
+    socket_template_match_result_ready : handle_results
+}
+
+def parameters_from_database(database, image_asset_id, template_match_id, **kwargs):
+    image_info = get_image_info_from_db(database, image_asset=image_asset_id)
+    tm_info = get_tm_info_from_db(database,image_asset_id, template_match_id)
+    par = RefineTemplateParameters(input_search_image=image_info["FILENAME"],
+                             pixel_size=image_info["image_pixel_size"],
+                             voltate_kV=image_info["VOLTAGE"],
+                             spherical_aberration_mm=image_info["SPHERICAL_ABERRATION"],
+                             amplitude_contrast=image_info["AMPLITUDE_CONTRAST"],
+                             defocus1=image_info["DEFOCUS1"],
+                             defocus2=image_info["DEFOCUS2"],
+                             defocus_angle=image_info["DEFOCUS_ANGLE"],
+                             mip_input_filename=tm_info["MIP_OUTPUT_FILE"],
+                             scaled_mip_input_filename=tm_info["SCALED_MIP_OUTPUT_FILE"],
+                             best_defocus_input_filename=tm_info["DEFOCUS_OUTPUT_FILE"],
+                             best_pixel_size_input_filename=tm_info["PIXEL_SIZE_OUTPUT_FILE"],
+                             best_psi_input_filename=tm_info["PSI_OUTPUT_FILE"],
+                             best_theta_input_filename=tm_info["THETA_OUTPUT_FILE"],
+                             best_phi_input_filename=tm_info["PHI_OUTPUT_FILE"],
+                             **kwargs)
+    return(par)
+
+def run(parameters: RefineTemplateParameters):
+    byte_result = asyncio.run(cistem_program.run("refine_template", parameters, signal_handlers=signal_handlers))[0]
+    struct.unpack_from("<i",byte_result,offset=0)[0]
+    peak_numbers = struct.unpack_from("<i",byte_result,offset=4)[0]
+    struct.unpack_from("<i",byte_result,offset=8)[0]
+    struct.unpack_from("<f",byte_result,offset=12)[0]
+
+    result_peaks = pd.DataFrame({"peak_number": pd.Series(dtype="int"),
+                                 "x": pd.Series(dtype="float"),
+                                 "y": pd.Series(dtype="float"),
+                                 "psi": pd.Series(dtype="float"),
+                                 "theta": pd.Series(dtype="float"),
+                                 "phi": pd.Series(dtype="float"),
+                                 "defocus": pd.Series(dtype="float"),
+                                 "pixel_size": pd.Series(dtype="float"),
+                                 "peak_value": pd.Series(dtype="float")})
+
+    for peak_number in range(peak_numbers):
+        (x, y, psi, theta, phi, defocus, pixel_size, peak_height) = struct.unpack_from("<ffffffff",byte_result,offset=16+peak_number*32)
+        a_series = pd.Series([peak_number, x, y, psi, theta, phi, defocus, pixel_size, peak_height], index = result_peaks.columns)
+        result_peaks = result_peaks.append(a_series, ignore_index=True)
+
+    return(result_peaks)
+

pycistem/programs/resample.py

@@ -0,0 +1,30 @@
+import asyncio
+
+from dataclasses import dataclass
+from typing import Union, List
+
+
+from pycistem.programs import cistem_program
+from pycistem.programs._cistem_constants import socket_send_next_job
+
+@dataclass
+class ResampleParameters:
+    input_filename: str 
+    output_filename: str 
+    is_a_volume: bool = False  
+    new_x_size: int = 64
+    new_y_size: int = 64
+    new_z_size: int = 1    
+  
+
+
+
+
+def run(parameters: Union[ResampleParameters,list[ResampleParameters]],**kwargs):
+
+    if not isinstance(parameters, list):
+        parameters = [parameters]
+   
+    asyncio.run(cistem_program.run("resample", parameters, **kwargs))
+    
+        

pycistem/programs/run_profile.py

@@ -0,0 +1,17 @@
+
+host_gpu_info = {
+    "kyiv": 8,
+    "warsaw": 8,
+}
+
+def generate_local_gpu_prefix(num_gpus):
+    for gpu in range(num_gpus):
+        yield f'CUDA_VISIBLE_DEVICES={gpu} '
+
+def generate_gpu_prefix(host_gpu_info):
+    for host in host_gpu_info:
+        for gpu in range(host_gpu_info[host]):
+            yield f'ssh {host} "CUDA_VISIBLE_DEVICES={gpu} '
+
+def generate_num_procs(host_gpu_info):
+    return sum(host_gpu_info.values())