octopi 1.0__py3-none-any.whl

octopi/entry_points/run_create_targets.py

@@ -0,0 +1,281 @@
+import octopi.processing.create_targets_from_picks as create_targets
+from typing import List, Tuple, Union
+from collections import defaultdict
+import argparse, copick, yaml, os
+from octopi import utils, io 
+from tqdm import tqdm
+import numpy as np
+
+def create_sub_train_targets(
+    config: str,
+    pick_targets: List[Tuple[str, Union[str, None], Union[str, None]]],  # Updated type without radius
+    seg_targets: List[Tuple[str, Union[str, None], Union[str, None]]],
+    voxel_size: float,
+    radius_scale: float,    
+    tomogram_algorithm: str,
+    target_segmentation_name: str,
+    target_user_id: str,
+    target_session_id: str,
+    run_ids: List[str],    
+    ):
+
+    # Load Copick Project 
+    root = copick.from_file(config)
+
+    # Create empty dictionary for all targets
+    train_targets = defaultdict(dict)
+
+    # Create dictionary for particle targets
+    for t in pick_targets:
+        # Parse the target
+        obj_name, user_id, session_id = t 
+        obj = root.get_object(obj_name)
+
+        # Check if the object is valid
+        if obj is None:
+            print(f'Warning - Skipping Particle Target: "{obj_name}", as it is not a valid name in the config file.')
+            continue
+        
+        # Get the label and radius of the object
+        label = obj.label
+        info = {
+            "label": label,
+            "user_id": user_id,
+            "session_id": session_id,
+            "is_particle_target": True,
+            "radius": root.get_object(obj_name).radius,
+        }
+        train_targets[obj_name] = info
+
+    # Create dictionary for segmentation targets
+    train_targets = add_segmentation_targets(root, seg_targets, train_targets)
+
+    create_targets.generate_targets(
+        root, train_targets, voxel_size, tomogram_algorithm, radius_scale,
+        target_segmentation_name, target_user_id, 
+        target_session_id, run_ids
+    )
+
+
+def create_all_train_targets(
+    config: str,
+    seg_targets: List[List[Tuple[str, Union[str, None], Union[str, None]]]],
+    picks_session_id: str,
+    picks_user_id: str,
+    voxel_size: float,
+    radius_scale: float,
+    tomogram_algorithm: str,
+    target_segmentation_name: str,
+    target_user_id: str,
+    target_session_id: str,
+    run_ids: List[str],    
+    ):     
+
+    # Load Copick Project 
+    root = copick.from_file(config)
+
+    # Create empty dictionary for all targets
+    target_objects = defaultdict(dict)
+
+    # Create dictionary for particle targets
+    for object in root.pickable_objects:
+        info = {
+            "label": object.label,
+            "radius": object.radius,
+            "user_id": picks_user_id,
+            "session_id": picks_session_id,
+            "is_particle_target": True,
+        }
+        target_objects[object.name] = info
+
+    # Create dictionary for segmentation targets
+    target_objects = add_segmentation_targets(root, seg_targets, target_objects)
+
+    create_targets.generate_targets(
+        root, target_objects, voxel_size, tomogram_algorithm, 
+        radius_scale, target_segmentation_name, target_user_id, 
+        target_session_id, run_ids 
+    )
+
+def add_segmentation_targets(
+    root,
+    seg_targets,
+    train_targets: dict,
+    ):
+
+    # Create dictionary for segmentation targets
+    for s in seg_targets:
+
+        # Parse Segmentation Target
+        obj_name, user_id, session_id = s
+
+        # Add Segmentation Target
+        try:
+            info = {
+                "label": root.get_object(obj_name).label,
+                "user_id": user_id,
+                "session_id": session_id,
+                "is_particle_target": False,                 
+                "radius": None,    
+            }
+            train_targets[obj_name] = info
+
+        # If Segmentation Target is not found, print warning
+        except:
+            print(f'Warning - Skipping Segmentation Name: "{obj_name}", as it is not a valid object in the Copick project.')
+
+    return train_targets    
+
+def parse_args():
+    """
+    Parse command-line arguments for generating segmentation targets from CoPick configurations.
+
+    This tool allows researchers to specify protein labels for training in two ways:
+    
+    1. **Manual Specification:** Users can define a subset of pickable objects from the CoPick configuration file.
+       - Specify a target protein using `--target name`, or refine selection with `--target name,user_id,session_id`.
+       - This enables flexible training target customization from multiple sources.
+
+    2. **Automated Query:** Instead of specifying targets explicitly, users can provide a session ID (`--picks-session-id`) and/or 
+       user ID (`--picks-user-id`). DeepFindET will automatically retrieve all pickable objects associated with the query.
+    
+    The tool also allows customization of tomogram reconstruction settings and segmentation parameters.
+
+    Example Usage:
+        - Manual Specification:
+            ```bash
+            python create_targets.py --config config.json --target ribosome --target apoferritin,123,456
+            ```
+        - Automated Query:
+            ```bash
+            python create_targets.py --config config.json --picks-session-id 123 --picks-user-id 456
+            ```
+
+    Output segmentation data is saved in a structured YAML file.
+    """
+    parser = argparse.ArgumentParser(
+        description=f"""Generate segmentation targets from CoPick configurations with either --target flag (which lets users specify a subset of pickable objects) or --picks-session-id and --picks-user-id flags (which lets users specify a sessionID and userID to automatically retrieve all pickable objects associated with the query).""",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
+
+    input_group = parser.add_argument_group("Input Arguments")
+    input_group.add_argument("--config", type=str, required=True, help="Path to the CoPick configuration file.")
+    input_group.add_argument("--target", type=utils.parse_target, action="append", default=None, help='Target specifications: "name" or "name,user_id,session_id".')
+    input_group.add_argument("--picks-session-id", type=str, default=None, help="Session ID for the picks.")
+    input_group.add_argument("--picks-user-id", type=str, default=None, help="User ID associated with the picks.")
+    input_group.add_argument("--seg-target", type=utils.parse_target, action="append", default=[], help='Segmentation targets: "name" or "name,user_id,session_id".')
+    input_group.add_argument("--run-ids", type=utils.parse_list, default=None, help="List of run IDs.")
+
+    # Parameters
+    parameters_group = parser.add_argument_group("Parameters")
+    parameters_group.add_argument("--tomo-alg", type=str, default="wbp", help="Tomogram reconstruction algorithm.")    
+    parameters_group.add_argument("--radius-scale", type=float, default=0.8, help="Scale factor for object radius.")
+    parameters_group.add_argument("--voxel-size", type=float, default=10, help="Voxel size for tomogram reconstruction.")    
+    
+    output_group = parser.add_argument_group("Output Arguments")
+    output_group.add_argument("--target-segmentation-name", type=str, default='targets', help="Name for the target segmentation.")
+    output_group.add_argument("--target-user-id", type=str, default="octopi", help="User ID associated with the target segmentation.")
+    output_group.add_argument("--target-session-id", type=str, default="1", help="Session ID for the target segmentation.")
+
+    return parser.parse_args()
+
+def cli():
+    args = parse_args()
+
+    # Save JSON with Parameters
+    output_yaml = f'create-targets_{args.target_user_id}_{args.target_session_id}_{args.target_segmentation_name}.yaml'
+    save_parameters(args, output_yaml)      
+
+    if args.target:
+        # If at least one --target is provided, call create_sub_train_targets
+        create_sub_train_targets(
+            config=args.config,
+            pick_targets=args.target,
+            seg_targets=args.seg_target,
+            voxel_size=args.voxel_size,
+            radius_scale=args.radius_scale,
+            tomogram_algorithm=args.tomo_alg,
+            target_segmentation_name=args.target_segmentation_name,
+            target_user_id=args.target_user_id,
+            target_session_id=args.target_session_id,
+            run_ids=args.run_ids,
+        )
+    else:
+        # If no --target is provided, call create_all_train_targets
+        create_all_train_targets(
+            config=args.config,
+            seg_targets=args.seg_target,
+            picks_session_id=args.picks_session_id,
+            picks_user_id=args.picks_user_id,
+            voxel_size=args.voxel_size,
+            radius_scale=args.radius_scale,
+            tomogram_algorithm=args.tomo_alg,
+            target_segmentation_name=args.target_segmentation_name,
+            target_user_id=args.target_user_id,
+            target_session_id=args.target_session_id,
+            run_ids=args.run_ids,
+        )
+
+def save_parameters(args, output_path: str):
+    """
+    Save parameters to a YAML file with subgroups for input, output, and parameters.
+    Append to the file if it already exists.
+
+    Args:
+        args: Parsed arguments from argparse.
+        output_path: Path to save the YAML file.
+    """
+
+    print('\nGenerating Target Segmentation Masks from the Following Copick-Query:')
+    if args.picks_session_id is None or args.picks_user_id is None:
+        print(f'    - {args.target}\n')
+        input_group = {
+            "config": args.config,
+            "target": args.target,
+        }
+    else:
+        print(f'    - {args.picks_session_id}, {args.picks_user_id}\n')
+        input_group = {
+            "config": args.config,
+            "picks_session_id": args.picks_session_id,
+            "picks_user_id": args.picks_user_id
+        }
+    if len(args.seg_target) > 0:
+        input_group["seg_target"] = args.seg_target
+        
+    # Organize parameters into subgroups
+    input_key = f'{args.target_user_id}_{args.target_session_id}_{args.target_segmentation_name}'
+    new_entry = {
+        input_key : {
+            "input": input_group ,
+            "parameters": {
+                "radius_scale": args.radius_scale,
+                "tomogram_algorithm": args.tomo_alg,
+                "voxel_size": args.voxel_size,            
+            }
+        }
+    }
+
+     # Check if the YAML file already exists
+    if os.path.exists(output_path):
+        # Load the existing content
+        with open(output_path, 'r') as f:
+            try:
+                existing_data = yaml.safe_load(f)
+                if existing_data is None:
+                    existing_data = {}  # Ensure it's a dictionary
+                elif not isinstance(existing_data, dict):
+                    raise ValueError("Existing YAML data is not a dictionary. Cannot update.")
+            except yaml.YAMLError:
+                existing_data = {}  # Treat as empty if the file is malformed
+    else:
+        existing_data = {}  # Initialize as empty list if the file does not exist
+
+    # Append the new entry
+    existing_data[input_key] = new_entry[input_key]
+
+    # Save back to the YAML file
+    utils.save_parameters_yaml(existing_data, output_path)
+
+if __name__ == "__main__":
+    cli()

octopi/entry_points/run_evaluate.py

@@ -0,0 +1,65 @@
+import octopi.processing.evaluate as evaluate
+import octopi.utils as utils
+from typing import List
+import argparse
+
+def my_evaluator(
+    copick_config_path: str,
+    ground_truth_user_id: str,
+    ground_truth_session_id: str,
+    predict_user_id: str,
+    predict_session_id: str,
+    save_path: str,
+    distance_threshold_scale: float,
+    object_names: List[str] = None,
+    runIDs: List[str] = None
+    ):
+
+    eval = evaluate.evaluator(
+        copick_config_path,
+        ground_truth_user_id,
+        ground_truth_session_id,
+        predict_user_id,
+        predict_session_id, 
+        object_names=object_names
+    )
+
+    eval.run(save_path=save_path, distance_threshold_scale=distance_threshold_scale, runIDs=runIDs)
+
+# Entry point with argparse
+def cli():
+    """
+    CLI entry point for running evaluation.
+    """
+    parser = argparse.ArgumentParser(
+        description='Run evaluation on pick and place predictions.',
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
+    
+    parser.add_argument('--config', type=str, required=True, help='Path to the copick configuration file')
+    parser.add_argument('--ground-truth-user-id', type=str, required=True, help='User ID for ground truth data')
+    parser.add_argument('--ground-truth-session-id', type=str, required=False, default= None, help='Session ID for ground truth data')
+    parser.add_argument('--predict-user-id', type=str, required=True, help='User ID for prediction data')
+    parser.add_argument('--predict-session-id', type=str, required=False, default= None, help='Session ID for prediction data')
+    parser.add_argument('--save-path', type=str, required=False, default= None, help='Path to save evaluation results')
+    parser.add_argument('--distance-threshold-scale', type=float, required=False, default = 0.8, help='Compute Distance Threshold Based on Particle Radius')
+    parser.add_argument('--object-names', type=utils.parse_list, default=None, required=False, help='Optional list of object names to evaluate, e.g., ribosome,apoferritin or [ribosome,apoferritin].')
+    parser.add_argument('--run-ids', type=utils.parse_list, default=None, required=False, help='Optional list of run IDs to evaluate, e.g., run1,run2,run3 or [run1,run2,run3].')
+
+    args = parser.parse_args()
+
+    # Call the evaluate function with parsed arguments
+    my_evaluator(
+        copick_config_path=args.config,
+        ground_truth_user_id=args.ground_truth_user_id,
+        ground_truth_session_id=args.ground_truth_session_id,
+        predict_user_id=args.predict_user_id,
+        predict_session_id=args.predict_session_id,
+        save_path=args.save_path,
+        distance_threshold_scale=args.distance_threshold_scale,
+        object_names=args.object_names,
+        runIDs=args.run_ids
+    )
+
+if __name__ == "__main__":
+    cli()

octopi/entry_points/run_extract_mb_picks.py

@@ -0,0 +1,141 @@
+from octopi.extract import membranebound_extract as extract
+from octopi import utils, io
+import argparse, json, pprint, copick, json
+from typing import List, Tuple, Optional
+import multiprocess as mp
+from tqdm import tqdm
+
+def extract_membrane_bound_picks(
+    config: str,
+    voxel_size: float,
+    distance_threshold: float,
+    picks_info: Tuple[str, str, str],
+    organelle_info: Tuple[str, str, str],
+    membrane_info: Tuple[str, str, str],
+    save_user_id: str,
+    save_session_id: str,
+    runIDs: List[str],
+    n_procs: int = None
+    ):  
+
+    # Load Copick Project for Writing 
+    root = copick.from_file( config ) 
+    
+    # Either Specify Input RunIDs or Run on All RunIDs
+    if runIDs:  print('Extracting Membrane Bound Proteins on the Following RunIDs: ', runIDs)
+    run_ids = runIDs if runIDs else [run.name for run in root.runs]
+    n_run_ids = len(run_ids)    
+
+    # Determine the number of processes to use
+    if n_procs is None:
+        n_procs = min(mp.cpu_count(), n_run_ids)
+    print(f"Using {n_procs} processes to parallelize across {n_run_ids} run IDs.")   
+
+    # Initialize tqdm progress bar
+    with tqdm(total=n_run_ids, desc="Membrane-Protein Isolation", unit="run") as pbar:
+        for _iz in range(0, n_run_ids, n_procs):
+
+            start_idx = _iz
+            end_idx = min(_iz + n_procs, n_run_ids)  # Ensure end_idx does not exceed n_run_ids
+            print(f"\nProcessing runIDs from {start_idx} -> {end_idx } (out of {n_run_ids})")
+
+            processes = []                
+            for _in in range(n_procs):
+                _iz_this = _iz + _in
+                if _iz_this >= n_run_ids:
+                    break
+                run_id = run_ids[_iz_this]
+                run = root.get_run(run_id)
+                p = mp.Process(
+                    target=extract.process_membrane_bound_extract,
+                    args=(run,  
+                          voxel_size, 
+                          picks_info, 
+                          membrane_info,
+                          organelle_info,
+                          save_user_id, 
+                          save_session_id,
+                          distance_threshold),
+                )
+                processes.append(p)
+
+            for p in processes:
+                p.start()
+
+            for p in processes:
+                p.join()
+
+            for p in processes:
+                p.close()
+
+            # Update tqdm progress bar
+            pbar.update(len(processes))
+
+    print('Extraction of Membrane-Bound Proteins Complete!')
+
+def cli():
+    parser = argparse.ArgumentParser(
+        description='Extract membrane-bound picks based on proximity to segmentation.',
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
+    parser.add_argument('--config', type=str, required=True, help='Path to the configuration file.')
+    parser.add_argument('--voxel-size', type=float, required=False, default=10, help='Voxel size.')
+    parser.add_argument('--distance-threshold', type=float, required=False, default=10, help='Distance threshold.')
+    parser.add_argument('--picks-info', type=utils.parse_target, required=True, help='Query for the picks (e.g., "name" or "name,user_id,session_id".).')
+    parser.add_argument('--membrane-info', type=utils.parse_target, required=False, help='Query for the membrane segmentation (e.g., "name" or "name,user_id,session_id".).')
+    parser.add_argument('--organelle-info', type=utils.parse_target, required=False, help='Query for the organelles segmentations (e.g., "name" or "name,user_id,session_id".).')
+    parser.add_argument('--save-user-id', type=str, required=False, default=None, help='User ID to save the new picks.')
+    parser.add_argument('--save-session-id', type=str, required=True, help='Session ID to save the new picks.')
+    parser.add_argument('--runIDs', type=utils.parse_list, required=False, help='List of run IDs to process.')
+    parser.add_argument('--n-procs', type=int, required=False, default=None, help='Number of processes to use.')
+
+    args = parser.parse_args()
+
+    # Increment session ID for the second class
+    if args.save_user_id is None: 
+        args.save_user_id = args.picks_user_id
+
+    # Save JSON with Parameters
+    output_yaml = f'membrane-extract_{args.save_user_id}_{args.save_session_id}.yaml'        
+    save_parameters(args, output_yaml)
+
+    extract_membrane_bound_picks(
+        config=args.config,
+        voxel_size=args.voxel_size,
+        distance_threshold=args.distance_threshold,
+        picks_info=args.picks_info,
+        membrane_info=args.membrane_info,
+        organelle_info=args.organelle_info,
+        save_user_id=args.save_user_id,
+        save_session_id=args.save_session_id,
+        runIDs=args.runIDs,
+        n_procs=args.n_procs,
+    )
+
+def save_parameters(args: argparse.Namespace, 
+                    output_path: str):
+
+    params_dict = {
+        "input": {
+            k: getattr(args, k) for k in [
+                "config", "voxel_size", "picks_info", 
+                "membrane_info", "organelle_info"
+            ]
+        },
+        "output": {
+            k: getattr(args, k) for k in ["save_user_id", "save_session_id"]
+        },
+        "parameters": {
+            k: getattr(args, k) for k in ["distance_threshold", "runIDs"]
+        }
+    }
+
+    # Print the parameters
+    print(f"\nParameters for Extraction of Membrane-Bound Picks:")
+    pprint.pprint(params_dict); print()
+
+    # Save parameters to YAML file
+    utils.save_parameters_yaml(params_dict, output_path) 
+
+if __name__ == "__main__":
+    cli()

octopi/entry_points/run_extract_midpoint.py

@@ -0,0 +1,143 @@
+from octopi.extract import midpoint_extract
+from typing import List, Tuple, Optional
+import argparse, pprint, copick
+from octopi import utils
+import multiprocess as mp
+from tqdm import tqdm
+
+def extract_midpoint(
+    config: str,
+    voxel_size: float,
+    picks_info: Tuple[str, str, str],
+    organelle_info: Tuple[str, str, str],
+    distance_min: float,
+    distance_max: float,
+    distance_threshold: float,
+    save_session_id: str,
+    runIDs: List[str],
+    n_procs: int = None
+    ):  
+
+    # Load Copick Project for Writing 
+    root = copick.from_file( config ) 
+    
+    # Either Specify Input RunIDs or Run on All RunIDs
+    if runIDs:  print('Extracting Midpoints on the Following RunIDs: ', runIDs)
+    run_ids = runIDs if runIDs else [run.name for run in root.runs]
+    n_run_ids = len(run_ids)   
+
+    # Determine the number of processes to use
+    if n_procs is None:
+        n_procs = min(mp.cpu_count(), n_run_ids)
+    print(f"Using {n_procs} processes to parallelize across {n_run_ids} run IDs.")   
+
+    # Initialize tqdm progress bar
+    with tqdm(total=n_run_ids, desc="Mid-Point SuperComplex Extraction", unit="run") as pbar:
+        for _iz in range(0, n_run_ids, n_procs):
+
+            start_idx = _iz
+            end_idx = min(_iz + n_procs, n_run_ids)  # Ensure end_idx does not exceed n_run_ids
+            print(f"\nProcessing runIDs from {start_idx} -> {end_idx } (out of {n_run_ids})")
+
+            processes = []                
+            for _in in range(n_procs):
+                _iz_this = _iz + _in
+                if _iz_this >= n_run_ids:
+                    break
+                run_id = run_ids[_iz_this]
+                run = root.get_run(run_id)
+                p = mp.Process(
+                    target=midpoint_extract.process_midpoint_extract,
+                    args=(run,  
+                          voxel_size, 
+                          picks_info, 
+                          organelle_info,
+                          distance_min,
+                          distance_max,
+                          distance_threshold,
+                          save_session_id)
+                )
+                processes.append(p)
+
+            for p in processes:
+                p.start()
+
+            for p in processes:
+                p.join()
+
+            for p in processes:
+                p.close()
+
+            # Update tqdm progress bar
+            pbar.update(len(processes))
+
+    print('Extraction of Midpoints Complete!')        
+
+def cli():
+    parser = argparse.ArgumentParser(
+        description='Extract membrane-bound picks based on proximity to segmentation.',
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
+    parser.add_argument('--config', type=str, required=True, help='Path to the configuration file.')
+    parser.add_argument('--voxel-size', type=float, required=False, default=10, help='Segmentation Voxel size.')
+    parser.add_argument('--picks-info', type=utils.parse_target, required=True, help='Query for the picks (e.g., "name" or "name,user_id,session_id".).')
+    parser.add_argument('--organelle-info', type=utils.parse_target, required=False, help='Query for the organelles segmentations (e.g., "name" or "name,user_id,session_id".).')
+    parser.add_argument('--distance-min', type=float, required=False, default=10, help='Minimum distance for valid nearest neighbors.') 
+    parser.add_argument('--distance-max', type=float, required=False, default=70, help='Maximum distance for valid nearest neighbors.')
+    parser.add_argument('--distance-threshold', type=float, required=False, default=25, help='Distance threshold for picks to associated organelles.')
+    parser.add_argument('--save-session-id', type=str, required=False, default=None, help='(Optional)SessionID to save the new picks. If none provided, will use the sessionID from the picks.')
+    parser.add_argument('--runIDs', type=utils.parse_list, required=False, help='(Optional) List of run IDs to process.')
+    parser.add_argument('--n-procs', type=int, required=False, default=None, help='Number of processes to use. In none providd, will use the total number of CPUs available.')
+
+    args = parser.parse_args()
+
+    # Increment session ID for the second class
+    if args.save_session_id is None: 
+        args.save_session_id = args.picks_info[2]
+    args.save_user_id = args.picks_info[1]       
+
+    # Save JSON with Parameters
+    output_yaml = f'midpoint-extract_{args.picks_info[1]}_{args.save_session_id}.yaml'
+    save_parameters(args, output_yaml)
+
+
+    extract_midpoint(
+        config=args.config,
+        voxel_size=args.voxel_size,
+        picks_info=args.picks_info,
+        organelle_info=args.organelle_info,
+        distance_min=args.distance_min,
+        distance_max=args.distance_max,
+        distance_threshold=args.distance_threshold,
+        save_session_id=args.save_session_id,
+        runIDs=args.runIDs,
+        n_procs=args.n_procs,
+    )
+
+def save_parameters(args: argparse.Namespace, 
+                    output_path: str):
+
+    params_dict = {
+        "input": {
+            k: getattr(args, k) for k in [
+                "config", "voxel_size", "picks_info", 
+                "organelle_info"
+            ]
+        },
+        "output": {
+            k: getattr(args, k) for k in ["save_user_id", "save_session_id"]
+        },
+        "parameters": {
+            k: getattr(args, k) for k in ["distance_min", "distance_max", "distance_threshold", "runIDs"]
+        }
+    }
+
+    # Print the parameters
+    print(f"\nParameters for Extraction of Membrane-Bound Picks:")
+    pprint.pprint(params_dict); print()
+
+    # Save parameters to YAML file
+    utils.save_parameters_yaml(params_dict, output_path) 
+
+if __name__ == "__main__":
+    cli()