celldetective 1.3.0.post1__py3-none-any.whl → 1.3.2__py3-none-any.whl

celldetective/preprocessing.py

@@ -6,12 +6,13 @@ from tqdm import tqdm
 import numpy as np
 import os
 from celldetective.io import get_config, get_experiment_wells, interpret_wells_and_positions, extract_well_name_and_number, get_positions_in_well, extract_position_name, get_position_movie_path, load_frames, auto_load_number_of_frames
-from celldetective.utils import estimate_unreliable_edge, unpad, ConfigSectionMap, _extract_channel_indices_from_config, _extract_nbr_channels_from_config, _get_img_num_per_channel
+from celldetective.utils import interpolate_nan, estimate_unreliable_edge, unpad, ConfigSectionMap, _extract_channel_indices_from_config, _extract_nbr_channels_from_config, _get_img_num_per_channel
 from celldetective.segmentation import filter_image, threshold_image
 from csbdeep.io import save_tiff_imagej_compatible
 from gc import collect
 from lmfit import Parameters, Model
 import tifffile.tifffile as tiff
+from scipy.ndimage import shift
 
 def estimate_background_per_condition(experiment, threshold_on_std=1, well_option='*', target_channel="channel_name", frame_range=[0,5], mode="timeseries", activation_protocol=[['gauss',2],['std',4]], show_progress_per_pos=False, show_progress_per_well=True):
 	
@@ -879,7 +880,7 @@ def fit_and_apply_model_background_to_stack(stack_path,
 
 	stack_length_auto = auto_load_number_of_frames(stack_path)
 	if stack_length_auto is None and stack_length is None:
-		print('stack length not provided')
+		print('Stack length not provided...')
 		return None
 	if stack_length_auto is not None:
 		stack_length = stack_length_auto
@@ -899,6 +900,7 @@ def fit_and_apply_model_background_to_stack(stack_path,
 				
 				frames = load_frames(list(np.arange(i,(i+nbr_channels))), stack_path, normalize_input=False).astype(float)
 				target_img = frames[:,:,target_channel_index].copy()
+				
 				correction = field_correction(target_img, threshold_on_std=threshold_on_std, operation=operation, model=model, clip=clip, activation_protocol=activation_protocol)
 				frames[:,:,target_channel_index] = correction.copy()
 
@@ -919,6 +921,7 @@ def fit_and_apply_model_background_to_stack(stack_path,
 			
 			frames = load_frames(list(np.arange(i,(i+nbr_channels))), stack_path, normalize_input=False).astype(float)
 			target_img = frames[:,:,target_channel_index].copy()
+			
 			correction = field_correction(target_img, threshold_on_std=threshold_on_std, operation=operation, model=model, clip=clip, activation_protocol=activation_protocol)
 			frames[:,:,target_channel_index] = correction.copy()
 
@@ -981,6 +984,8 @@ def field_correction(img, threshold_on_std=1, operation='divide', model='parabol
 	"""
 
 	target_copy = img.copy().astype(float)
+	if np.percentile(target_copy.flatten(),99.9)==0.0:
+		return target_copy
 
 	std_frame = filter_image(target_copy,filters=activation_protocol)
 	edge = estimate_unreliable_edge(activation_protocol)
@@ -1050,4 +1055,168 @@ def fit_background_model(img, cell_masks=None, model='paraboloid', edge_exclusio
 	if bg is not None:
 		bg = np.array(bg)
 
-	return bg
+	return bg
+
+
+def correct_channel_offset(
+						   experiment, 
+						   well_option='*',
+						   position_option='*',
+						   target_channel="channel_name",
+						   correction_horizontal = 0,
+						   correction_vertical = 0,
+						   show_progress_per_well = True,
+						   show_progress_per_pos = False,
+						   export = False,
+						   return_stacks = False,
+						   movie_prefix=None,
+						   export_prefix='Corrected',
+						   return_stack = True,
+						   **kwargs,
+						   ):
+
+
+	config = get_config(experiment)
+	wells = get_experiment_wells(experiment)
+	len_movie = float(ConfigSectionMap(config,"MovieSettings")["len_movie"])
+	if movie_prefix is None:
+		movie_prefix = ConfigSectionMap(config,"MovieSettings")["movie_prefix"]	
+
+	well_indices, position_indices = interpret_wells_and_positions(experiment, well_option, position_option)
+	channel_indices = _extract_channel_indices_from_config(config, [target_channel])
+	nbr_channels = _extract_nbr_channels_from_config(config)
+	img_num_channels = _get_img_num_per_channel(channel_indices, int(len_movie), nbr_channels)
+	
+	stacks = []
+
+	for k, well_path in enumerate(tqdm(wells[well_indices], disable=not show_progress_per_well)):
+		
+		well_name, _ = extract_well_name_and_number(well_path)
+		positions = get_positions_in_well(well_path)
+		selection = positions[position_indices]
+		if isinstance(selection[0],np.ndarray):
+			selection = selection[0]
+
+		for pidx,pos_path in enumerate(tqdm(selection, disable=not show_progress_per_pos)):
+			
+			stack_path = get_position_movie_path(pos_path, prefix=movie_prefix)
+			print(f'Applying the correction to position {extract_position_name(pos_path)}...')
+			len_movie_auto = auto_load_number_of_frames(stack_path)
+			if len_movie_auto is not None:
+				len_movie = len_movie_auto
+				img_num_channels = _get_img_num_per_channel(channel_indices, int(len_movie), nbr_channels)
+
+			corrected_stack = correct_channel_offset_single_stack(stack_path, 
+														target_channel_index=channel_indices[0],
+														nbr_channels=nbr_channels,
+														stack_length=len_movie,
+														correction_vertical=correction_vertical,
+														correction_horizontal=correction_horizontal,
+														export=export,
+														prefix=export_prefix,
+														return_stacks = return_stacks,
+													  )
+			print('Correction successful.')
+			if return_stacks:
+				stacks.append(corrected_stack)
+			else:
+				del corrected_stack
+			collect()
+
+	if return_stacks:
+		return stacks
+
+
+def correct_channel_offset_single_stack(stack_path,
+										target_channel_index=0,
+										nbr_channels=1,
+										stack_length=45,
+										correction_vertical=0,
+										correction_horizontal=0,
+										export=False,
+										prefix="Corrected",
+										return_stacks=True,
+											):
+
+	assert os.path.exists(stack_path),f"The stack {stack_path} does not exist... Abort."
+
+	stack_length_auto = auto_load_number_of_frames(stack_path)
+	if stack_length_auto is None and stack_length is None:
+		print('Stack length not provided...')
+		return None
+	if stack_length_auto is not None:
+		stack_length = stack_length_auto
+
+	corrected_stack = []
+
+	if export:
+		path,file = os.path.split(stack_path)
+		if prefix is None:
+			newfile = 'temp_'+file
+		else:
+			newfile = '_'.join([prefix,file])
+		
+		with tiff.TiffWriter(os.sep.join([path,newfile]),imagej=True) as tif:
+
+			for i in tqdm(range(0,int(stack_length*nbr_channels),nbr_channels)):
+				
+				frames = load_frames(list(np.arange(i,(i+nbr_channels))), stack_path, normalize_input=False).astype(float)
+				target_img = frames[:,:,target_channel_index].copy()
+
+				if np.percentile(target_img.flatten(), 99.9)==0.0:
+					correction = target_img
+				elif np.any(target_img.flatten()!=target_img.flatten()):
+					# Routine to interpolate NaN for the spline filter then mask it again
+					target_interp = interpolate_nan(target_img)
+					correction = shift(target_interp, [correction_vertical, correction_horizontal])
+					correction_nan = shift(target_img, [correction_vertical, correction_horizontal], prefilter=False)
+					nan_i, nan_j = np.where(correction_nan!=correction_nan)
+					correction[nan_i, nan_j] = np.nan
+				else:
+					correction = shift(target_img, [correction_vertical, correction_horizontal])					
+
+				frames[:,:,target_channel_index] = correction.copy()
+
+				if return_stacks:
+					corrected_stack.append(frames)
+
+				if export:
+					tif.write(np.moveaxis(frames,-1,0).astype(np.dtype('f')), contiguous=True)
+				del frames
+				del target_img
+				del correction
+				collect()
+
+		if prefix is None:
+			os.replace(os.sep.join([path,newfile]), os.sep.join([path,file]))
+	else:
+		for i in tqdm(range(0,int(stack_length*nbr_channels),nbr_channels)):
+			
+			frames = load_frames(list(np.arange(i,(i+nbr_channels))), stack_path, normalize_input=False).astype(float)
+			target_img = frames[:,:,target_channel_index].copy()
+			
+			if np.percentile(target_img.flatten(), 99.9)==0.0:
+				correction = target_img
+			elif np.any(target_img.flatten()!=target_img.flatten()):
+				# Routine to interpolate NaN for the spline filter then mask it again
+				target_interp = interpolate_nan(target_img)
+				correction = shift(target_interp, [correction_vertical, correction_horizontal])
+				correction_nan = shift(target_img, [correction_vertical, correction_horizontal], prefilter=False)
+				nan_i, nan_j = np.where(correction_nan!=correction_nan)
+				correction[nan_i, nan_j] = np.nan
+			else:
+				correction = shift(target_img, [correction_vertical, correction_horizontal])
+
+			frames[:,:,target_channel_index] = correction.copy()
+
+			corrected_stack.append(frames)
+
+			del frames
+			del target_img
+			del correction
+			collect()
+
+	if return_stacks:
+		return np.array(corrected_stack)
+	else:
+		return None

celldetective/relative_measurements.py

@@ -90,6 +90,10 @@ def measure_pairs(pos, neighborhood_protocol):
 				cosine_dot_vector[:] = np.nan
 
 				coords_neighbor = group_neigh[['POSITION_X', 'POSITION_Y']].to_numpy()[0]
+				intersection = np.nan
+				if 'intersection' in list(group_neigh.columns):
+					intersection = group_neigh['intersection'].values[0]
+
 				neighbor_vector[0] = coords_neighbor[0] - coords_reference[0]
 				neighbor_vector[1] = coords_neighbor[1] - coords_reference[1]
 
@@ -109,7 +113,7 @@ def measure_pairs(pos, neighborhood_protocol):
 							{'REFERENCE_ID': tid, 'NEIGHBOR_ID': nc,
 							'reference_population': reference_population,
 							'neighbor_population': neighbor_population,
-							'FRAME': t, 'distance': relative_distance,
+							'FRAME': t, 'distance': relative_distance, 'intersection': intersection,
 							'angle': angle * 180 / np.pi,
 							f'status_{neighborhood_description}': 1,
 							f'class_{neighborhood_description}': 0,
@@ -201,9 +205,14 @@ def measure_pair_signals_at_position(pos, neighborhood_protocol, velocity_kwargs
 			neighbor_dicts = group.loc[: , f'{neighborhood_description}'].values
 			timeline_reference = group['FRAME'].to_numpy()
 			coords_reference = group[['POSITION_X', 'POSITION_Y']].to_numpy()
+			if "area" in list(group.columns):
+				ref_area = group['area'].to_numpy()
+			else:
+				ref_area = [np.nan]*len(coords_reference)
 
 			neighbor_ids = []
 			neighbor_ids_per_t = []
+			intersection_values = []
 
 			time_of_first_entrance_in_neighborhood = {}
 			t_departure={}
@@ -218,10 +227,16 @@ def measure_pair_signals_at_position(pos, neighborhood_protocol, velocity_kwargs
 					for neigh in neighbors_at_t:
 						if neigh['id'] not in neighbor_ids:
 							time_of_first_entrance_in_neighborhood[neigh['id']]=t
+						if 'intersection' in neigh:
+							intersection_values.append({"frame": t, "neigh_id": neigh['id'], "intersection": neigh['intersection']})
+						else:
+							intersection_values.append({"frame": t, "neigh_id": neigh['id'], "intersection": np.nan})
 						neighbor_ids.append(neigh['id'])
 						neighs_t.append(neigh['id'])
 				neighbor_ids_per_t.append(neighs_t)
 
+			intersection_values = pd.DataFrame(intersection_values)
+
 			#print(neighbor_ids_per_t)
 			unique_neigh = list(np.unique(neighbor_ids))
 			print(f'Reference cell {tid}: found {len(unique_neigh)} neighbour cells: {unique_neigh}...')
@@ -232,6 +247,11 @@ def measure_pair_signals_at_position(pos, neighborhood_protocol, velocity_kwargs
 				
 				coords_neighbor = group_neigh[['POSITION_X', 'POSITION_Y']].to_numpy()
 				timeline_neighbor = group_neigh['FRAME'].to_numpy()
+				if "area" in list(group_neigh.columns):
+					neigh_area = group_neigh['area'].to_numpy()
+				else:
+					neigh_area = [np.nan]*len(timeline_neighbor)
+
 
 				# # Perform timeline matching to have same start-end points and no gaps
 				full_timeline, _, _ = timeline_matching(timeline_reference, timeline_neighbor)
@@ -239,6 +259,9 @@ def measure_pair_signals_at_position(pos, neighborhood_protocol, velocity_kwargs
 				neighbor_vector = np.zeros((len(full_timeline), 2))
 				neighbor_vector[:,:] = np.nan
 
+				intersection_vector = np.zeros((len(full_timeline)))
+				intersection_vector[:] = np.nan
+
 				centre_of_mass_columns = [(c,c.replace('POSITION_X','POSITION_Y')) for c in list(neighbor_properties.columns) if c.endswith('centre_of_mass_POSITION_X')]
 				centre_of_mass_labels = [c.replace('_centre_of_mass_POSITION_X','') for c in list(neighbor_properties.columns) if c.endswith('centre_of_mass_POSITION_X')]
 
@@ -319,7 +342,20 @@ def measure_pair_signals_at_position(pos, neighborhood_protocol, velocity_kwargs
 
 					if t in timeline_reference: # meaning position exists on both sides
 
-						idx_reference = list(timeline_reference).index(t) 
+						idx_reference = list(timeline_reference).index(t)
+						inter = intersection_values.loc[(intersection_values['neigh_id']==nc)&(intersection_values["frame"]==t),"intersection"].values
+						if len(inter)==0:
+							inter = np.nan
+						else:
+							inter = inter[0]
+
+						neigh_inter_fraction = np.nan
+						if inter==inter and neigh_area[t]==neigh_area[t]:
+							neigh_inter_fraction = inter / neigh_area[t]
+
+						ref_inter_fraction = np.nan
+						if inter==inter and ref_area[t]==ref_area[t]:
+							ref_inter_fraction = inter / ref_area[t]						
 
 						if nc in neighbor_ids_per_t[idx_reference]:
 
@@ -328,7 +364,7 @@ def measure_pair_signals_at_position(pos, neighborhood_protocol, velocity_kwargs
 									{'REFERENCE_ID': tid, 'NEIGHBOR_ID': nc,
 									'reference_population': reference_population,
 									'neighbor_population': neighbor_population,
-									'FRAME': t, 'distance': relative_distance[t],
+									'FRAME': t, 'distance': relative_distance[t], 'intersection': inter, 'reference_frac_area_intersection': ref_inter_fraction, 'neighbor_frac_area_intersection': neigh_inter_fraction,
 									'velocity': rel_velocity[t],
 									'velocity_smooth': rel_velocity_long_timescale[t], 
 									'angle': angle[t] * 180 / np.pi,
@@ -349,7 +385,7 @@ def measure_pair_signals_at_position(pos, neighborhood_protocol, velocity_kwargs
 									{'REFERENCE_ID': tid, 'NEIGHBOR_ID': nc,
 									'reference_population': reference_population,
 									'neighbor_population': neighbor_population,
-									'FRAME': t, 'distance': relative_distance[t],
+									'FRAME': t, 'distance': relative_distance[t], 'intersection': inter, 'reference_frac_area_intersection': ref_inter_fraction, 'neighbor_frac_area_intersection': neigh_inter_fraction,
 									'velocity': rel_velocity[t], 
 									'velocity_smooth': rel_velocity_long_timescale[t],
 									'angle': angle[t] * 180 / np.pi,
@@ -642,4 +678,92 @@ def extract_neighborhood_settings(neigh_string, population='targets'):
 	return neigh_protocol
 
 
+def expand_pair_table(data):
+
+	"""
+	Expands a pair table by merging reference and neighbor trajectory data from CSV files based on the specified 
+	reference and neighbor populations, and their associated positions and frames.
+
+	Parameters
+	----------
+	data : pandas.DataFrame
+		DataFrame containing the pair table, which should include the columns:
+		- 'reference_population': The reference population type.
+		- 'neighbor_population': The neighbor population type.
+		- 'position': The position identifier for each pair.
+
+	Returns
+	-------
+	pandas.DataFrame
+		Expanded DataFrame that includes merged reference and neighbor data, sorted by position, reference population, 
+		neighbor population, and frame. Rows without values in 'REFERENCE_ID', 'NEIGHBOR_ID', 'reference_population', 
+		or 'neighbor_population' are dropped.
+
+	Notes
+	-----
+	- For each unique pair of `reference_population` and `neighbor_population`, the function identifies corresponding 
+	  trajectories CSV files based on the position identifier.
+	- The function reads the trajectories CSV files, prefixes columns with 'reference_' or 'neighbor_' to avoid 
+	  conflicts, and merges data from reference and neighbor tables based on `TRACK_ID` or `ID`, and `FRAME`.
+	- Merges are performed in an outer join manner to retain all rows, regardless of missing values in the target files.
+	- The final DataFrame is sorted and cleaned to ensure only valid pairings are included.
+
+	Example
+	-------
+	>>> expanded_df = expand_pair_table(pair_table)
+	>>> expanded_df.head()
+
+	Raises
+	------
+	AssertionError
+		If 'reference_population' or 'neighbor_population' is not found in the columns of `data`.
+	"""
+
+	assert 'reference_population' in list(data.columns),"Please provide a valid pair table..."
+	assert 'neighbor_population' in list(data.columns),"Please provide a valid pair table..."
+
+	expanded_table = []
+	
+	for neigh, group in data.groupby(['reference_population','neighbor_population']):
+		
+		ref_pop = neigh[0]; neigh_pop = neigh[1];
+
+		for pos,pos_group in group.groupby('position'):
+			
+			ref_tab = os.sep.join([pos,'output','tables',f'trajectories_{ref_pop}.csv'])
+			neigh_tab = os.sep.join([pos,'output','tables',f'trajectories_{neigh_pop}.csv'])
+			
+			if os.path.exists(ref_tab):
+				df_ref = pd.read_csv(ref_tab)
+				if 'TRACK_ID' in df_ref.columns:
+					if not np.all(df_ref['TRACK_ID'].isnull()):
+						ref_merge_cols = ['TRACK_ID','FRAME']
+					else:
+						ref_merge_cols = ['ID','FRAME']
+				else:
+					ref_merge_cols = ['ID','FRAME']
+
+			if os.path.exists(neigh_tab):
+				df_neigh = pd.read_csv(neigh_tab)
+				if 'TRACK_ID' in df_neigh.columns:
+					if not np.all(df_neigh['TRACK_ID'].isnull()):
+						neigh_merge_cols = ['TRACK_ID','FRAME']
+					else:
+						neigh_merge_cols = ['ID','FRAME']
+				else:
+					neigh_merge_cols = ['ID','FRAME']
+
+			df_ref = df_ref.add_prefix('reference_',axis=1)
+			df_neigh = df_neigh.add_prefix('neighbor_',axis=1)
+			ref_merge_cols = ['reference_'+c for c in ref_merge_cols]
+			neigh_merge_cols = ['neighbor_'+c for c in neigh_merge_cols]
+
+			merge_ref = pos_group.merge(df_ref, how='outer', left_on=['REFERENCE_ID','FRAME'], right_on=ref_merge_cols, suffixes=('', '_reference'))
+			merge_neigh = merge_ref.merge(df_neigh, how='outer', left_on=['NEIGHBOR_ID','FRAME'], right_on=neigh_merge_cols, suffixes=('_reference', '_neighbor'))
+			expanded_table.append(merge_neigh)
+
+	df_expanded = pd.concat(expanded_table, axis=0, ignore_index = True)
+	df_expanded = df_expanded.sort_values(by=['position', 'reference_population','neighbor_population','REFERENCE_ID','NEIGHBOR_ID','FRAME'])
+	df_expanded = df_expanded.dropna(axis=0, subset=['REFERENCE_ID','NEIGHBOR_ID','reference_population','neighbor_population'])
 
+	return df_expanded

celldetective/scripts/measure_cells.py

@@ -251,14 +251,14 @@ def measure_index(indices):
 			iso_table = measure_isotropic_intensity(positions_at_t, img, channels=channel_names, intensity_measurement_radii=intensity_measurement_radii, column_labels=column_labels, operations=isotropic_operations, verbose=False)
 
 		if do_iso_intensities and do_features:
-			measurements_at_t = iso_table.merge(feature_table, how='outer', on='class_id',suffixes=('', '_delme'))
+			measurements_at_t = iso_table.merge(feature_table, how='outer', on='class_id',suffixes=('_delme', ''))
 			measurements_at_t = measurements_at_t[[c for c in measurements_at_t.columns if not c.endswith('_delme')]]
 		elif do_iso_intensities * (not do_features):
 			measurements_at_t = iso_table
 		elif do_features:
-			measurements_at_t = positions_at_t.merge(feature_table, how='outer', on='class_id',suffixes=('', '_delme'))
+			measurements_at_t = positions_at_t.merge(feature_table, how='outer', on='class_id',suffixes=('_delme', ''))
 			measurements_at_t = measurements_at_t[[c for c in measurements_at_t.columns if not c.endswith('_delme')]]
-	
+
 		center_of_mass_x_cols = [c for c in list(measurements_at_t.columns) if c.endswith('centre_of_mass_x')]
 		center_of_mass_y_cols = [c for c in list(measurements_at_t.columns) if c.endswith('centre_of_mass_y')]
 		for c in center_of_mass_x_cols: