insituTEM 0.1.6__py3-none-any.whl → 0.1.7__py3-none-any.whl

insituTEM/__ init __.py

@@ -0,0 +1,14 @@
+# __init__.py
+__version__ = '1.0.0'
+__author__ = 'Meng Li'
+
+import insitu_alignment as AL
+import insitu_DP as DP
+import insitu_IO as IO
+import insitu_Preprocess as PP
+import insitu_Diff as Diff
+import insitu_EMraw as EM
+import insitu_DENS as DENS
+
+
+__all__ = ['function1', 'Class2']

insituTEM/insitu_DENS.py

@@ -0,0 +1,318 @@
+"""
+in-situ TEM Toolbox - DENS holder data process
+
+Assembles of functions related to process raw data by DENS holders
+
+Example:
+    
+    from insitu_TEM import insitu_DENS as DENS
+    DENS.XXX
+
+Created on Tue May 13 2025
+@author: Meng Li mli4@bnl.gov
+
+"""
+
+"""
+Codes for Stream data process
+"""
+import pandas as pd
+
+def read_pstrace_csv(filepath):
+    # Try several encodings until one works
+    for encoding in ['utf-16', 'utf-8', 'ISO-8859-1']:
+        try:
+            with open(filepath, 'r', encoding=encoding) as f:
+                lines = [line.strip() for line in f if line.strip()]
+            break
+        except UnicodeDecodeError:
+            continue
+    else:
+        raise UnicodeDecodeError("Failed to decode file using common encodings.")
+
+    # Detect measurement type (e.g., CV I vs E Scan 1)
+    measurement_type = None
+    for line in lines:
+        if "vs" in line and not line.lower().startswith("date"):
+            measurement_type = line
+            break
+
+    # Detect actual column header (like "V,µA" or "s,V")
+    header_index = None
+    for i, line in enumerate(lines):
+        if ',' in line and all(any(c.isalpha() for c in token) for token in line.split(',')):
+            header_index = i
+            break
+
+    if header_index is None:
+        raise ValueError("Could not find a valid data header (e.g., 'V,µA' or 's,V')")
+
+    headers = [h.strip() for h in lines[header_index].split(',')]
+    data_lines = lines[header_index + 1:]
+
+    # Read and convert numeric data only
+    data = []
+    for line in data_lines:
+        try:
+            row = [float(val) for val in line.split(',')]
+            data.append(row)
+        except ValueError:
+            continue  # skip non-numeric lines
+
+    df = pd.DataFrame(data, columns=headers)
+    return measurement_type, df
+
+
+import matplotlib.pyplot as plt
+
+def plot_multiple_xy(datasets, labels=None, xlabel=None, ylabel=None, title=None):
+    """
+    Plot multiple datasets (each as a 2-column DataFrame) on one plot.
+
+    Args:
+        datasets: list of DataFrames with 2 columns
+        labels: list of labels for each line
+        xlabel, ylabel: optional axis labels
+        title: optional plot title
+    """
+    fig, ax = plt.subplots(figsize=(6, 4), dpi=300)
+
+    colors = ['blue', 'green', 'red', 'orange', 'purple']
+
+    for i, df in enumerate(datasets):
+        x = df.iloc[:, 0]
+        y = df.iloc[:, 1]
+        label = labels[i] if labels else None
+        color = colors[i % len(colors)]
+        ax.plot(x, y, label=label, color=color, linewidth=1)
+
+    # X and Y = 0 guide lines
+    ax.axhline(0, color='gray', linestyle='-', linewidth=0.5)
+    ax.axvline(0, color='gray', linestyle='-', linewidth=0.5)
+
+    # Axis style: boxed
+    for side in ['top', 'bottom', 'left', 'right']:
+        ax.spines[side].set_linewidth(1)
+        ax.spines[side].set_color('black')
+
+    # Axis labels
+    ax.set_xlabel(xlabel if xlabel else datasets[0].columns[0], fontsize=12, fontweight='bold', labelpad=10)
+    ax.set_ylabel(ylabel if ylabel else datasets[0].columns[1], fontsize=12, fontweight='bold', labelpad=10)
+
+    # Axis limits
+        # Extract x/y and lengths
+    x_list = [df.iloc[:, 0].values for df in datasets]
+    y_list = [df.iloc[:, 1].values for df in datasets]
+    all_x = np.concatenate(x_list)
+    all_y = np.concatenate(y_list)
+    # ax.set_xlim(all_x.min()*1.1, all_x.max()*1.1)
+    # ax.set_ylim(all_y.min()*1.2, all_y.max()*1.1)
+        # Add 10% margin to x and y limits
+    x_margin = (all_x.max() - all_x.min()) * 0.05
+    y_margin = (all_y.max() - all_y.min()) * 0.05
+    
+    ax.set_xlim(all_x.min() - x_margin, all_x.max() + x_margin)
+    ax.set_ylim(all_y.min() - y_margin, all_y.max() + y_margin)
+
+
+    # Ticks and layout
+    ax.tick_params(direction='in', top=True, right=True)
+    ax.grid(False)
+
+    # Title and legend
+    if title:
+        ax.set_title(title, fontsize=12,fontweight='bold', pad=10)
+    if labels:
+        ax.legend()
+
+    plt.tight_layout()
+    plt.show()
+
+
+mport matplotlib.pyplot as plt
+
+def plot_xy(df, measurement_type=None):
+    """
+    Plot single xy curve
+    """
+    x_col = df.columns[0]
+    y_col = df.columns[1]
+    x = df[x_col]
+    y = df[y_col]
+
+    fig, ax = plt.subplots(figsize=(6,4),dpi=300)  # square aspect ratio
+
+    # Plot data
+    ax.plot(x, y, linewidth=1, color='blue')
+
+    # Keep left and bottom axis
+    ax.spines['left'].set_visible(True)
+    ax.spines['bottom'].set_visible(True)
+
+    # Show top and right spine to make a box
+    ax.spines['top'].set_visible(True)
+    ax.spines['right'].set_visible(True)
+
+    # Draw all 4 spines in black (like a square)
+    for side in ['top', 'bottom', 'left', 'right']:
+        ax.spines[side].set_linewidth(1)
+        ax.spines[side].set_color('black')
+
+    # Add x=0 and y=0 guide lines
+    ax.axhline(0, color='orange', linestyle='-', linewidth=0.5)
+    ax.axvline(0, color='orange', linestyle='-', linewidth=0.5)
+
+    # Axis label styling
+    ax.set_xlabel('Voltage (V)', fontsize=12, fontweight='bold', labelpad=10)
+    ax.set_ylabel('Current (uA)' , fontsize=12, fontweight='bold', labelpad=10)
+
+    # Title
+    if measurement_type:
+        ax.set_title(measurement_type, fontsize=12, fontweight='bold', pad=10)
+
+    # Tick styling
+    ax.tick_params(direction='in', top=True, right=True)
+
+    # Turn off grid
+    ax.grid(False)
+
+    plt.tight_layout()
+    plt.show()
+
+
+# Convert the plot to movie
+
+import matplotlib.pyplot as plt
+import matplotlib.animation as animation
+import numpy as np
+import cv2
+import tifffile
+from matplotlib.backends.backend_agg import FigureCanvasAgg
+import tqdm
+
+def animate_CVplots(datasets, labels=["Scan 1", "Scan 2", "Scan 3"], xlabel="Voltage (V)", ylabel="Current (µA)", title="CV Scan",
+                       fps=60, save_basename="cv_animation"):
+    """
+    Function to animate the plot
+      datasets,             # List of DataFrames, each with 2 columns (x, y) for a scan
+    labels=None,          # Optional list of labels for each scan (used in legend)
+    xlabel=None,          # Optional label for x-axis (default: first column name)
+    ylabel=None,          # Optional label for y-axis (default: second column name)
+    title=None,           # Optional plot title
+    fps=50,          # fps of the input data. Original speed: 4 fps
+    save_basename="cv_animation"  # Prefix for saved output files (AVI + TIFF stack)
+
+    """
+     # Setup plot and canvas
+    fig, ax = plt.subplots(figsize=(6, 4), dpi=150)
+    fig.subplots_adjust(left=0.15, right=0.95, top=0.9, bottom=0.15)
+    canvas = FigureCanvasAgg(fig)
+
+    # Extract x/y and lengths
+    x_list = [df.iloc[:, 0].values for df in datasets]
+    y_list = [df.iloc[:, 1].values for df in datasets]
+    lengths = [len(x) for x in x_list]
+    cum_lengths = np.cumsum(lengths)
+    total_frames = cum_lengths[-1]
+
+    # Axis formatting
+    for side in ['top', 'bottom', 'left', 'right']:
+        ax.spines[side].set_linewidth(1)
+        ax.spines[side].set_color('black')
+        
+    ax.axhline(0, color='gray', linewidth=0.5)
+    ax.axvline(0, color='gray', linewidth=0.5)
+    ax.tick_params(direction='in', top=True, right=True)
+    ax.set_xlabel(xlabel or datasets[0].columns[0], fontsize=12, fontweight='bold', labelpad=10)
+    ax.set_ylabel(ylabel or datasets[0].columns[1], fontsize=12, fontweight='bold', labelpad=10)
+    if title:
+        ax.set_title(title, fontsize=10)
+
+    # Prepare plot elements
+    colors = ['blue', 'green', 'red', 'purple']
+    lines, tips = [], []
+    for i in range(len(datasets)):
+        line, = ax.plot([], [], color=colors[i % len(colors)], label=(labels[i] if labels else None), linewidth=1)
+        tip, = ax.plot([], [], color=colors[i % len(colors)], marker='o', markersize=4)
+        lines.append(line)
+        tips.append(tip)
+
+    # Add live current value in top-left
+    value_text = ax.text(0.02, 0.95, '', transform=ax.transAxes,
+                         fontsize=10, color='black', ha='left', va='top')
+
+
+    if labels:
+        ax.legend()
+
+    # Axis limits
+    all_x = np.concatenate(x_list)
+    all_y = np.concatenate(y_list)
+
+    # Add 10% margin to x and y limits
+    x_margin = (all_x.max() - all_x.min()) * 0.05
+    y_margin = (all_y.max() - all_y.min()) * 0.05
+    
+    ax.set_xlim(all_x.min() - x_margin, all_x.max() + x_margin)
+    ax.set_ylim(all_y.min() - y_margin, all_y.max() + y_margin)
+    # ax.set_xlim(all_x.min()*1.1, all_x.max()*1.1)
+    # ax.set_ylim(all_y.min()*1.2, all_y.max()*1.1)
+
+    # Video writer setup
+    canvas.draw()
+    width, height = canvas.get_width_height()
+    # fps=int(1000/interval)
+    aviout=save_basename+"_"+str(fps)+".avi"
+    out = cv2.VideoWriter(aviout, cv2.VideoWriter_fourcc(*'MJPG'), fps, (width, height))
+    # tiff_frames = []
+
+    # Setup TIFF writer with BigTIFF support
+    tif_writer = tifffile.TiffWriter(f"{save_basename}.tif", bigtiff=True)
+    
+    # Render frames
+    print("Plotting the curves into movies~")
+    for frame in tqdm.tqdm(range(total_frames)):
+        ax.set_title(title, fontsize=12,fontweight='bold')  # reset title in case of matplotlib bugs
+
+        for i, (x, y, line, tip) in enumerate(zip(x_list, y_list, lines, tips)):
+            start = cum_lengths[i - 1] if i > 0 else 0
+            end = cum_lengths[i]
+            if frame < start:
+                # Not started yet
+                line.set_data([], [])
+                tip.set_data([], [])
+            elif frame >= end:
+                # Completed
+                line.set_data(x, y)
+                tip.set_data([], [])
+            else:
+                # Active scan
+                idx = frame - start + 1
+                line.set_data(x[:idx], y[:idx])
+                tip.set_data([x[idx - 1]], [y[idx - 1]])
+                value_text.set_text(
+                    f"{labels[i] if labels else f'Scan {i+1}'}\nV = {x[idx - 1]:.3f} V\nI = {y[idx - 1]:.3f} µA"
+                )
+
+        # Render and convert to image
+        canvas.draw()
+        buf = np.frombuffer(canvas.tostring_rgb(), dtype=np.uint8)
+        image = buf.reshape((height, width, 3))
+        image_bgr = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
+        out.write(image_bgr)
+        # Save TIFF frame (with compression)
+        tif_writer.write(image, photometric='rgb', compression='deflate')
+
+    out.release()
+    tif_writer.close()
+
+    # tifffile.imwrite(f"{save_basename}.tif", tiff_frames, photometric='rgb',compression='deflate')
+    print("Conversion done!")
+    print(f"Saved AVI: {save_basename}.avi")
+    print(f"Saved TIFF stack: {save_basename}.tif")
+
+from tkinter import Tk, filedialog
+def select_multiple_files(filetypes=(("csv files", "*.csv"), ("All files", "*.*"))):
+    Tk().withdraw()  # Hide the root window
+    filepaths = filedialog.askopenfilenames(title="Select PSTrace .csv files", filetypes=filetypes)
+    return list(filepaths)  # Convert from tuple to list

insituTEM/insitu_DP.py

@@ -10,7 +10,7 @@ Example:
     IO.f2tif(path,1)
 
 Created on Tue Jun 11 10:25:25 2019
-@author: Mona
+@author: Meng Li
 """
 import numpy as np
 
@@ -57,12 +57,12 @@ def readcsv(path,col_list):
     return D
 
     
-def writeCSV(pathout,data):
-#    import csv
-    import numpy as np
-#    (length,width)=np.shape(data)
-#    np.savetxt(pathout, data, fmt='%1.4d', delimiter=",") 
-    np.savetxt(pathout, data, fmt='%.4f', delimiter=",")   
+# def writeCSV(pathout,data):
+# #    import csv
+#     import numpy as np
+# #    (length,width)=np.shape(data)
+# #    np.savetxt(pathout, data, fmt='%1.4d', delimiter=",") 
+#     np.savetxt(pathout, data, fmt='%.4f', delimiter=",")   
     
 def plot2ani(x,y,outpath,label,w=800,h=600,fps=10,dpi=72,tl_size=20,c='r'):
     """
@@ -158,4 +158,224 @@ def AutoROIbg(img):
     else:
         ROI = 0
     return ROI
-            
+            
+def readscale(path):
+    """
+    Function to measure scalebar from movie first frame
+    Input: path - movie path
+    scalebarcolr- 0 for black, 255 for white
+    """
+    import os
+    os.environ["IMAGEIO_FFMPEG_EXE"] = "/opt/homebrew/bin/ffmpeg"
+    import moviepy.editor as mp
+    import cv2
+    
+    video = mp.VideoFileClip(path)
+    fr = video.get_frame(mp.cvsecs(0))
+    fr = cv2.cvtColor(fr, cv2.COLOR_BGR2GRAY)
+
+    # Select ROI using cv2.selectROI()
+    roi = cv2.selectROI('Select scalebar ROI, quit with SPACE', fr)
+    cv2.waitKey(10)
+    # cv2.destroyAllWindows()
+    cv2.destroyWindow('Select scalebar ROI, quit with SPACE')
+
+    # print(roi)
+    # Crop the image based on the selected ROI
+    cropped_image = fr[int(roi[1]):int(roi[1] + roi[3]), int(roi[0]):int(roi[0] + roi[2])]
+
+    # # # Display the cropped image
+    # from PIL import Image
+    # from IPython.display import display
+    # pil_image= Image.fromarray(cropped_image)
+    # display(pil_image)
+
+    # plt.imshow(cropped_image)
+
+    IO.showFr(cropped_image)
+    #calculate the number of pixels in scalebar
+    # print(cropped_image[-5,:])
+    import numpy as np
+
+    scalebar_color = int(input("Input the scalebar color,  0-black, 255-white: "))
+
+
+    scalebar_px =  np.count_nonzero(cropped_image[-10,:]==scalebar_color)
+
+    print("scalebar_px=", scalebar_px)
+
+    scale_nm = int(input("Input the scalebar in nm: "))
+    scale =scalebar_px/scale_nm
+    print("scale=",scale,"px/nm")
+    return scale
+
+def next_power_of_2(x):
+    return 1 if x == 0 else 2**(x - 1).bit_length()
+
+def pad_to_nearest_power_of_2(image):
+    import cv2
+    # Get original dimensions
+    h, w = image.shape
+    size=max(h,w)
+
+    # Calculate the nearest power of 2 dimensions
+    new_size = next_power_of_2(size)
+
+    # Calculate padding amounts
+    top_pad = (new_size - h) // 2
+    bottom_pad = new_size - h - top_pad
+    left_pad = (new_size - w) // 2
+    right_pad = new_size - w - left_pad
+
+    # Apply padding
+    padded_image = cv2.copyMakeBorder(
+        image,
+        top=top_pad, bottom=bottom_pad,
+        left=left_pad, right=right_pad,
+        borderType=cv2.BORDER_CONSTANT,
+        value=0  # Black color
+    )
+
+    return padded_image
+
+def getFFT(im):
+    """
+    Function to get FFT from grayscale image, auto pad the image to keep the image size  2^n
+    Input: grayscale image (np array)
+    Output: grayscale FFT with 2^n size, 32 bit
+    """
+
+    pad_im=pad_to_nearest_power_of_2(im)
+
+
+    fft_image = np.fft.fft2(pad_im)
+    fft_shifted = np.fft.fftshift(fft_image)
+    fft = 20 * np.log(np.abs(fft_shifted))
+    return fft
+
+def FFTstack(path):
+    """
+    Function to get FFT of the stack
+    input: path of the image stack
+    output: FFT stack with 2^n size 8 bit images
+    """
+    import tifffile
+    import tqdm
+    import cv2
+
+    stack = tifffile.imread(path)
+    nFrames,h,w = stack.shape #make sure the order of h and w!
+    pathout=path[:-4]+'_FFT.tif'
+
+    print("------------------------------------------")
+    print('Calculating FFT of each frame')
+    with tifffile.TiffWriter(pathout,bigtiff=True) as tif:
+        for i in tqdm.tqdm(range(nFrames)):
+            fr=stack[i]
+            fft=getFFT(fr)
+            #convert the 32 bit image to 8 bit
+            normalized_image = cv2.normalize(fft, None, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX)
+            # Convert the normalized image to 8-bit
+            fft_8bit = np.uint8(normalized_image)
+            tif.write(fft_8bit, contiguous=True)
+
+def FFT2Polar(fft):
+    """
+    Function to convert FFT to polar image, y= 180˚, x= Rho
+    Input: grayscale fft image (np array)
+    """
+    import cv2
+    #function to convert fft image to polar coordinates from the center from 0-180 deg
+    center = (fft.shape[1] // 2, fft.shape[0] // 2)
+    polar_fft = cv2.linearPolar(fft, center, center[0], cv2.WARP_FILL_OUTLIERS)
+    # polar_fft= polar_fft[:,:center[1]]
+    #resize the image to realistic size
+    resized_polar_fft = cv2.resize(polar_fft, (center[0], 360))
+        # plt.imshow(resized_polar_fft)
+    
+    p_fft1=resized_polar_fft[:180,:]
+    p_fft2=resized_polar_fft[180:,:]
+    p_fft=p_fft1+p_fft2
+    return p_fft
+
+def FFTidentify(fft,pathout,scale=1,sigmaMax=2.5,sigmaMin=2.4,thrs=0.96,sigma_ratio=10):
+    """
+    Function to identify DPs from FFT 
+    input: 
+        fft: image
+        scale: scale of the original image : 1 px  = XXX nm
+        parameters for blobs detection: sigmaMax, sigmaMin, thrs, sigma_ratio
+        pathout: path for output image (.tif) and csv
+
+    output:
+        idnetified image
+        indexed csv
+
+
+
+    """
+    import math
+    import matplotlib.pyplot as plt
+    from scipy import ndimage
+    from skimage.feature import blob_dog, blob_log, blob_doh
+    from math import sqrt
+    from insituTEM import insitu_IO as IO
+    
+    h,w = fft.shape
+    x_c=int(w/2)
+    y_c=int(h/2)
+    r=int(x_c/2)   
+    if w>320:
+        fft_crop=fft[x_c-r:x_c+r,y_c-r:y_c+r]
+        image= ndimage.maximum_filter(fft_crop, size=2, mode='wrap')
+    else:
+        fft_crop = fft
+        image= ndimage.maximum_filter(fft_crop, size=3, mode='wrap')
+
+
+
+
+    blobs_dog = blob_dog(image, max_sigma=sigmaMax,min_sigma=sigmaMin, threshold=thrs,sigma_ratio=sigma_ratio,exclude_border=True)
+    blobs_dog[:, 2] = blobs_dog[:, 2] * sqrt(2)
+
+    title = 'DP detection'
+
+    fig, axes = plt.subplots(1, 2, figsize=(10, 5), sharex=True, sharey=True)
+    ax = axes.ravel()
+
+    ax[0].set_title('FFT_')
+    ax[0].imshow(fft_crop)
+    # ax[0].set_axis_off()
+    ax[1].set_title(title)
+    ax[1].imshow(image)
+
+
+    plt.tight_layout()
+    
+   
+    data=np.zeros((len(blobs_dog),6))
+    data[:,0]=blobs_dog[:,1]
+    data[:,1]=blobs_dog[:,0]
+
+    for j in range(len(blobs_dog)):
+        y=data[j,1]-r
+        x=data[j,0]-r
+
+        data[j,2]=math.sqrt(x**2+y**2)
+        # if data[j,2] < 10:
+        #     data[j,2]=0
+        #     theta = 0
+        # else:
+        #     theta=math.atan(y/x)
+        theta=math.atan(y/x)
+        data[j,3]=math.degrees(theta)
+
+    data[:,4]=data[:,2]*scale
+    data[:,5]=1/data[:,4]
+
+    plt.scatter(data[:,0],data[:,1],marker='+',color='red')
+    for i, (x_i, y_i) in enumerate(zip(data[:,0],data[:,1])):
+        plt.annotate(str(i+1), (x_i, y_i), textcoords="offset points", xytext=(0,10), ha='center',color='white')
+    pathoutcsv=pathout[:-4]+'.csv'
+    IO.writeCSV(pathoutcsv,data,fmt='%1.3f')
+    plt.savefig(pathout)