Rhapso 0.1.92__py3-none-any.whl

Rhapso/detection/overlap_detection.py

@@ -0,0 +1,327 @@
+import numpy as np
+from bioio import BioImage
+import bioio_tifffile
+import zarr
+import s3fs
+import dask.array as da
+import math
+
+"""
+Overlap Detection figures out where image tile overlap. 
+"""
+
+# TIFF reader wants to be used as an abstract class
+class CustomBioImage(BioImage):
+    def standard_metadata(self):
+        pass
+    
+    def scale(self):
+        pass
+    
+    def time_interval(self):
+        pass
+
+class OverlapDetection():
+    def __init__(self, transform_models, dataframes, dsxy, dsz, prefix, file_type):
+        self.transform_models = transform_models
+        self.image_loader_df = dataframes['image_loader']
+        self.dsxy, self.dsz = dsxy, dsz
+        self.prefix = prefix
+        self.file_type = file_type
+        self.to_process = {}
+        self.image_shape_cache = {}
+        self.max_interval_size = 0
+    
+    def create_mipmap_transform(self):
+        """
+        Build a 4×4 homogeneous scaling matrix for the mipmap level
+        """
+        scale_matrix = np.array([
+            [self.dsxy, 0, 0, 0],  
+            [0, self.dsxy, 0, 0],  
+            [0, 0, self.dsz, 0],  
+            [0, 0, 0, 1]          
+        ])
+        
+        return scale_matrix
+    
+    def load_image_metadata(self, file_path):
+        if file_path in self.image_shape_cache:
+            return self.image_shape_cache[file_path]
+        
+        if self.file_type == 'zarr':
+            s3 = s3fs.S3FileSystem(anon=False)
+            store = s3fs.S3Map(root=file_path, s3=s3)
+            zarr_array = zarr.open(store, mode='r')
+            dask_array = da.from_zarr(zarr_array)
+            dask_array = da.expand_dims(dask_array, axis=2)
+            shape = dask_array.shape
+            self.image_shape_cache[file_path] = shape
+
+        elif self.file_type == 'tiff':
+            img = CustomBioImage(file_path, reader=bioio_tifffile.Reader)
+            data = img.get_dask_stack()
+            shape = data.shape
+            self.image_shape_cache[file_path] = shape
+        
+        return shape
+    
+    # def open_and_downsample(self, shape):
+    #     X = int(shape[5])
+    #     Y = int(shape[4])
+    #     Z = int(shape[3])
+
+    #     dsx = int(self.dsxy)
+    #     dsy = int(self.dsxy)
+    #     dsz = int(self.dsz)
+
+    #     def ceil_half_chain(n, f):
+    #         out = int(n)
+    #         while f >= 2:
+    #             out = (out + 1) // 2  # ceil(n/2)
+    #             f //= 2
+    #         return out
+
+    #     x_new = ceil_half_chain(X, dsx)
+    #     y_new = ceil_half_chain(Y, dsy)
+    #     z_new = ceil_half_chain(Z, dsz)
+
+    #     mipmap_transform = self.create_mipmap_transform()
+    #     return ((0, 0, 0), (x_new, y_new, z_new)), mipmap_transform
+    
+    def open_and_downsample(self, shape, dsxy, dsz):
+        """
+        Downsample a 3D volume by powers of two by repeatedly halving along each axis
+        """
+        dsx = dsxy
+        dsy = dsxy
+
+        # downsample x dimension
+        x_new = shape[5]
+        while dsx > 1:
+            x_new = x_new // 2 if x_new % 2 == 0 else (x_new // 2) + 1
+            dsx //= 2
+
+        # downsample y dimension
+        y_new = shape[4]
+        while dsy > 1:
+            y_new = y_new // 2 if y_new % 2 == 0 else (y_new // 2) + 1
+            dsy //= 2
+
+        # downsample z dimension
+        z_new = shape[3]
+        while dsz > 1:
+            z_new = z_new // 2 if z_new % 2 == 0 else (z_new // 2) + 1
+            dsz //= 2
+
+        return ((0, 0, 0), (x_new, y_new, z_new))
+    
+    def get_inverse_mipmap_transform(self, mipmap_transform):
+        """
+        Compute the inverse of the given mipmap transform
+        """
+        try:
+            inverse_scale_matrix = np.linalg.inv(mipmap_transform)
+        except np.linalg.LinAlgError:
+            print("Matrix cannot be inverted.")
+            return None
+        
+        return inverse_scale_matrix    
+    
+    def estimate_bounds(self, a, interval):
+        """
+        Transform an axis-aligned box through a 4x4 affine
+        """
+        # set lower bounds
+        t0, t1, t2 = 0, 0, 0
+        
+        # set upper bounds
+        if self.file_type == 'zarr':
+            s0 = interval[5] - t0
+            s1 = interval[4] - t1
+            s2 = interval[3] - t2 
+        elif self.file_type == 'tiff':
+            s0 = interval[5] - t0
+            s1 = interval[4] - t1
+            s2 = interval[3] - t2
+
+        # get dot product of uppper bounds and inverted downsampling matrix
+        matrix = np.array(a) 
+        tt = np.dot(matrix[:, :3], [t0, t1, t2]) + matrix[:, 3]
+        r_min = np.copy(tt)
+        r_max = np.copy(tt)
+
+        # set upper and lower bounds using inverted downsampling matrix
+        for i in range(3):
+            if matrix[i, 0] < 0:
+                r_min[i] += s0 * matrix[i, 0]
+            else:
+                r_max[i] += s0 * matrix[i, 0]
+            
+            if matrix[i, 1] < 0:
+                r_min[i] += s1 * matrix[i, 1]
+            else:
+                r_max[i] += s1 * matrix[i, 1]
+
+            if matrix[i, 2] < 0:
+                r_min[i] += s2 * matrix[i, 2]
+            else:
+                r_max[i] += s2 * matrix[i, 2]
+        
+        return r_min[:3], r_max[:3]
+
+    def calculate_intersection(self, bbox1, bbox2):
+        """
+        Compute the axis-aligned intersection of two 3D boxes given as (min, max) coordinates
+        """
+        intersect_min = np.maximum(bbox1[0], bbox2[0])
+        intersect_max = np.minimum(bbox1[1], bbox2[1])
+        
+        return (intersect_min, intersect_max)
+
+    def calculate_new_dims(self, lower_bound, upper_bound):
+        """
+        Compute per-axis lengths from bounds
+        """
+        new_dims = []
+        for lb, ub in zip(lower_bound, upper_bound):
+            if lb == 0:
+                new_dims.append(ub + 1)
+            else:
+                new_dims.append(ub - lb)
+        
+        return new_dims
+    
+    def floor_log2(self, n):
+        """
+        Return ⌊log2(n)⌋ - clamps n ≤ 1 to 1 so the result is 0 for n ≤ 1
+        """
+        return max(0, int(math.floor(math.log2(max(1, n)))))
+
+    def choose_zarr_level(self):
+        """
+        pick the highest power-of-two pyramid level ( ≤ 7) compatible with dsxy/dsz
+        """
+        max_level = 7
+        lvl_xy = self.floor_log2(self.dsxy)
+        lvl_z  = self.floor_log2(self.dsz)
+        best = min(lvl_xy, lvl_z, max_level)
+        factor = 1 << best  
+        leftovers = (max(1, self.dsxy // factor), max(1, self.dsxy // factor), max(1, self.dsz // factor))
+        return best, leftovers
+    
+    def affine_with_half_pixel_shift(self, sx, sy, sz):
+        """
+        Build a 4x4 scaling affine that also shifts by 0.5·(scale-1) per axis so voxel centers stay aligned after 
+        resampling (half-pixel compensation)
+        """
+        # translation = 0.5 * (scale - 1) per axis
+        tx = 0.5 * (sx - 1.0)
+        ty = 0.5 * (sy - 1.0)
+        tz = 0.5 * (sz - 1.0)
+        
+        return np.array([
+            [sx, 0.0, 0.0, tx],
+            [0.0, sy, 0.0, ty],
+            [0.0, 0.0, sz, tz],
+            [0.0, 0.0, 0.0, 1.0],
+        ], dtype=float)
+    
+    def size_interval(self, lb, ub):
+        """
+        Find the number of voxels in a 3D box with inclusive bounds
+        """
+        return int((int(ub[0]) - int(lb[0]) + 1) *
+                (int(ub[1]) - int(lb[1]) + 1) *
+                (int(ub[2]) - int(lb[2]) + 1))
+
+    def find_overlapping_area(self):
+        """
+        Compute XY Z overlap intervals against every other view, accounting for mipmap/downsampling and per-view affine transforms
+        """
+        for i, row_i in self.image_loader_df.iterrows():
+            view_id = f"timepoint: {row_i['timepoint']}, setup: {row_i['view_setup']}"
+            
+            # get inverted matrice of downsampling
+            all_intervals = []        
+            if self.file_type == 'zarr':
+                level, leftovers = self.choose_zarr_level()
+                dim_base = self.load_image_metadata(self.prefix + row_i['file_path'] + f'/{0}')
+
+                # isotropic pyramid
+                s = float(2 ** level)  
+                mipmap_of_downsample = self.affine_with_half_pixel_shift(s, s, s)
+
+                # TODO - update mipmap with leftovers if other than 1
+                _, dsxy, dsz = leftovers
+                
+            elif self.file_type == 'tiff':
+                dim_base = self.load_image_metadata(self.prefix + row_i['file_path'])
+                mipmap_of_downsample = self.create_mipmap_transform()
+                dsxy, dsz = self.dsxy, self.dsz
+                level = None
+
+            downsampled_dim_base = self.open_and_downsample(dim_base, dsxy, dsz)
+            t1 = self.get_inverse_mipmap_transform(mipmap_of_downsample) 
+
+            # compare with all view_ids
+            for j, row_j in self.image_loader_df.iterrows():
+                if i == j: continue
+                
+                view_id_other = f"timepoint: {row_j['timepoint']}, setup: {row_j['view_setup']}"
+
+                if self.file_type == 'zarr':
+                    dim_other = self.load_image_metadata(self.prefix + row_j['file_path'] + f'/{0}')
+                elif self.file_type == 'tiff':
+                    dim_other = self.load_image_metadata(self.prefix + row_j['file_path'])
+                
+                # get transforms matrix from both view_ids and downsampling matrices
+                matrix = self.transform_models.get(view_id)
+                matrix_other = self.transform_models.get(view_id_other)
+
+                if self.file_type == 'zarr':
+                    s = float(2 ** level)  
+                    mipmap_of_downsample_other = self.affine_with_half_pixel_shift(s, s, s)
+                elif self.file_type == 'tiff':
+                    mipmap_of_downsample_other = self.create_mipmap_transform()
+
+                inverse_mipmap_of_downsample_other = self.get_inverse_mipmap_transform(mipmap_of_downsample_other)
+                inverse_matrix = self.get_inverse_mipmap_transform(matrix)
+
+                concatenated_matrix = np.dot(inverse_matrix, matrix_other) 
+                t2 = np.dot(inverse_mipmap_of_downsample_other, concatenated_matrix)
+
+                intervals = self.estimate_bounds(t1, dim_base)
+                intervals_other = self.estimate_bounds(t2, dim_other)
+
+                bounding_boxes = tuple(map(lambda x: np.round(x).astype(int), intervals))
+                bounding_boxes_other = tuple(map(lambda x: np.round(x).astype(int), intervals_other))
+
+                # find upper and lower bounds of intersection
+                if np.all((bounding_boxes[1] >= bounding_boxes_other[0]) & (bounding_boxes_other[1] >= bounding_boxes[0])):
+                    intersected_boxes = self.calculate_intersection(bounding_boxes, bounding_boxes_other)
+                    intersect = self.calculate_intersection(downsampled_dim_base, intersected_boxes)     
+                    intersect_dict = {
+                        'lower_bound': intersect[0],
+                        'upper_bound': intersect[1],
+                        'span': self.calculate_new_dims(intersect[0], intersect[1])
+                    }
+
+                    lb, ub = intersect[0], intersect[1]
+                    sz = self.size_interval(lb, ub)
+                    if sz > self.max_interval_size:
+                        self.max_interval_size = sz
+
+                    # add max size
+                    all_intervals.append(intersect_dict)        
+    
+            self.to_process[view_id] = all_intervals
+        
+        return dsxy, dsz, level, mipmap_of_downsample
+                
+    def run(self):
+        """
+        Executes the entry point of the script.
+        """
+        dsxy, dsz, level, mipmap_of_dowsample = self.find_overlapping_area()
+        return self.to_process, dsxy, dsz, level, self.max_interval_size, mipmap_of_dowsample

Rhapso/detection/points_validation.py

@@ -0,0 +1,49 @@
+import numpy as np
+
+"""
+Points Validation prints out the metrics for the results of interest point detection
+"""
+
+class PointsValidation:
+    def __init__(self, consolidated_data):
+        self.consolidated_data = consolidated_data
+
+    def validation_suite(self):
+        total_points = sum(len(points) for points in self.consolidated_data.values())
+        print(f"\nTotal Interest Points Found: {total_points}")
+
+        print("\nInterest Points by View ID:")
+        for view_id, points in self.consolidated_data.items():
+
+            # Sort points by index
+            sorted_points = sorted(points, key=lambda x: x[0][2])  # x[1] is the (x,y,z) index to sort by
+
+            if len(sorted_points) == 0:
+                print(f"\nView ID: {view_id} | Num points: 0")
+                print("\n--- Detection Stats ---")
+                print("No points found for this view.\n")
+                continue
+
+            coords = np.array([p[0] for p in sorted_points])
+            intensities = np.array([p[1] for p in sorted_points])
+
+            # Print metrics on interest points
+            print("\n--- Detection Stats ---")
+            print(f"Total Points: {len(coords)}")
+            print(f"Intensity: min={intensities.min():.2f}, max={intensities.max():.2f}, mean={intensities.mean():.2f}, std={intensities.std():.2f}")
+
+            for dim, name in zip(range(3), ['X', 'Y', 'Z']):
+                values = coords[:, dim]
+                print(f"{name} Range: {values.min():.2f} – {values.max():.2f} | Spread (std): {values.std():.2f}")
+
+            # Density per 1000x1000x1000 space
+            volume = np.ptp(coords[:, 0]) * np.ptp(coords[:, 1]) * np.ptp(coords[:, 2])
+            density = len(coords) / (volume / 1e9) if volume > 0 else 0
+            print(f"Estimated Density: {density:.2f} points per 1000³ volume")
+            print("-----------------------\n")
+    
+    def run(self):
+        """
+        Executes the entry point of the script.
+        """
+        self.validation_suite()

Rhapso/detection/save_interest_points.py

@@ -0,0 +1,265 @@
+import zarr
+import numpy as np
+import xml.etree.ElementTree as ET
+import s3fs
+import boto3
+from io import BytesIO
+import io
+import json
+
+"""
+Save Interest Points saves interest points as N5 and updates the xml with pathways
+"""
+
+class SaveInterestPoints:
+    def __init__(self, dataframes, consolidated_data, xml_file_path, xml_output_file_path, n5_output_file_prefix, downsample_xy, downsample_z, min_intensity, 
+                 max_intensity, sigma, threshold): 
+        self.consolidated_data = consolidated_data
+        self.image_loader_df = dataframes['image_loader']
+        self.xml_file_path = xml_file_path
+        self.xml_output_file_path = xml_output_file_path
+        self.n5_output_file_prefix = n5_output_file_prefix
+        self.downsample_xy = downsample_xy
+        self.downsample_z = downsample_z
+        self.min_intensity = min_intensity
+        self.max_intensity = max_intensity
+        self.sigma = sigma
+        self.threshold = threshold
+        self.s3_filesystem = s3fs.S3FileSystem()
+        self.overlappingOnly = "true"
+        self.findMin = "true"
+        self.findMax = "true"
+        self.default_block_size = 300000
+    
+    def load_xml_file(self, file_path):
+        tree = ET.parse(file_path)
+        root = tree.getroot()
+        return tree, root
+    
+    def fetch_from_s3(self, s3, bucket_name, input_file):
+        response = s3.get_object(Bucket=bucket_name, Key=input_file)
+        return response['Body'].read().decode('utf-8')
+    
+    def save_to_xml(self):
+        """
+        Rebuild the <ViewInterestPoints> section and write the updated XML back
+        """
+        if self.xml_file_path.startswith("s3://"):
+            bucket, key = self.xml_file_path.replace("s3://", "", 1).split("/", 1)
+            s3 = boto3.client('s3')
+            xml_string = self.fetch_from_s3(s3, bucket, key)
+            tree = ET.parse(io.BytesIO(xml_string.encode('utf-8')))
+            root = tree.getroot()
+        else:
+            tree, root = self.load_xml_file(self.xml_file_path)
+
+        interest_points_section = root.find('.//ViewInterestPoints')
+        
+        if interest_points_section is None:
+            interest_points_section = ET.SubElement(root, 'ViewInterestPoints')
+            interest_points_section.text = '\n    ' 
+        
+        else:
+            interest_points_section.clear()
+            interest_points_section.text = '\n    '  
+
+        for view_id, _ in self.consolidated_data.items():
+            parts = view_id.split(',') 
+            timepoint_part = parts[0].strip()  
+            setup_part = parts[1].strip() 
+
+            timepoint = int(timepoint_part.split(':')[1].strip())  
+            setup = int(setup_part.split(':')[1].strip())
+            label = "beads"
+            params = "DOG (Spark) s={} t={} overlappingOnly={} min={} max={} downsampleXY={} downsampleZ={} minIntensity={} maxIntensity={}".format(
+                self.sigma, self.threshold, self.overlappingOnly, self.findMin, self.findMax,
+                self.downsample_xy, self.downsample_z, self.min_intensity, self.max_intensity)
+            value = f"tpId_{timepoint}_viewSetupId_{setup}/beads"
+
+            new_interest_point = ET.SubElement(interest_points_section, 'ViewInterestPointsFile', {
+                'timepoint': str(timepoint),
+                'setup': str(setup),
+                'label': label,
+                'params': params
+            })
+            new_interest_point.text = value
+            new_interest_point.tail = '\n    '
+        
+        interest_points_section.tail = '\n  '
+
+        if self.xml_output_file_path.startswith("s3://"):
+            bucket, key = self.xml_output_file_path.replace("s3://", "", 1).split("/", 1)
+            xml_bytes = BytesIO()
+            tree.write(xml_bytes, encoding='utf-8', xml_declaration=True)
+            xml_bytes.seek(0)
+            s3 = boto3.client('s3') 
+            s3.upload_fileobj(xml_bytes, bucket, key)
+
+        else:
+            tree.write(self.xml_output_file_path, encoding='utf-8', xml_declaration=True)
+        
+    def write_json_to_s3(self, id_dataset_path, loc_dataset_path, attributes):
+        """
+        Write attributes file into both the ID and LOC dataset directories on S3
+        """
+        bucket, key = id_dataset_path.replace("s3://", "", 1).split("/", 1)
+        json_path = key + '/attributes.json'
+        json_bytes = json.dumps(attributes).encode('utf-8')
+        s3 = boto3.client('s3')
+        s3.put_object(Bucket=bucket, Key=json_path, Body=json_bytes)
+
+        bucket, key = loc_dataset_path.replace("s3://", "", 1).split("/", 1)
+        json_path = key + '/attributes.json'
+        json_bytes = json.dumps(attributes).encode('utf-8')
+        s3 = boto3.client('s3')
+        s3.put_object(Bucket=bucket, Key=json_path, Body=json_bytes)
+
+    def save_intensities_to_n5(self, view_id, n5_path):
+        """
+        Write intensities into an N5 group
+        """
+        if self.n5_output_file_prefix.startswith("s3://"):
+            output_path = self.n5_output_file_prefix + n5_path + "/interestpoints"
+            store = s3fs.S3Map(root=output_path, s3=self.s3_filesystem, check=False)
+            root = zarr.group(store=store, overwrite=False)
+            root.attrs['n5'] = '4.0.0'
+        
+        else:
+            store = zarr.N5Store(self.n5_output_file_prefix + n5_path + "/interestpoints")
+            root = zarr.group(store, overwrite=False)
+            root.attrs['n5'] =  '4.0.0'
+        
+        intensities_path = 'intensities'
+
+        if intensities_path in root:
+            try:
+                del root[intensities_path]
+            except Exception as e:
+                print(f"Warning: failed to delete existing dataset at {intensities_path}: {e}")
+
+        try: 
+            if view_id in self.consolidated_data:
+                intensities = [point[1] for point in self.consolidated_data[view_id]] 
+                dataset = root.create_dataset(
+                    intensities_path,
+                    data=intensities,
+                    dtype='f4',  
+                    chunks=(self.default_block_size,),  
+                    compressor=zarr.GZip()
+                )
+                dataset.attrs["dimensions"] = [1, len(intensities)]
+                dataset.attrs["blockSize"] = [1, self.default_block_size]
+            else: 
+                root.create_dataset(
+                    intensities_path,
+                    shape=(0,), 
+                    dtype='f4', 
+                    chunks=(1,),  
+                    compressor=zarr.GZip()  
+                )
+        except Exception as e:
+            print(f"Error creating intensities dataset at {intensities_path}: {e}")
+
+    def save_interest_points_to_n5(self, view_id, n5_path): 
+        """
+        Write interest point IDs and 3D locations into an N5 group
+        """
+        if self.n5_output_file_prefix.startswith("s3://"):
+            output_path = self.n5_output_file_prefix + n5_path + "/interestpoints"
+            store = s3fs.S3Map(root=output_path, s3=self.s3_filesystem, check=False)
+            root = zarr.group(store=store, overwrite=False)
+            root.attrs["pointcloud"] = "1.0.0"
+            root.attrs["type"] = "list"
+            root.attrs["list version"] = "1.0.0"
+
+        else:
+            store = zarr.N5Store(self.n5_output_file_prefix + n5_path + "/interestpoints")
+            root = zarr.group(store, overwrite=False)
+            root.attrs["pointcloud"] = "1.0.0"
+            root.attrs["type"] = "list"
+            root.attrs["list version"] = "1.0.0"
+
+        id_dataset = "id"
+        loc_dataset = "loc"
+
+        if self.n5_output_file_prefix.startswith("s3://"):
+            id_path = f"{output_path}/id"
+            loc_path = f"{output_path}/loc"
+            attrs_dict = dict(root.attrs)
+            self.write_json_to_s3(id_path, loc_path, attrs_dict)
+
+        if (view_id in self.consolidated_data) and (len(self.consolidated_data[view_id]) > 0):
+            interest_points = [point[0] for point in self.consolidated_data[view_id]]
+            interest_point_ids = np.arange(len(interest_points), dtype=np.uint64).reshape(-1, 1)
+            n = 3
+
+            if id_dataset in root:
+                del root[id_dataset]
+            root.create_dataset(
+                id_dataset,
+                data=interest_point_ids,
+                dtype='u8',
+                chunks=(self.default_block_size,),
+                compressor=zarr.GZip()
+            )
+
+            if loc_dataset in root:
+                del root[loc_dataset]
+            root.create_dataset(
+                loc_dataset,
+                data=interest_points,
+                dtype='f8',
+                chunks=(self.default_block_size, n),
+                compressor=zarr.GZip()
+            )
+        
+        # save as empty lists
+        else:
+            if id_dataset in root:
+                del root[id_dataset]
+            root.create_dataset(
+                id_dataset,
+                shape=(0,),
+                dtype='u8',
+                chunks=(1,),
+                compressor=zarr.GZip()
+            )
+
+            if loc_dataset in root:
+                del root[loc_dataset]
+            root.create_dataset(
+                loc_dataset,
+                shape=(0,),
+                dtype='f8',
+                chunks=(1,),
+                compressor=zarr.GZip()
+            )
+
+    def save_points(self):
+        """
+        Orchestrate interest points and intensities into an N5 layout - inject attributes file
+        """
+        for _, row in self.image_loader_df.iterrows():
+            view_id = f"timepoint: {row['timepoint']}, setup: {row['view_setup']}"
+            n5_path = f"interestpoints.n5/tpId_{row['timepoint']}_viewSetupId_{row['view_setup']}/beads"
+            self.save_interest_points_to_n5(view_id, n5_path)
+            self.save_intensities_to_n5(view_id, n5_path)
+
+        path = self.n5_output_file_prefix + "interestpoints.n5"
+        
+        if path.startswith("s3://"):
+            bucket_key = path.replace("s3://", "", 1)
+            store = s3fs.S3Map(root=bucket_key, s3=self.s3_filesystem, check=False)
+            root = zarr.group(store=store, overwrite=False)
+            root.attrs['n5'] = '4.0.0'
+        else:
+            store = zarr.N5Store(path)
+            root = zarr.group(store, overwrite=False)
+            root.attrs['n5'] =  '4.0.0'
+
+    def run(self):
+        """
+        Executes the entry point of the script.
+        """
+        self.save_points()
+        self.save_to_xml()