Rhapso 0.1.92__py3-none-any.whl

Rhapso/__init__.py

@@ -0,0 +1 @@
+

Rhapso/data_prep/__init__.py

@@ -0,0 +1,2 @@
+
+# This file can be empty or contain initialization code for the data_preparation module

Rhapso/data_prep/n5_reader.py

@@ -0,0 +1,188 @@
+# -----------------------------------------------------------------------------
+# n5_reader.py – N5 data verification script
+#
+# Setup & run:
+#   1. python -m venv n5Venv             # create a fresh Python virtual environment
+#   2. source n5Venv/bin/activate        # activate the virtual environment
+#   3. pip install .[n5_reader]          # install n5_reader dependencies from setup.py
+#   4. python Rhapso/data_prep/n5_reader.py
+#      # run the N5 reader for inspecting datasets
+# -----------------------------------------------------------------------------
+
+import zarr
+import s3fs
+import os
+import numpy as np
+import matplotlib.pyplot as plt
+import json
+from zarr.storage import FSStore
+
+def list_files_under_prefix(node, path):
+    try:
+        for item in node[path]:
+            new_path = f"{path}/{item}"
+            if isinstance(node[new_path], zarr.hierarchy.Group):
+                print(f"Group: {new_path}")
+                list_files_under_prefix(node, new_path)
+            else:
+                print(f"Dataset: {new_path} - {node[new_path].shape}")
+    except KeyError:
+        print(f"No items found under the path {path}")
+
+# Amount of interest points in view 18,0 is 1061
+# Max value for view 18,0 in corr ip index is 1017
+
+def read_n5_data(n5_path):
+    import zarr, s3fs, os
+
+    # guard missing local path
+    if not n5_path.startswith("s3://") and not os.path.isdir(n5_path):
+        print(f"❌ Local N5 path not found: {n5_path}")
+        return
+
+    # open the store (S3 or local N5)
+    if n5_path.startswith("s3://"):
+        s3 = s3fs.S3FileSystem(anon=False)
+        store = s3fs.S3Map(root=n5_path, s3=s3)
+    else:
+        store = zarr.N5Store(n5_path)
+
+    print(f"\n🔍 Reading N5 data at: {n5_path}")
+    root = zarr.open(store, mode='r')
+
+    def visit_fn(path, node):
+        if isinstance(node, zarr.Array):
+            print(f"\n📂 Dataset: {path}")
+            print(f"  🔢 dtype: {node.dtype}")
+            shape = node.shape
+            print(f"  📏 shape: {shape}")
+            if len(shape) > 1:
+                print(f"  📊 count: {shape[0]} arrays of shape {shape[1:]}")
+            else:
+                print(f"  📊 count: {shape[0]} elements")
+            print(f"  🗂 chunks: {node.chunks}")
+            print(f"  🛠 compressor: {node.compressor}")
+
+            print("  🔎 first 5 entries:")
+            sample = node[:5]
+            for i, entry in enumerate(sample, start=1):
+                # ensure nested array is printed clearly
+                val = entry.tolist() if hasattr(entry, "tolist") else entry
+                print(f"    {i}. {val}")
+
+    root.visititems(visit_fn)
+
+# # read_n5_data('/home/martin/Documents/Allen/Data/IP_TIFF_XML_2/interestpoints.n5')
+
+def read_correspondences(dataset_path):
+    if dataset_path.startswith("s3://"):
+        store = zarr.storage.FSStore(dataset_path, mode="r")
+        root = zarr.open(store, mode="r")
+    else:
+        store = zarr.N5Store(dataset_path)
+        root = zarr.open(store, mode="r")
+    
+    if "data" not in root:
+        print("Key 'data' not found in root.")
+        return
+    
+    group = root["data"]
+    data = group[:]
+    print(f"Loaded {len(data)} entries.")
+
+    # for i, entry in enumerate(data):
+    #     print(f"{i}: {entry}")
+    
+    # print("hi")
+
+# Big Stitcher Output
+# # base_path = "/Users/seanfite/Desktop/interest_point_detection/interestpoints.n5"
+# base_path = "/Users/seanfite/Desktop/ip_rigid_alignment/interestpoints.n5"
+# # base_path = "/Users/seanfite/Desktop/ip_affine_alignment/interestpoints.n5"
+# # base_path = "s3://rhapso-matching-test/output/interestpoints.n5"
+# for tp_id in [0]:
+#     for setup_id in range(20):  
+#         path = f"{base_path}/tpId_{tp_id}_viewSetupId_{setup_id}/beads/correspondences"
+#         print(f"Reading: {path}")
+#         read_correspondences(path)
+
+def read_interest_points(full_path):
+    if full_path.startswith("s3://"):
+        # s3 = s3fs.S3FileSystem(anon=False)
+        # store = s3fs.S3Map(root=full_path, s3=s3)
+        # zarray = zarr.open_array(store, mode='r')
+        # data = zarray[:]
+
+        path = full_path.replace("s3://", "", 1)
+        bucket = path.split("/")[0]
+        prefix = "/".join(path.split("/")[1:])
+        
+        s3 = s3fs.S3FileSystem()
+        store = FSStore(f"{bucket}/{prefix}", fs=s3, mode='r')
+        root = zarr.open(store, mode="r")
+
+        group = root["data"]
+        data = group[:]
+        count = len(data)
+        print(count)
+        print("")
+        
+    
+    else:
+        full_path = full_path.rstrip("/")  # remove trailing slash if any
+        components = full_path.split("/")
+
+        # Find index of the N5 root (assumes .n5 marks the root)
+        try:
+            n5_index = next(i for i, c in enumerate(components) if c.endswith(".n5"))
+        except StopIteration:
+            raise ValueError("No .n5 directory found in path")
+
+        dataset_path = "/".join(components[:n5_index + 1])            # the store root
+        dataset_rel_path = "/".join(components[n5_index + 1:])        # relative dataset path
+
+        # Open N5 store and dataset
+        store = zarr.N5Store(dataset_path)
+        root = zarr.open(store, mode='r')
+
+        if dataset_rel_path not in root:
+            print(f"Skipping: {dataset_rel_path} (not found)")
+            return
+        
+        zarray = root[dataset_rel_path + "/loc"]
+        data = zarray[:]
+
+    print("\n--- Detection Stats (Raw Rhapso Output) ---")
+    print(f"Total Points: {len(data)}")
+
+    # for dim, name in zip(range(3), ['X', 'Y', 'Z']):
+    #     values = data[:, dim]
+    #     print(f"{name} Range: {values.min():.2f} – {values.max():.2f} | Spread (std): {values.std():.2f}")
+
+    # volume = np.ptp(data[:, 0]) * np.ptp(data[:, 1]) * np.ptp(data[:, 2])
+    # density = len(data) / (volume / 1e9) if volume > 0 else 0
+    # print(f"Estimated Density: {density:.2f} points per 1000³ volume")
+    # print("-----------------------")
+
+    # # --- 3D Plot ---
+    # max_points = 1000000000000
+    # sample = data if len(data) <= max_points else data[np.random.choice(len(data), max_points, replace=False)]
+
+    # fig = plt.figure(figsize=(10, 8))
+    # ax = fig.add_subplot(111, projection='3d')
+    # ax.scatter(sample[:, 0], sample[:, 1], sample[:, 2], c='blue', alpha=0.5, s=1)
+    # ax.set_xlabel('X')
+    # ax.set_ylabel('Y')
+    # ax.set_zlabel('Z')
+    # ax.set_title(f"Interest Points in 3D (showing {len(sample)} points)")
+    # plt.tight_layout()
+    # plt.show()
+
+# base_path = "s3://rhapso-matching-test/output/interestpoints.n5"
+# base_path = "/Users/seanfite/Desktop/IP_TIFF_XML/interestpoints.n5"
+base_path = "/Users/seanfite/Desktop/interestpoints.n5"
+for tp_id in [0]:
+    for setup_id in range(20): 
+        path = f"{base_path}/tpId_{tp_id}_viewSetupId_{setup_id}/beads/interestpoints"
+        print(f"For view: {setup_id}")
+        read_interest_points(path)

Rhapso/data_prep/s3_big_stitcher_reader.py

@@ -0,0 +1,55 @@
+import json
+import os
+import s3fs
+
+"""
+Utility class for downloading BigStitcher outputs from S3 to local storage for N5 reader compatibility
+"""
+
+class S3BigStitcherReader:
+    def __init__(self, s3_uri, local_directory):
+           self.s3_uri = s3_uri
+           self.local_directory = local_directory
+    
+    def download_n5_from_s3_to_local(self):
+        """
+        Recursively download an N5 dataset from S3 to a local directory.
+        """
+        s3 = s3fs.S3FileSystem(anon=False)
+        s3_path = self.s3_uri.replace("s3://", "")
+        all_keys = s3.find(s3_path, detail=True)
+
+        for key, obj in all_keys.items():
+            if obj["type"] == "file":
+                rel_path = key.replace(s3_path + "/", "")
+                local_file_path = os.path.join(self.local_directory, rel_path)
+                os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
+                s3.get(key, local_file_path)
+
+                # Check for the specific interestpoints path
+                if rel_path.endswith("beads/interestpoints/attributes.json") and "interestpoints.n5" in rel_path:
+                    # Construct the path to the attributes file
+                    attributes_path = os.path.join(os.path.dirname(local_file_path), "attributes.json")
+                    attributes_data = {
+                        "pointcloud": "1.0.0",
+                        "type": "list",
+                        "list version": "1.0.0"
+                    }
+
+                    with open(attributes_path, "w") as f:
+                                        json.dump(attributes_data, f, indent=2)
+
+    def run(self):
+        self.download_n5_from_s3_to_local()
+
+        s3_path = self.s3_uri.replace("s3://", "")
+        full_local_path = os.path.join(self.local_directory, s3_path)
+
+        # Final paths
+        xml_input_path = os.path.join(full_local_path, "bigstitcher_ip.xml")
+        n5_output_path = os.path.join(full_local_path, "interestpoints.n5")
+
+        print("XML Input Path:", xml_input_path)
+        print("N5 Output Path:", n5_output_path)
+
+        return xml_input_path, n5_output_path

Rhapso/data_prep/xml_to_dataframe.py

@@ -0,0 +1,215 @@
+import pandas as pd
+import xml.etree.ElementTree as ET
+
+# This component recieves an XML file containing Tiff or Zarr image metadata and converts
+# it into several Dataframes
+
+class XMLToDataFrame:
+    def __init__(self, xml_file):
+        self.xml_content = xml_file
+
+    def parse_image_loader_zarr(self, root):
+        """
+        Parses image loader configuration from a Zarr file's XML structure and constructs a DataFrame containing the 
+        metadata for each image group.
+        """
+        image_loader_data = []
+
+        for il in root.findall(".//ImageLoader/zgroups/zgroup"):
+            view_setup = il.get("setup")
+            timepoint = il.get("timepoint")
+            file_path = il.find("path").text if il.find("path") is not None else None
+            channel = file_path.split("_ch_", 1)[1].split(".ome.zarr", 1)[0]
+
+            image_loader_data.append(
+                {
+                    "view_setup": view_setup,
+                    "timepoint": timepoint,
+                    "series": 1,
+                    "channel": channel,
+                    "file_path": file_path,
+                }
+            )
+
+        return pd.DataFrame(image_loader_data)
+
+    def parse_image_loader_tiff(self, root):
+        """
+        Parses image loader configuration from a TIFF file's XML structure and constructs a DataFrame containing 
+        metadata for each image group.
+        """
+        image_loader_data = []
+
+        # Ensure that file mappings are present in the XML
+        if not root.findall(".//ImageLoader/files/FileMapping"):
+            raise Exception("There are no files in this XML")
+        
+        # Check for required labels in the XML
+        if not self.check_labels(root):
+            raise Exception("Required labels do not exist")
+
+        # Validate that the lengths of view setups, registrations, and tiles match
+        if not self.check_length(root):
+            raise Exception(
+                "The amount of view setups, view registrations, and tiles do not match"
+            )
+
+        # Iterate over each file mapping in the XML
+        for fm in root.findall(".//ImageLoader/files/FileMapping"):
+            view_setup = fm.get("view_setup")
+            timepoint = fm.get("timepoint")
+            series = fm.get("series")
+            channel = fm.get("channel")
+            file_path = fm.find("file").text if fm.find("file") is not None else None
+
+            image_loader_data.append(
+                {
+                    "view_setup": view_setup,
+                    "timepoint": timepoint,
+                    "series": series,
+                    "channel": channel,
+                    "file_path": file_path,
+                }
+            )
+
+        # Convert the list to a DataFrame and return
+        return pd.DataFrame(image_loader_data)
+    
+    def parse_image_loader_split_zarr(self):
+        pass
+
+    def route_image_loader(self, root):
+        """
+        Directs the XML parsing process based on the image loader format specified in the XML.
+        """
+        format_node = root.find(".//ImageLoader")
+        format_type = format_node.get("format")
+
+        if "filemap" in format_type:
+            return self.parse_image_loader_tiff(root)
+        else:
+            return self.parse_image_loader_zarr(root)
+
+    def parse_view_setups(self, root):
+        """
+        Parses the view setups from an XML structure and constructs a DataFrame containing metadata for each view setup.
+        """
+        viewsetups_data = []
+
+        for vs in root.findall(".//ViewSetup"):
+            id_ = vs.find("id").text
+            # name = vs.find("name").text
+            name = vs.findtext("name")
+            size = vs.find("size").text
+            voxel_unit = vs.find(".//voxelSize/unit").text
+            voxel_size = " ".join(vs.find(".//voxelSize/size").text.split())
+            attributes = {attr.tag: attr.text for attr in vs.find("attributes")}
+            viewsetups_data.append(
+                {
+                    "id": id_,
+                    "name": name,
+                    "size": size,
+                    "voxel_unit": voxel_unit,
+                    "voxel_size": voxel_size,
+                    **attributes,
+                }
+            )
+        return pd.DataFrame(viewsetups_data)
+
+    def parse_view_registrations(self, root):
+        """
+        Parses view registrations from an XML structure and constructs a DataFrame containing registration metadata 
+        for each view.
+        """
+        viewregistrations_data = []
+        for vr in root.findall(".//ViewRegistration"):
+            timepoint = vr.get("timepoint")
+            setup = vr.get("setup")
+
+            for vt in vr.findall(".//ViewTransform"):
+                affine_text = (
+                    vt.find("affine").text.replace("\n", "").replace(" ", ", ")
+                )
+                viewregistrations_data.append(
+                    {
+                        "timepoint": timepoint,
+                        "setup": setup,
+                        "type": vt.get("type"),
+                        "name": vt.find("Name").text.strip(),
+                        "affine": affine_text,
+                    }
+                )
+        return pd.DataFrame(viewregistrations_data)
+
+    def parse_view_interest_points(self, root):
+        """
+        Parses interest points data from an XML structure and constructs a DataFrame containing metadata and paths 
+        for each set of interest points.
+        """
+        view_interest_points_data = []
+
+        # if self.key == "detection":
+        #     if len(root.findall(".//ViewInterestPointsFile")) != 0:
+        #         raise Exception("There should be no interest points in this file yet.")
+
+        for vip in root.findall(".//ViewInterestPointsFile"):
+            timepoint = vip.get("timepoint")
+            setup = vip.get("setup")
+            label = vip.get("label")
+            params = vip.get("params")
+            path = vip.text.strip() if vip.text is not None else None
+            view_interest_points_data.append(
+                {
+                    "timepoint": timepoint,
+                    "setup": setup,
+                    "label": label,
+                    "params": params,
+                    "path": path,
+                }
+            )
+        return pd.DataFrame(view_interest_points_data)
+
+    def check_labels(self, root):
+        """
+        Verifies the presence of required XML labels including bounding boxes, point spread functions, 
+        stitching results, and intensity adjustments.
+        """
+        labels = True
+        if root.find(".//BoundingBoxes") is None:
+            labels = False
+        if root.find(".//PointSpreadFunctions") is None:
+            labels = False
+        if root.find(".//StitchingResults") is None:
+            labels = False
+        if root.find(".//IntensityAdjustments") is None:
+            labels = False
+
+        return labels
+
+    def check_length(self, root):
+        """
+        Validates that the count of elements within the XML structure aligns with expected relationships
+        between file mappings, view setups, and view registrations.
+        """
+        length = True
+        if len(root.findall(".//ImageLoader/files/FileMapping")) != len(root.findall(".//ViewRegistration")) or \
+            len(root.findall(".//ViewSetup")) != len(root.findall(".//ViewRegistration")) * (1 / 2):
+            length = False  # Set to False if the relationships do not match expected counts
+        return length
+
+    def run(self):
+        """
+        Executes the entry point of the script.
+        """
+        root = ET.fromstring(self.xml_content)
+        image_loader_df = self.route_image_loader(root)
+        view_setups_df = self.parse_view_setups(root)
+        view_registrations_df = self.parse_view_registrations(root)
+        view_interest_points_df = self.parse_view_interest_points(root)
+
+        return {
+            "image_loader": image_loader_df,
+            "view_setups": view_setups_df,
+            "view_registrations": view_registrations_df,
+            "view_interest_points": view_interest_points_df,
+        }

Rhapso/detection/__init__.py

@@ -0,0 +1,5 @@
+def main(args):
+    print("Hello, Detection!")
+    print("Received the following arguments:")
+    for arg in vars(args):
+        print(f"{arg}: {getattr(args, arg)}")

Rhapso/detection/advanced_refinement.py

@@ -0,0 +1,203 @@
+from collections import OrderedDict
+from scipy.spatial import cKDTree
+import numpy as np
+from collections import defaultdict, OrderedDict
+
+"""
+Advanced Refinement is the final pass over detected interest points: it groups chunks by view/overlap interval, keeps only the 
+strongest points scaled to interval size, then merges and de-duplicates nearby points with a KD-tree.
+"""
+
+class AdvancedRefinement:
+    def __init__(self, interest_points, combine_distance, dataframes, overlapping_area, max_interval_size, max_spots):
+        self.interest_points = interest_points
+        self.consolidated_data = {}
+        self.combine_distance = combine_distance
+        self.image_loader_df = dataframes['image_loader']
+        self.overlapping_area = overlapping_area
+        self.max_interval_size = max_interval_size
+        self.max_spots = max_spots
+        self.overlapping_only = True
+        self.sorted_view_ids = None
+        self.result = interest_points  
+        self.store_intensities = False
+        self._max_spots = 0
+        self.max_spots_per_overlap = False
+        self.to_process = interest_points
+        self.interest_points_per_view_id = {}
+        self.intensities_per_view_id = {}
+        self.intervals_per_view_id = {}
+
+    def kd_tree(self, ips_lists_by_view, ints_lists_by_view):
+        """
+        KD-tree implementation to filter out duplicates. Merging into the tree per bound, per iteration
+        """
+        radius = float(self.combine_distance)
+        out = OrderedDict()
+
+        for view_id in sorted(ips_lists_by_view.keys()):
+            ips_lists = ips_lists_by_view[view_id]
+            ints_lists = ints_lists_by_view[view_id]
+
+            my_ips: list = []
+            my_ints: list = []
+
+            for l, ips in enumerate(ips_lists):
+                intens = ints_lists[l]
+
+                # First list - accept all 
+                if not my_ips:
+                    my_ips.extend(ips)
+                    my_ints.extend(intens)
+                    continue
+
+                # Build KDTree from the CURRENT accepted points for this view
+                base = np.asarray(my_ips, dtype=np.float32)
+                tree = cKDTree(base)
+
+                # Batch query all new points against the tree
+                cand = np.asarray(ips, dtype=np.float32)
+                
+                if cand.size == 0:
+                    continue
+                
+                dists, _ = tree.query(cand, k=1)  
+
+                # Keep only points farther than combineDistance
+                mask = dists > radius
+                if np.any(mask):
+                    for p, val in zip(cand[mask], np.asarray(intens)[mask]):
+                        my_ips.append(p.tolist())   
+                        my_ints.append(float(val))
+
+            out[view_id] = list(zip(my_ips, my_ints))
+
+        self.consolidated_data = out
+    
+    def size(self, interval):
+        """
+        Finds the number of voxels in a 3D interval
+        """
+        lb, ub = interval[0], interval[1]
+        prod = 1
+        for l, u in zip(lb, ub):
+            prod *= (int(u) - int(l) + 1)
+        return prod
+    
+    def contains(self, containing, contained):
+        """
+        Boolean check if the 3D interval `contained` lies fully inside the 3D interval `containing`
+        """
+        lc, uc = containing[0], containing[1]
+        li, ui = contained[0],  contained[1]
+        return all(lc[d] <= li[d] and uc[d] >= ui[d] for d in range(3))
+    
+    def filter_lists(self, ips, intensities, my_max_spots):
+        """
+        Pick the top-N interest points by intensity and return them
+        """
+        if intensities is None or len(ips) == 0 or my_max_spots <= 0:
+            return ips, intensities
+
+        intens_arr = np.asarray(intensities)
+        n = min(len(ips), intens_arr.shape[0])
+        if n == 0:
+            return ips, intensities
+
+        # indices of top-N by descending intensity
+        top_idx = np.argsort(intens_arr[:n])[::-1][:my_max_spots]
+
+        if isinstance(ips, np.ndarray):
+            ips_filtered = ips[top_idx]
+        else:
+            ips_filtered = [ips[i] for i in top_idx]
+
+        intens_filtered = intens_arr[top_idx]
+        if isinstance(intensities, list):
+            intens_filtered = intens_filtered.tolist()
+
+        return ips_filtered, intens_filtered
+        
+    def filter(self):
+        """
+        Merge all interest-point chunks that fall inside the requested overlap intervals, 
+        then keep only the strongest points per interval (scaled by interval size)
+        """
+        ips_lists_by_view = defaultdict(list)
+        ints_lists_by_view = defaultdict(list)
+        intervals_by_view = defaultdict(list)
+
+        # Group incoming interest-point chunks by view
+        for entry in self.interest_points:
+            vid = entry["view_id"]
+            ips = entry["interest_points"]      
+            intens = entry["intensities"] 
+            interval = entry["interval_key"]     
+            ips_lists_by_view[vid].append(ips)
+            ints_lists_by_view[vid].append(intens)
+            intervals_by_view[vid].append(interval)
+
+        # Process each view from the image metadata table.
+        for i, row_i in self.image_loader_df.iterrows():
+            view_id = f"timepoint: {row_i['timepoint']}, setup: {row_i['view_setup']}"
+
+            ips_list = ips_lists_by_view[view_id]
+            intensities_list = ints_lists_by_view[view_id]
+            interval_list = intervals_by_view[view_id]
+
+            if not interval_list or not ips_list:
+                continue
+
+            interval_data = []
+
+            # Build the set of intervals to process from overlap metadata.
+            to_process = [
+                {'view_id': vid, **d}
+                for vid, lst in self.overlapping_area.items()
+                for d in lst
+            ]
+            
+            # Collect all chunks fully contained in each target interval
+            for row in to_process:
+                vid = row['view_id']
+                lb = row['lower_bound']
+                ub = row['upper_bound']
+                if vid == view_id:
+                    to_process_interval = (lb, ub)
+                    ips_block = []
+                    intensities_block = []
+
+                    for i in range(len(ips_list)): 
+                        block_interval = interval_list[i]
+                        
+                        # Merge all blocks that fall inside the target interval
+                        if self.contains(to_process_interval, block_interval):
+                            ips_block.extend(ips_list[i])
+                            intensities_block.extend(intensities_list[i])
+                    
+                    interval_data.append((to_process_interval, ips_block, intensities_block))
+            
+            ips_lists_by_view[view_id] = []
+            ints_lists_by_view[view_id] = []
+
+            # Cap the number of spots by interval size, then keep best
+            for interval, ips, intensities in interval_data:
+                size = self.size(interval)
+                my_max_spots = int(round(self.max_spots * (size / self.max_interval_size)))
+                
+                if my_max_spots > 0 and my_max_spots < len(ips):
+                    ips, intensities = self.filter_lists(ips, intensities, my_max_spots)
+
+                ips_lists_by_view[view_id].append(ips)
+                ints_lists_by_view[view_id].append(intensities)
+        
+        return ips_lists_by_view, ints_lists_by_view
+
+    def run(self):
+        """
+        Executes the entry point of the script.
+        """
+        ips_lists_by_view, ints_lits_by_view = self.filter()
+        self.kd_tree(ips_lists_by_view, ints_lits_by_view)
+        
+        return self.consolidated_data