nettracer3d 0.8.0__py3-none-any.whl → 0.8.2__py3-none-any.whl

nettracer3d/nettracer.py

@@ -348,6 +348,11 @@ def create_and_save_dataframe(pairwise_connections, excel_filename = None):
 
 #General supporting methods below:
 
+def invert_dict(d):
+    inverted = {}
+    for key, value in d.items():
+        inverted.setdefault(value, []).append(key)
+    return inverted
 
 def invert_array(array):
     """Internal method used to flip node array indices. 0 becomes 255 and vice versa."""
@@ -1484,6 +1489,13 @@ def remove_zeros(input_list):
     return result_array
 
 
+def overlay_arrays_simple(edge_labels_1, edge_labels_2):
+    """
+    Superimpose edge_labels_2 on top of edge_labels_1 without any offset.
+    Where edge_labels_2 > 0, use those values directly.
+    """
+    mask = edge_labels_1 > 0
+    return np.where(mask, edge_labels_1, edge_labels_2)
 
 def combine_edges(edge_labels_1, edge_labels_2):
     """
@@ -1616,7 +1628,7 @@ def downsample(data, factor, directory=None, order=0):
     return data
 
 
-def otsu_binarize(image_array):
+def otsu_binarize(image_array, non_bool = False):
 
     """Automated binarize method for seperating the foreground"""
 
@@ -1624,6 +1636,10 @@ def otsu_binarize(image_array):
 
     threshold = threshold_otsu(image_array)
     binary_mask = image_array > threshold
+
+    if non_bool:
+        binary_mask = binary_mask * 255
+        
     return binary_mask
 
 def binarize(arrayimage, directory = None):
@@ -1814,12 +1830,6 @@ def label_branches(array, peaks = 0, branch_removal = 0, comp_dil = 0, max_vol =
 
 def fix_branches_network(array, G, communities, fix_val = None):
 
-    def invert_dict(d):
-        inverted = {}
-        for key, value in d.items():
-            inverted.setdefault(value, []).append(key)
-        return inverted
-
     def get_degree_threshold(community_degrees):
         degrees = np.array(community_degrees, dtype=float)
         hist, bins = np.histogram(degrees, bins='auto')        
@@ -3893,11 +3903,6 @@ class Network_3D:
 
     def com_to_node(self, targets = None):
 
-        def invert_dict(d):
-            inverted = {}
-            for key, value in d.items():
-                inverted.setdefault(value, []).append(key)
-            return inverted
 
         def update_array(array_3d, value_dict, targets = None):
             ref_array = copy.deepcopy(array_3d)
@@ -4291,7 +4296,7 @@ class Network_3D:
 
 
 
-    def get_degrees(self, down_factor = 1, directory = None, called = False, no_img = 0):
+    def get_degrees(self, down_factor = 1, directory = None, called = False, no_img = 0, heatmap = False):
         """
         Method to obtain information on the degrees of nodes in the network, also generating overlays that relate this information to the 3D structure.
         Overlays include a grayscale image where nodes are assigned a grayscale value corresponding to their degree, and a numerical index where numbers are drawn at nodes corresponding to their degree.
@@ -4301,6 +4306,27 @@ class Network_3D:
         :returns: A dictionary of degree values for each node.
         """
 
+        if heatmap:
+            import statistics
+            degrees_dict = {node: val for (node, val) in self.network.degree()}
+            pred = statistics.mean(list(degrees_dict.values()))
+
+            node_intensity = {}
+            import math
+            node_centroids = {}
+
+            for node in list(self.network.nodes()):
+                node_intensity[node] = math.log(self.network.degree(node)/pred)
+                node_centroids[node] = self.node_centroids[node]
+
+            from . import neighborhoods
+
+            overlay = neighborhoods.create_node_heatmap(node_intensity, node_centroids, shape = self.nodes.shape, is_3d=True, labeled_array = self.nodes)
+
+            return degrees_dict, overlay
+
+
+
         if down_factor > 1:
             centroids = self._node_centroids.copy()
             for item in self._node_centroids:
@@ -4657,18 +4683,16 @@ class Network_3D:
                     dim = 3
                     break
 
-
         if ignore_dims:
 
             factor = 0.25
 
+            big_array = proximity.convert_centroids_to_array(list(self.node_centroids.values()))
 
             if bounds is None:
-                if dim == 2:
-                    min_coords = np.array([0,0])
-                else:
-                    min_coords = np.array([0,0,0])
-                max_coords = np.max(points_array, axis=0)
+                min_coords = np.array([0,0,0])
+                max_coords = [np.max(big_array[:, 0]), np.max(big_array[:, 1]), np.max(big_array[:, 2])]
+                del big_array
                 max_coords = np.flip(max_coords)
                 bounds = (min_coords, max_coords)
             else:
@@ -4683,11 +4707,16 @@ class Network_3D:
 
             new_list = []
 
+
             if dim == 3:
                 for centroid in roots:
 
                     if ((centroid[2] - min_coords[0]) > dim_list[0] * factor) and ((max_coords[0] - centroid[2]) > dim_list[0]  * factor) and ((centroid[1] - min_coords[1]) > dim_list[1] * factor) and ((max_coords[1] - centroid[1]) > dim_list[1] * factor) and ((centroid[0] - min_coords[2]) > dim_list[2] * factor) and ((max_coords[2] - centroid[0]) > dim_list[2] * factor):
                         new_list.append(centroid)
+
+
+                    #if ((centroid[2] - min_coords[0]) > dim_list[0] * factor) and ((max_coords[0] - centroid[2]) > dim_list[0]  * factor) and ((centroid[1] - min_coords[1]) > dim_list[1] * factor) and ((max_coords[1] - centroid[1]) > dim_list[1] * factor) and ((centroid[0] - min_coords[2]) > dim_list[2] * factor) and ((max_coords[2] - centroid[0]) > dim_list[2] * factor):
+                        #new_list.append(centroid)
                         #print(f"dim_list: {dim_list}, centroid: {centroid}, min_coords: {min_coords}, max_coords: {max_coords}")
             else:
                 for centroid in roots:
@@ -4807,9 +4836,9 @@ class Network_3D:
                 (z1, y1, x1), (z2, y2, x2) = bounds
                 z1, y1, x1 = int(z1), int(y1), int(x1)
                 z2, y2, x2 = int(z2), int(y2), int(x2)
-                z_range = np.arange(z1, z2 + 1)
-                y_range = np.arange(y1, y2 + 1)
-                x_range = np.arange(x1, x2 + 1)
+                z_range = np.arange(z1, z2 + 1 )
+                y_range = np.arange(y1, y2 + 1 )
+                x_range = np.arange(x1, x2 + 1 )
                 z_grid, y_grid, x_grid = np.meshgrid(z_range, y_range, x_range, indexing='ij')
                 del z_range
                 del y_range
@@ -4872,11 +4901,6 @@ class Network_3D:
 
     def community_id_info(self):
 
-        def invert_dict(d):
-            inverted = {}
-            for key, value in d.items():
-                inverted.setdefault(value, []).append(key)
-            return inverted
 
         community_dict = invert_dict(self.communities)
         summation = 0
@@ -4914,13 +4938,7 @@ class Network_3D:
 
         return output
 
-    def community_id_info_per_com(self, umap = False, label = False):
-
-        def invert_dict(d):
-            inverted = {}
-            for key, value in d.items():
-                inverted.setdefault(value, []).append(key)
-            return inverted
+    def community_id_info_per_com(self, umap = False, label = False, limit = 0, proportional = False):
 
         community_dict = invert_dict(self.communities)
         summation = 0
@@ -4930,62 +4948,122 @@ class Network_3D:
             id_dict[iden] = i
 
         output = {}
+        umap_dict = {}
 
-        for community in community_dict:
+        if not proportional:
 
-            counter = np.zeros(len(id_set))
+            for community in community_dict:
 
-            nodes = community_dict[community]
-            size = len(nodes)
+                counter = np.zeros(len(id_set))
 
-            # Count identities in this community
-            for node in nodes:
-                counter[id_dict[self.node_identities[node]]] += 1 # Keep them as arrays
+                nodes = community_dict[community]
+                size = len(nodes)
+
+                # Count identities in this community
+                for node in nodes:
+                    counter[id_dict[self.node_identities[node]]] += 1 # Keep them as arrays
+
+                for i in range(len(counter)): # Translate them into proportions out of 1
+
+                    counter[i] = counter[i]/size
 
-            for i in range(len(counter)): # Translate them into proportions out of 1
+                output[community] = counter #Assign the finding here
 
-                counter[i] = counter[i]/size
+                if size >= limit:
+                    umap_dict[community] = counter
+
+        else:
+            idens = invert_dict(self.node_identities)
+            iden_count = {}
+            template = {}
+            node_count = len(list(self.communities.keys()))
+
+
+            for iden in id_set:
+                template[iden] = 0
+
+            for iden, nodes in idens.items():
+                iden_count[iden] = len(nodes)
+
+            for community in community_dict:
+
+                iden_tracker = copy.deepcopy(template)
+
+                nodes = community_dict[community]
+                size = len(nodes)
+                counter = np.zeros(len(id_set))
+
+                for node in nodes:
+                    iden_tracker[self.node_identities[node]] += 1 
+
+                i = 0
+
+                if not umap: # External calls just get the proportion for now
+
+                    for iden, val in iden_tracker.items(): # Translate them into proportions of total number of that node of all nodes of that ID
+
+                        counter[i] = (val/iden_count[iden])
+                        i += 1
+
+                    output[community] = counter #Assign the finding here
+
+                    if size >= limit:
+                        umap_dict[community] = counter
+
+                else: # Internal calls for the umap get the relative proportion, demonstrating overrepresentation per community
+
+
+                    for iden, val in iden_tracker.items(): # Translate them into proportions of total number of that node of all nodes of that ID
+
+                        counter[i] = (val/iden_count[iden])/(size/node_count) # The proportion of that ID in the community vs all of that ID divided by the proportion of that community size vs all the nodes
+                        i += 1
+
+                    output[community] = counter #Assign the finding here
+
+                    if size >= limit:
+                        umap_dict[community] = counter
 
-            output[community] = counter #Assign the finding here
 
         if umap:
             from . import neighborhoods
-            neighborhoods.visualize_cluster_composition_umap(output, id_set, label = label) 
+
+            neighborhoods.visualize_cluster_composition_umap(umap_dict, id_set, label = label) 
 
         return output, id_set
 
 
-    def assign_neighborhoods(self, seed, count, limit = None, prev_coms = None):
+    def assign_neighborhoods(self, seed, count, limit = None, prev_coms = None, proportional = False, mode = 0):
 
         from . import neighborhoods
 
-        def invert_dict(d):
-            inverted = {}
-            for key, value in d.items():
-                inverted.setdefault(value, []).append(key)
-            return inverted
-
         if prev_coms is not None:
             self.communities = copy.deepcopy(prev_coms)
 
         identities, _ = self.community_id_info_per_com()
 
+        zero_group = {}
+
+
         if limit is not None:
 
             coms = invert_dict(self.communities)
 
-            zero_group = {}
 
             for com, nodes in coms.items():
 
                 if len(nodes) < limit:
 
-                    zero_group[com] = 0
-
                     del identities[com]
 
+        if count > len(identities):
+            print(f"Requested neighborhoods too large for available communities. Using {len(identities)} neighborhoods (max for these coms)")
+            count = len(identities)
+
 
-        clusters = neighborhoods.cluster_arrays(identities, count, seed = seed) # dict: {cluster_id: {'keys': [keys], 'arrays': [arrays]}}
+        if mode == 0:
+            clusters = neighborhoods.cluster_arrays(identities, count, seed = seed)
+        elif mode == 1:
+            clusters = neighborhoods.cluster_arrays_dbscan(identities, seed = seed)
 
         coms = {}
 
@@ -4997,19 +5075,60 @@ class Network_3D:
 
                 coms[com] = i + 1
 
-        if limit is not None:
-            coms.update(zero_group)
+        copy_dict = copy.deepcopy(self.communities)
+
+        for node, com in copy_dict.items():
+
+            try:
 
-        for node, com in self.communities.items():
+                self.communities[node] = coms[com]
+
+            except:
+                del self.communities[node]
+                zero_group[node] = 0
+
+        self.com_by_size()
+
+
+        if len(zero_group) > 0:
+            self.communities.update(zero_group)
 
-            self.communities[node] = coms[com]
 
         identities, id_set = self.community_id_info_per_com()
 
-        neighborhoods.plot_dict_heatmap(identities, id_set)
+        len_dict = {}
+
+        coms = invert_dict(self.communities)
+        node_count = len(list(self.communities.keys()))
+
+        for com, nodes in coms.items():
+
+            len_dict[com] = len(nodes)/node_count
+
+        matrixes = []
+
+        output = neighborhoods.plot_dict_heatmap(identities, id_set, title = "Neighborhood Heatmap by Proportional Composition Per Neighborhood")
+
+        matrixes.append(output)
+
+        if proportional:
+
+            identities2, id_set2 = self.community_id_info_per_com(proportional = True)
+            output = neighborhoods.plot_dict_heatmap(identities2, id_set2, title = "Neighborhood Heatmap by Proportional Composition of Nodes in Neighborhood vs All Nodes")
+            matrixes.append(output)
+
+            identities3 = {}
+            for iden in identities2:
+                identities3[iden] = identities2[iden]/len_dict[iden]
+
+            output = neighborhoods.plot_dict_heatmap(identities3, id_set2, title = "Neighborhood Heatmap by Proportional Composition of Nodes in Neighborhood vs All Nodes Divided by Neighborhood Total Proportion of All Nodes (val < 1 = underrepresented, val > 1 = overrepresented)", center_at_one = True)
+            matrixes.append(output)
+
+        return len_dict, matrixes, id_set
 
 
-    def kd_network(self, distance = 100, targets = None, make_array = False):
+
+    def kd_network(self, distance = 100, targets = None, make_array = False, max_neighbors = None):
 
         centroids = copy.deepcopy(self._node_centroids)
 
@@ -5030,14 +5149,20 @@ class Network_3D:
                     centroids[node] = [centroid[0], centroid[1] * refactor, centroid[2] * refactor]
 
 
-        neighbors = proximity.find_neighbors_kdtree(distance, targets = targets, centroids = centroids)
+        neighbors = proximity.find_neighbors_kdtree(distance, targets = targets, centroids = centroids, max_neighbors = max_neighbors)
+
+        print("Creating Dataframe")
 
         network = create_and_save_dataframe(neighbors)
 
+        print("Converting df to network")
+
         self._network_lists = network_analysis.read_excel_to_lists(network)
 
         #self._network is a networkx graph that stores the connections
 
+        print("Removing Edge Weights")
+
         self.remove_edge_weights()
 
         if make_array:
@@ -5048,7 +5173,7 @@ class Network_3D:
 
     def community_cells(self, size = 32, xy_scale = 1, z_scale = 1):
 
-        def invert_dict(d):
+        def revert_dict(d):
             inverted = {}
             for key, value_list in d.items():
                 for value in value_list:
@@ -5066,18 +5191,12 @@ class Network_3D:
 
             com_dict = proximity.partition_objects_into_cells(self.node_centroids, (size_z, size_x, size_x))
 
-        self.communities = invert_dict(com_dict)
+        self.communities = revert_dict(com_dict)
 
-    def community_heatmap(self, num_nodes = None, is3d = True):
+    def community_heatmap(self, num_nodes = None, is3d = True, numpy = False):
 
         import math
 
-        def invert_dict(d):
-            inverted = {}
-            for key, value in d.items():
-                inverted.setdefault(value, []).append(key)
-            return inverted
-
         if num_nodes == None:
 
             try:
@@ -5106,10 +5225,21 @@ class Network_3D:
         for com, nodes in coms.items():
             heat_dict[com] = math.log(len(nodes)/rand_dens)
 
+        try:
+            shape = self.nodes.shape
+        except:
+            big_array = proximity.convert_centroids_to_array(list(self.node_centroids.values()))
+            shape = [np.max(big_array[0, :]) + 1, np.max(big_array[1, :]) + 1, np.max(big_array[2, :]) + 1]
+
         from . import neighborhoods
-        neighborhoods.create_community_heatmap(heat_dict, self.communities, self.node_centroids, is_3d=is3d)
+        if not numpy:
+            neighborhoods.create_community_heatmap(heat_dict, self.communities, self.node_centroids, shape = shape, is_3d=is3d)
+
+            return heat_dict
+        else:
+            overlay = neighborhoods.create_community_heatmap(heat_dict, self.communities, self.node_centroids, shape = shape, is_3d=is3d, labeled_array = self.nodes)
+            return heat_dict, overlay
 
-        return heat_dict
 
     def merge_node_ids(self, path, data):
 
@@ -5161,6 +5291,84 @@ class Network_3D:
                     pass
 
 
+    def nearest_neighbors_avg(self, root, targ, xy_scale = 1, z_scale = 1, num = 1, heatmap = False, threed = True, numpy = False):
+
+        root_set = []
+
+        compare_set = []
+
+        if root is None:
+
+            root_set = list(self.node_centroids.keys())
+            compare_set = root_set
+            title = "Nearest Neighbors Between Nodes Heatmap"
+
+        else:
+
+            title = f"Nearest Neighbors of ID {targ} from ID {root} Heatmap"
+
+            for node, iden in self.node_identities.items():
+
+                if iden == root:
+
+                    root_set.append(node)
+
+                elif (iden == targ) or (targ == 'All Others (Excluding Self)'):
+
+                    compare_set.append(node)
+
+        if root == targ:
+
+            compare_set = root_set
+            if len(compare_set) - 1 < num:
+
+                num = len(compare_set) - 1
+
+                print(f"Error: Not enough neighbor nodes for requested number of neighbors. Using max available neighbors: {num}")
+                
+
+        if len(compare_set) < num:
+
+            num = len(compare_set)
+
+            print(f"Error: Not enough neighbor nodes for requested number of neighbors. Using max available neighbors: {num}")
+            
+        avg, output = proximity.average_nearest_neighbor_distances(self.node_centroids, root_set, compare_set, xy_scale=self.xy_scale, z_scale=self.z_scale, num = num)
+
+        if heatmap:
+
+
+            from . import neighborhoods
+            try:
+                shape = self.nodes.shape
+            except:
+                big_array = proximity.convert_centroids_to_array(list(self.node_centroids.values()))
+                shape = [np.max(big_array[0, :]) + 1, np.max(big_array[1, :]) + 1, np.max(big_array[2, :]) + 1]
+
+            pred = avg
+
+            node_intensity = {}
+            import math
+            node_centroids = {}
+
+            for node in root_set:
+                node_intensity[node] = math.log(pred/output[node])
+                node_centroids[node] = self.node_centroids[node]
+
+            if numpy:
+
+                overlay = neighborhoods.create_node_heatmap(node_intensity, node_centroids, shape = shape, is_3d=threed, labeled_array = self.nodes, colorbar_label="Clustering Intensity", title = title)
+
+                return avg, output, overlay
+
+            else:
+                neighborhoods.create_node_heatmap(node_intensity, node_centroids, shape = shape, is_3d=threed, labeled_array = None, colorbar_label="Clustering Intensity", title = title)
+
+        return avg, output
+
+
+
+