bmtool 0.6.0__py3-none-any.whl → 0.6.1__py3-none-any.whl

bmtool/SLURM.py

@@ -3,6 +3,7 @@ import os
 import subprocess
 import json
 import requests
+import shutil
 
 
 def check_job_status(job_id):
@@ -106,7 +107,6 @@ class seedSweep:
         
         print(f"JSON file '{self.json_file_path}' modified successfully with {self.param_name}={new_value}.", flush=True)
         
-    
     def change_json_file_path(self,new_json_file_path):
         self.json_file_path = new_json_file_path
 
@@ -156,8 +156,7 @@ class multiSeedSweep(seedSweep):
 
 
 class SimulationBlock:
-    def __init__(self, block_name, time, partition, nodes, ntasks, mem, simulation_cases, output_base_dir,account=None,additional_commands=None,
-                 status_list = ['COMPLETED', 'FAILED', 'CANCELLED']):
+    def __init__(self, block_name, time, partition, nodes, ntasks, mem, simulation_cases, output_base_dir,account=None,additional_commands=None,status_list = ['COMPLETED', 'FAILED', 'CANCELLED'],component_path=None):
         """
         Initializes the SimulationBlock instance.
 
@@ -187,6 +186,7 @@ class SimulationBlock:
         self.additional_commands = additional_commands if additional_commands is not None else []
         self.status_list = status_list
         self.job_ids = []
+        self.component_path = component_path
 
     def create_batch_script(self, case_name, command):
         """
@@ -207,6 +207,8 @@ class SimulationBlock:
         additional_commands_str = "\n".join(self.additional_commands)
         # Conditional account linegit
         account_line = f"#SBATCH --account={self.account}\n" if self.account else ""
+        env_var_component_path = f"export COMPONENT_PATH={self.component_path}" if self.component_path else ""
+        
 
         # Write the batch script to the file
         with open(batch_script_path, 'w') as script_file:
@@ -224,6 +226,9 @@ class SimulationBlock:
 # Additional user-defined commands
 {additional_commands_str}
 
+#enviroment vars
+{env_var_component_path}
+
 export OUTPUT_DIR={case_output_dir}
 
 {command}
@@ -272,7 +277,6 @@ export OUTPUT_DIR={case_output_dir}
                 return False
         return True
 
-
     def check_block_running(self):
         """checks if a job is running
 
@@ -284,9 +288,21 @@ export OUTPUT_DIR={case_output_dir}
             if status != 'RUNNING': # 
                 return False
         return True
+    
+    def check_block_submited(self):
+        """checks if a job is running
 
+        Returns:
+            bool: True if jobs are RUNNING false if anything else
+        """
+        for job_id in self.job_ids:
+            status = check_job_status(job_id)
+            if status != 'PENDING': # 
+                return False
+        return True
 
-class SequentialBlockRunner:
+
+class BlockRunner:
     """
     Class to handle submitting multiple blocks sequentially.
 
@@ -297,17 +313,23 @@ class SequentialBlockRunner:
         webhook (str): a microsoft webhook for teams. When used will send teams messages to the hook!
     """
 
-    def __init__(self, blocks, json_editor=None, param_values=None, check_interval=200,webhook=None):
+    def __init__(self, blocks, json_editor=None,json_file_path=None, param_name=None, 
+                 param_values=None, check_interval=60,syn_dict_list = None,
+                 webhook=None):
         self.blocks = blocks
         self.json_editor = json_editor
         self.param_values = param_values
         self.check_interval = check_interval
         self.webhook = webhook
+        self.param_name = param_name
+        self.json_file_path = json_file_path
+        self.syn_dict_list = syn_dict_list
 
     def submit_blocks_sequentially(self):
         """
-        Submits all blocks sequentially, ensuring each block starts only after the previous block has completed.
+        Submits all blocks sequentially, ensuring each block starts only after the previous block has completed or is running.
         Updates the JSON file with new parameters before each block run.
+        json file path should be the path WITH the components folder 
         """
         for i, block in enumerate(self.blocks):
             # Update JSON file with new parameter value
@@ -317,15 +339,14 @@ class SequentialBlockRunner:
                 if len(self.blocks) != len(self.param_values):
                     raise Exception("Number of blocks needs to each number of params given")
                 new_value = self.param_values[i]
-                # NGL didnt test the multi but should work
-                if isinstance(self.json_editor, multiSeedSweep):
-                    self.json_editor.edit_all_jsons(new_value)
-                elif isinstance(self.json_editor,seedSweep):
-                    print(f"Updating JSON file with parameter value for block: {block.block_name}", flush=True)
-                    self.json_editor.edit_json(new_value)
-                else:
-                    raise Exception("json editor provided but not a seedSweep class not sure what your doing?!?")
                 
+                if self.syn_dict_list == None:
+                    json_editor = seedSweep(self.json_file_path, self.param_name)
+                    json_editor.edit_json(new_value)
+                else:
+                    json_editor = multiSeedSweep(self.json_file_path,self.param_name,
+                                                self.syn_dict_list,base_ratio=1)
+                    json_editor.edit_all_jsons(new_value) 
 
             # Submit the block
             print(f"Submitting block: {block.block_name}", flush=True)
@@ -335,7 +356,7 @@ class SequentialBlockRunner:
                 send_teams_message(self.webhook,message)
 
             # Wait for the block to complete
-            if i == len(self.blocks) - 1:  # Corrected index to check the last block
+            if i == len(self.blocks) - 1:  
                 while not block.check_block_completed():
                     print(f"Waiting for the last block {i} to complete...")
                     time.sleep(self.check_interval)
@@ -350,3 +371,43 @@ class SequentialBlockRunner:
             message = "SIMULATION UPDATE: Simulation are Done!"
             send_teams_message(self.webhook,message)
 
+    def submit_blocks_parallel(self):
+        """
+        submits all the blocks at once onto the queue. To do this the components dir will be cloned and each block will have its own.
+        Also the json_file_path should be the path after the components dir
+        """
+        if self.webhook:
+            message = "SIMULATION UPDATE: Simulations have been submited in parallel!"
+            send_teams_message(self.webhook,message)
+        for i, block in enumerate(self.blocks):
+            if block.component_path == None:
+                raise Exception("Unable to use parallel submitter without defining the component path")
+            new_value = self.param_values[i]
+            
+            source_dir = block.component_path
+            destination_dir = f"{source_dir}{i+1}"
+            block.component_path = destination_dir
+
+            shutil.copytree(source_dir, destination_dir) # create new components folder 
+            json_file_path = os.path.join(destination_dir,self.json_file_path)
+            if self.syn_dict_list == None:
+                json_editor = seedSweep(json_file_path, self.param_name)
+                json_editor.edit_json(new_value)
+            else:
+                json_editor = multiSeedSweep(json_file_path,self.param_name,
+                                             self.syn_dict_list,base_ratio=1)
+                json_editor.edit_all_jsons(new_value)  
+            
+            # submit block with new component path 
+            print(f"Submitting block: {block.block_name}", flush=True)
+            block.submit_block()
+            if i == len(self.blocks) - 1:  
+                while not block.check_block_completed():
+                    print(f"Waiting for the last block {i} to complete...")
+                    time.sleep(self.check_interval)
+                
+        if self.webhook:
+            message = "SIMULATION UPDATE: Simulations are Done!"
+            send_teams_message(self.webhook,message)
+            
+            

bmtool/bmplot.py

@@ -306,46 +306,47 @@ def gap_junction_matrix(config=None,title=None,sources=None, targets=None, sids=
     
     
     def filter_rows(syn_info, data, source_labels, target_labels):
-        new_syn_info = syn_info
-        new_data = data
-        new_source_labels = source_labels
-        new_target_labels = target_labels
-        for row in new_data:
-            row_index = -1
-            try:
-                if((np.isnan(row).all())): #checks if all of a row is nan
-                    row_index = np.where(np.isnan(new_data)==np.isnan(row))[0][0]  
-            except:
-                row_index = -1
-            finally:          
-                if(all(x==0 for x in row)): #checks if all of a row is zeroes
-                    row_index = np.where(new_data==row)[0][0] 
-                if row_index!=-1:   #deletes corresponding row accordingly in all relevant variables.
-                    new_syn_info = np.delete(new_syn_info,row_index,0) 
-                    new_data = np.delete(new_data,row_index,0)
-                    new_source_labels = np.delete(new_source_labels,row_index)
-        return new_syn_info, new_data,new_source_labels,new_target_labels
-
-    def filter_rows_and_columns(syn_info,data,source_labels,target_labels):
+        # Identify rows with all NaN or all zeros
+        valid_rows = ~np.all(np.isnan(data), axis=1) & ~np.all(data == 0, axis=1)
+
+        # Filter rows based on valid_rows mask
+        new_syn_info = syn_info[valid_rows]
+        new_data = data[valid_rows]
+        new_source_labels = np.array(source_labels)[valid_rows]
+
+        return new_syn_info, new_data, new_source_labels, target_labels
+
+    def filter_rows_and_columns(syn_info, data, source_labels, target_labels):
+        # Filter rows first
         syn_info, data, source_labels, target_labels = filter_rows(syn_info, data, source_labels, target_labels)
-        transposed_syn_info = np.transpose(syn_info) #transpose everything and put it in to make sure columns get filtered
+
+        # Transpose data to filter columns
+        transposed_syn_info = np.transpose(syn_info)
         transposed_data = np.transpose(data)
         transposed_source_labels = target_labels
         transposed_target_labels = source_labels
-        syn_info, data, source_labels, target_labels = filter_rows(transposed_syn_info, transposed_data, transposed_source_labels, transposed_target_labels)
-        filtered_syn_info = np.transpose(syn_info) #transpose everything back to original order after filtering.
-        filtered_data = np.transpose(data)
-        filtered_source_labels = target_labels
-        filtered_target_labels = source_labels
-        return filtered_syn_info,filtered_data,filtered_source_labels,filtered_target_labels
+
+        # Filter columns (by treating them as rows in transposed data)
+        transposed_syn_info, transposed_data, transposed_source_labels, transposed_target_labels = filter_rows(
+            transposed_syn_info, transposed_data, transposed_source_labels, transposed_target_labels
+        )
+
+        # Transpose back to original orientation
+        filtered_syn_info = np.transpose(transposed_syn_info)
+        filtered_data = np.transpose(transposed_data)
+        filtered_source_labels = transposed_target_labels  # Back to original source_labels
+        filtered_target_labels = transposed_source_labels  # Back to original target_labels
+
+        return filtered_syn_info, filtered_data, filtered_source_labels, filtered_target_labels
+
     
     syn_info, data, source_labels, target_labels = filter_rows_and_columns(syn_info, data, source_labels, target_labels)
 
     if title == None or title=="":
         title = 'Gap Junction'
-        if type == 'convergence':
+        if method == 'convergence':
             title+=' Syn Convergence'
-        elif type == 'percent':
+        elif method == 'percent':
             title+=' Percent Connectivity'
     plot_connection_info(syn_info,data,source_labels,target_labels,title, save_file=save_file)
     return
@@ -535,35 +536,42 @@ def edge_histogram_matrix(config=None,sources = None,targets=None,sids=None,tids
     fig.text(0.04, 0.5, 'Source', va='center', rotation='vertical')
     plt.draw()
 
-def plot_connection_info(text, num, source_labels,target_labels, title, syn_info='0', save_file=None,return_dict=None):
+def plot_connection_info(text, num, source_labels, target_labels, title, syn_info='0', save_file=None, return_dict=None):
     """
-    write about function here
+    Function to plot connection information as a heatmap, including handling missing source and target values.
+    If there is no source or target, set the value to 0.
     """
     
-    #num = pd.DataFrame(num).fillna('nc').to_numpy() # replace nan with nc * does not work with imshow
+    # Ensure text dimensions match num dimensions
+    num_source = len(source_labels)
+    num_target = len(target_labels)
     
-    num_source=len(source_labels)
-    num_target=len(target_labels)
+    # Set color map
     matplotlib.rc('image', cmap='viridis')
     
-    fig1, ax1 = plt.subplots(figsize=(num_source,num_target))
+    # Create figure and axis for the plot
+    fig1, ax1 = plt.subplots(figsize=(num_source, num_target))
+    num = np.nan_to_num(num, nan=0) # replace NaN with 0
     im1 = ax1.imshow(num)
-    #fig.colorbar(im, ax=ax,shrink=0.4)
-    # We want to show all ticks...
+    
+    # Set ticks and labels for source and target
     ax1.set_xticks(list(np.arange(len(target_labels))))
     ax1.set_yticks(list(np.arange(len(source_labels))))
-    # ... and label them with the respective list entries
     ax1.set_xticklabels(target_labels)
-    ax1.set_yticklabels(source_labels,size=12, weight = 'semibold')
-    # Rotate the tick labels and set their alignment.
+    ax1.set_yticklabels(source_labels, size=12, weight='semibold')
+    
+    # Rotate the tick labels for better visibility
     plt.setp(ax1.get_xticklabels(), rotation=45, ha="right",
-            rotation_mode="anchor", size=12, weight = 'semibold')
+             rotation_mode="anchor", size=12, weight='semibold')
     
+    # Dictionary to store connection information
     graph_dict = {}
-    # Loop over data dimensions and create text annotations.
+    
+    # Loop over data dimensions and create text annotations
     for i in range(num_source):
         for j in range(num_target):
-            edge_info = text[i, j]
+            # Get the edge info, or set it to '0' if it's missing
+            edge_info = text[i, j] if text[i, j] is not None else 0
             
             # Initialize the dictionary for the source node if not already done
             if source_labels[i] not in graph_dict:
@@ -571,35 +579,41 @@ def plot_connection_info(text, num, source_labels,target_labels, title, syn_info
             
             # Add edge info for the target node
             graph_dict[source_labels[i]][target_labels[j]] = edge_info
-            if syn_info =='2' or syn_info =='3':
-                if num_source > 8 and num_source <20:
+            
+            # Set text annotations based on syn_info type
+            if syn_info == '2' or syn_info == '3':
+                if num_source > 8 and num_source < 20:
                     fig_text = ax1.text(j, i, edge_info,
-                            ha="center", va="center", color="w",rotation=37.5, size=8, weight = 'semi\bold')
+                                        ha="center", va="center", color="w", rotation=37.5, size=8, weight='semibold')
                 elif num_source > 20:
                     fig_text = ax1.text(j, i, edge_info,
-                            ha="center", va="center", color="w",rotation=37.5, size=7, weight = 'semibold')
+                                        ha="center", va="center", color="w", rotation=37.5, size=7, weight='semibold')
                 else:
                     fig_text = ax1.text(j, i, edge_info,
-                            ha="center", va="center", color="w",rotation=37.5, size=11, weight = 'semibold')
+                                        ha="center", va="center", color="w", rotation=37.5, size=11, weight='semibold')
             else:
                 fig_text = ax1.text(j, i, edge_info,
-                            ha="center", va="center", color="w", size=11, weight = 'semibold')
-
-    ax1.set_ylabel('Source', size=11, weight = 'semibold')
-    ax1.set_xlabel('Target', size=11, weight = 'semibold')
-    ax1.set_title(title,size=20, weight = 'semibold')
-    #plt.tight_layout()
-    notebook = is_notebook()
+                                    ha="center", va="center", color="w", size=11, weight='semibold')
+    
+    # Set labels and title for the plot
+    ax1.set_ylabel('Source', size=11, weight='semibold')
+    ax1.set_xlabel('Target', size=11, weight='semibold')
+    ax1.set_title(title, size=20, weight='semibold')
+    
+    # Display the plot or save it based on the environment and arguments
+    notebook = is_notebook()  # Check if running in a Jupyter notebook
     if notebook == False:
         fig1.show()
+    
     if save_file:
         plt.savefig(save_file)
+    
     if return_dict:
         return graph_dict
     else:
         return
 
-def connector_percent_matrix(csv_path: str = None, exclude_strings=None, title: str = 'Percent connection matrix', pop_order=None) -> None:
+def connector_percent_matrix(csv_path: str = None, exclude_strings=None, assemb_key=None, title: str = 'Percent connection matrix', pop_order=None) -> None:
     """
     Generates and plots a connection matrix based on connection probabilities from a CSV file produced by bmtool.connector.
 
@@ -633,6 +647,7 @@ def connector_percent_matrix(csv_path: str = None, exclude_strings=None, title:
     # Filter the DataFrame based on exclude_strings
     def filter_dataframe(df, column_name, exclude_strings):
         def process_string(string):
+            
             match = re.search(r"\[\'(.*?)\'\]", string)
             if exclude_strings and any(ex_string in string for ex_string in exclude_strings):
                 return None
@@ -640,17 +655,55 @@ def connector_percent_matrix(csv_path: str = None, exclude_strings=None, title:
                 filtered_string = match.group(1)
                 if 'Gap' in string:
                         filtered_string = filtered_string + "-Gap"
+                if assemb_key in string:
+                    filtered_string = filtered_string + assemb_key
                 return filtered_string  # Return matched string
 
             return string  # If no match, return the original string
-
+        
         df[column_name] = df[column_name].apply(process_string)
         df = df.dropna(subset=[column_name])
+        
         return df
 
     df = filter_dataframe(df, 'Source', exclude_strings)
     df = filter_dataframe(df, 'Target', exclude_strings)
+    
+    #process assem rows and combine them into one prob per assem type
+    assems = df[df['Source'].str.contains(assemb_key)]
+    unique_sources = assems['Source'].unique()
 
+    for source in unique_sources:
+        source_assems = assems[assems['Source'] == source]
+        unique_targets = source_assems['Target'].unique()  # Filter targets for the current source
+
+        for target in unique_targets:
+            # Filter the assemblies with the current source and target
+            unique_assems = source_assems[source_assems['Target'] == target]
+            
+            # find the prob of a conn
+            forward_probs = []
+            for _,row in unique_assems.iterrows():
+                selected_percentage = row[selected_column]
+                selected_percentage = [float(p) for p in selected_percentage.strip('[]').split()]
+                if len(selected_percentage) == 1 or len(selected_percentage) == 2:
+                    forward_probs.append(selected_percentage[0])
+                if len(selected_percentage) == 3:
+                    forward_probs.append(selected_percentage[0])
+                    forward_probs.append(selected_percentage[1])
+                    
+            mean_probs = np.mean(forward_probs)
+            source = source.replace(assemb_key, "")
+            target = target.replace(assemb_key, "")
+            new_row = pd.DataFrame({
+                'Source': [source],
+                'Target': [target],
+                'Percent connectionivity within possible connections': [mean_probs],
+                'Percent connectionivity within all connections': [0]
+            })
+
+            df = pd.concat([df, new_row], ignore_index=False)
+            
     # Prepare connection data
     connection_data = {}
     for _, row in df.iterrows():
@@ -671,14 +724,18 @@ def connector_percent_matrix(csv_path: str = None, exclude_strings=None, title:
         if source in populations and target in populations:
             source_idx = populations.index(source)
             target_idx = populations.index(target)
-            connection_matrix[source_idx][target_idx] = probabilities[0]
-            if len(probabilities) == 1:
+
+            if type(probabilities) == float:
+                connection_matrix[source_idx][target_idx] = probabilities
+            elif len(probabilities) == 1:
                 connection_matrix[source_idx][target_idx] = probabilities[0]
-            if len(probabilities) == 2:
+            elif len(probabilities) == 2:
                 connection_matrix[source_idx][target_idx] = probabilities[0]
-            if len(probabilities) == 3:
+            elif len(probabilities) == 3:
                 connection_matrix[source_idx][target_idx] = probabilities[0] 
                 connection_matrix[target_idx][source_idx] = probabilities[1]
+            else:
+                raise Exception("unsupported format")
 
     # Plotting
     fig, ax = plt.subplots(figsize=(10, 8))

{bmtool-0.6.0.dist-info → bmtool-0.6.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bmtool
-Version: 0.6.0
+Version: 0.6.1
 Summary: BMTool
 Home-page: https://github.com/cyneuro/bmtool
 Download-URL: 

{bmtool-0.6.0.dist-info → bmtool-0.6.1.dist-info}/RECORD

@@ -1,7 +1,7 @@
-bmtool/SLURM.py,sha256=AX5MKV7dD-XwS8SROnW1IyesZ3jDwMf0txO6mHxTbuw,13694
+bmtool/SLURM.py,sha256=woANwh6ToP5nsEkVX1qlINvkfGMp3P7Roq4O8X611M8,16371
 bmtool/__init__.py,sha256=ZStTNkAJHJxG7Pwiy5UgCzC4KlhMS5pUNPtUJZVwL_Y,136
 bmtool/__main__.py,sha256=TmFkmDxjZ6250nYD4cgGhn-tbJeEm0u-EMz2ajAN9vE,650
-bmtool/bmplot.py,sha256=vIknbCwBSJNsr4m53SymteKdvyE0Omn3Xv8udzroeKc,51825
+bmtool/bmplot.py,sha256=HuzBRrVsD6xFM-siWdT_t6XqffFhK6LFcGF7RttR6pQ,54013
 bmtool/connectors.py,sha256=2vVUsqYMaCuWZ-4C5eUzqwsFItFM9vm0ytZdRQdWgoc,72243
 bmtool/graphs.py,sha256=K8BiughRUeXFVvAgo8UzrwpSClIVg7UfmIcvtEsEsk0,6020
 bmtool/manage.py,sha256=_lCU0qBQZ4jSxjzAJUd09JEetb--cud7KZgxQFbLGSY,657
@@ -16,9 +16,9 @@ bmtool/util/commands.py,sha256=zJF-fiLk0b8LyzHDfvewUyS7iumOxVnj33IkJDzux4M,64396
 bmtool/util/util.py,sha256=00vOAwTVIifCqouBoFoT0lBashl4fCalrk8fhg_Uq4c,56654
 bmtool/util/neuron/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 bmtool/util/neuron/celltuner.py,sha256=xSRpRN6DhPFz4q5buq_W8UmsD7BbUrkzYBEbKVloYss,87194
-bmtool-0.6.0.dist-info/LICENSE,sha256=qrXg2jj6kz5d0EnN11hllcQt2fcWVNumx0xNbV05nyM,1068
-bmtool-0.6.0.dist-info/METADATA,sha256=8qbBKGJWEedc2N_xdm9OYb66bCQScXxb2faMQvdJvsQ,19113
-bmtool-0.6.0.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
-bmtool-0.6.0.dist-info/entry_points.txt,sha256=0-BHZ6nUnh0twWw9SXNTiRmKjDnb1VO2DfG_-oprhAc,45
-bmtool-0.6.0.dist-info/top_level.txt,sha256=gpd2Sj-L9tWbuJEd5E8C8S8XkNm5yUE76klUYcM-eWM,7
-bmtool-0.6.0.dist-info/RECORD,,
+bmtool-0.6.1.dist-info/LICENSE,sha256=qrXg2jj6kz5d0EnN11hllcQt2fcWVNumx0xNbV05nyM,1068
+bmtool-0.6.1.dist-info/METADATA,sha256=Ir2-vUb_0qfRxIJlj5iDuKVbnp5gFe6SOsqgCBWRdU4,19113
+bmtool-0.6.1.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
+bmtool-0.6.1.dist-info/entry_points.txt,sha256=0-BHZ6nUnh0twWw9SXNTiRmKjDnb1VO2DfG_-oprhAc,45
+bmtool-0.6.1.dist-info/top_level.txt,sha256=gpd2Sj-L9tWbuJEd5E8C8S8XkNm5yUE76klUYcM-eWM,7
+bmtool-0.6.1.dist-info/RECORD,,