bmtool 0.7.2.1__py3-none-any.whl → 0.7.4__py3-none-any.whl

bmtool/bmplot/connections.py

@@ -13,6 +13,8 @@ import numpy as np
 import pandas as pd
 from IPython import get_ipython
 
+from neuron import h
+
 from ..util import util
 
 use_description = """
@@ -132,6 +134,7 @@ def total_connection_matrix(
         title = "All Synapse .mod Files Used"
     if synaptic_info == "3":
         title = "All Synapse .json Files Used"
+    
     plot_connection_info(
         text, num, source_labels, target_labels, title, syn_info=synaptic_info, save_file=save_file
     )
@@ -980,9 +983,10 @@ def distance_delay_plot(
     plt.show()
 
 
-def plot_synapse_location(config: str, source: str, target: str, sids: str, tids: str) -> tuple:
+def plot_synapse_location(config: str, source: str, target: str, sids: str, tids: str, syn_feature: str = 'afferent_section_id') -> tuple:
     """
     Generates a connectivity matrix showing synaptic distribution across different cell sections.
+    Note does exclude gap junctions since they dont have an afferent id stored in the h5 file!
 
     Parameters
     ----------
@@ -996,6 +1000,8 @@ def plot_synapse_location(config: str, source: str, target: str, sids: str, tids
         Column name in nodes file containing source population identifiers
     tids : str
         Column name in nodes file containing target population identifiers
+    syn_feature : str, default 'afferent_section_id'
+        Synaptic feature to analyze ('afferent_section_id' or 'afferent_section_pos')
 
     Returns
     -------
@@ -1009,37 +1015,50 @@ def plot_synapse_location(config: str, source: str, target: str, sids: str, tids
     RuntimeError
         If template loading or cell instantiation fails
     """
-    import matplotlib.pyplot as plt
-    import numpy as np
-    from neuron import h
-
     # Validate inputs
     if not all([config, source, target, sids, tids]):
         raise ValueError(
             "Missing required parameters: config, source, target, sids, and tids must be provided"
         )
 
+    # Fix the validation logic - it was using 'or' instead of 'and'
+    if syn_feature not in ["afferent_section_id", "afferent_section_pos"]:
+        raise ValueError("Currently only syn features supported are afferent_section_id or afferent_section_pos")
+
     try:
         # Load mechanisms and template
         util.load_templates_from_config(config)
     except Exception as e:
         raise RuntimeError(f"Failed to load templates from config: {str(e)}")
-
+    
     try:
         # Load node and edge data
         nodes, edges = util.load_nodes_edges_from_config(config)
         if source not in nodes or f"{source}_to_{target}" not in edges:
             raise ValueError(f"Source '{source}' or target '{target}' networks not found in data")
 
-        nodes = nodes[source]
+        target_nodes = nodes[target]
+        source_nodes = nodes[source]
         edges = edges[f"{source}_to_{target}"]
+        
+        # Find edges with NaN values in the specified feature
+        nan_edges = edges[edges[syn_feature].isna()]
+        # Print information about removed edges
+        if not nan_edges.empty:
+            unique_indices = sorted(list(set(nan_edges.index.tolist())))
+            print(f"Removing {len(nan_edges)} edges with missing {syn_feature}")
+            print(f"Unique indices removed: {unique_indices}")
+            
+        # Filter out edges with NaN values in the specified feature
+        edges = edges[edges[syn_feature].notna()]
+
     except Exception as e:
         raise RuntimeError(f"Failed to load nodes and edges: {str(e)}")
 
     # Map identifiers while checking for missing values
-    edges["target_model_template"] = edges["target_node_id"].map(nodes["model_template"])
-    edges["target_pop_name"] = edges["target_node_id"].map(nodes[tids])
-    edges["source_pop_name"] = edges["source_node_id"].map(nodes[sids])
+    edges["target_model_template"] = edges["target_node_id"].map(target_nodes["model_template"])
+    edges["target_pop_name"] = edges["target_node_id"].map(target_nodes[tids])
+    edges["source_pop_name"] = edges["source_node_id"].map(source_nodes[sids])
 
     if edges["target_model_template"].isnull().any():
         print("Warning: Some target nodes missing model template")
@@ -1101,7 +1120,7 @@ def plot_synapse_location(config: str, source: str, target: str, sids: str, tids
                 section_mapping = section_mappings[target_model_template]
 
                 # Calculate section distribution
-                section_counts = filtered_edges["afferent_section_id"].value_counts()
+                section_counts = filtered_edges[syn_feature].value_counts()
                 section_percentages = (section_counts / total_connections * 100).round(1)
 
                 # Format section distribution text - show all sections
@@ -1110,16 +1129,17 @@ def plot_synapse_location(config: str, source: str, target: str, sids: str, tids
                     section_name = section_mapping.get(section_id, f"sec_{section_id}")
                     section_display.append(f"{section_name}:{percentage}%")
 
+
                 num_connections[source_idx, target_idx] = total_connections
                 text_data[source_idx, target_idx] = "\n".join(section_display)
 
             except Exception as e:
                 print(f"Warning: Error processing {target_model_template}: {str(e)}")
                 num_connections[source_idx, target_idx] = total_connections
-                text_data[source_idx, target_idx] = "Section info N/A"
+                text_data[source_idx, target_idx] = "Feature info N/A"
 
     # Create the plot
-    title = f"Synaptic Distribution by Section: {source} to {target}"
+    title = f"Synaptic Distribution by {syn_feature.replace('_', ' ').title()}: {source} to {target}"
     fig, ax = plot_connection_info(
         text=text_data,
         num=num_connections,

bmtool/bmplot/lfp.py

@@ -1,7 +1,6 @@
 import matplotlib.pyplot as plt
 import numpy as np
-
-from bmtool.analysis.lfp import gen_aperiodic
+from fooof.sim.gen import gen_aperiodic
 
 
 def plot_spectrogram(

bmtool/synapses.py

@@ -20,68 +20,104 @@ from tqdm.notebook import tqdm
 
 from bmtool.util.util import load_templates_from_config
 
+DEFAULT_GENERAL_SETTINGS = {
+    "vclamp": True,
+    "rise_interval": (0.1, 0.9),
+    "tstart": 500.0,
+    "tdur": 100.0,
+    "threshold": -15.0,
+    "delay": 1.3,
+    "weight": 1.0,
+    "dt": 0.025,
+    "celsius": 20,
+}
+
+DEFAULT_GAP_JUNCTION_GENERAL_SETTINGS = {
+    "tstart": 500.0,
+    "tdur": 500.0,
+    "dt": 0.025,
+    "celsius": 20,
+}
+
 
 class SynapseTuner:
     def __init__(
         self,
-        mechanisms_dir: str = None,
-        templates_dir: str = None,
-        config: str = None,
-        conn_type_settings: dict = None,
-        connection: str = None,
-        general_settings: dict = None,
-        json_folder_path: str = None,
+        mechanisms_dir: Optional[str] = None,
+        templates_dir: Optional[str] = None,
+        config: Optional[str] = None,
+        conn_type_settings: Optional[dict] = None,
+        connection: Optional[str] = None,
+        general_settings: Optional[dict] = None,
+        json_folder_path: Optional[str] = None,
         current_name: str = "i",
-        other_vars_to_record: list = None,
-        slider_vars: list = None,
+        other_vars_to_record: Optional[list] = None,
+        slider_vars: Optional[list] = None,
+        hoc_cell: Optional[object] = None,
     ) -> None:
         """
-        Initialize the SynapseModule class with connection type settings, mechanisms, and template directories.
+        Initialize the SynapseTuner class with connection type settings, mechanisms, and template directories.
 
         Parameters:
         -----------
-        mechanisms_dir : str
+        mechanisms_dir : Optional[str]
             Directory path containing the compiled mod files needed for NEURON mechanisms.
-        templates_dir : str
+        templates_dir : Optional[str]
             Directory path containing cell template files (.hoc or .py) loaded into NEURON.
-        conn_type_settings : dict
+        conn_type_settings : Optional[dict]
             A dictionary containing connection-specific settings, such as synaptic properties and details.
-        connection : str
+        connection : Optional[str]
             Name of the connection type to be used from the conn_type_settings dictionary.
-        general_settings : dict
+        general_settings : Optional[dict]
             General settings dictionary including parameters like simulation time step, duration, and temperature.
-        json_folder_path : str, optional
+        json_folder_path : Optional[str]
             Path to folder containing JSON files with additional synaptic properties to update settings.
         current_name : str, optional
             Name of the synaptic current variable to be recorded (default is 'i').
-        other_vars_to_record : list, optional
+        other_vars_to_record : Optional[list]
             List of additional synaptic variables to record during the simulation (e.g., 'Pr', 'Use').
-        slider_vars : list, optional
+        slider_vars : Optional[list]
             List of synaptic variables you would like sliders set up for the STP sliders method by default will use all parameters in spec_syn_param.
-
+        hoc_cell : Optional[object]
+            An already loaded NEURON cell object. If provided, template loading and cell setup will be skipped.
         """
-        if config is None and (mechanisms_dir is None or templates_dir is None):
-            raise ValueError(
-                "Either a config file or both mechanisms_dir and templates_dir must be provided."
-            )
+        self.hoc_cell = hoc_cell
 
-        if config is None:
-            neuron.load_mechanisms(mechanisms_dir)
-            h.load_file(templates_dir)
-        else:
-            # loads both mech and templates
-            load_templates_from_config(config)
+        if hoc_cell is None:
+            if config is None and (mechanisms_dir is None or templates_dir is None):
+                raise ValueError(
+                    "Either a config file, both mechanisms_dir and templates_dir, or a hoc_cell must be provided."
+                )
 
-        self.conn_type_settings = conn_type_settings
+            if config is None:
+                neuron.load_mechanisms(mechanisms_dir)
+                h.load_file(templates_dir)
+            else:
+                # loads both mech and templates
+                load_templates_from_config(config)
+
+        if conn_type_settings is None:
+            raise ValueError("conn_type_settings must be provided.")
+        if connection is None:
+            raise ValueError("connection must be provided.")
+        if connection not in conn_type_settings:
+            raise ValueError(f"connection '{connection}' not found in conn_type_settings.")
+
+        self.conn_type_settings: dict = conn_type_settings
         if json_folder_path:
             print(f"updating settings from json path {json_folder_path}")
             self._update_spec_syn_param(json_folder_path)
-        self.general_settings = general_settings
+        # Use default general settings if not provided
+        if general_settings is None:
+            self.general_settings: dict = DEFAULT_GENERAL_SETTINGS.copy()
+        else:
+            # Merge defaults with user-provided
+            self.general_settings = {**DEFAULT_GENERAL_SETTINGS, **general_settings}
         self.conn = self.conn_type_settings[connection]
         self.synaptic_props = self.conn["spec_syn_param"]
-        self.vclamp = general_settings["vclamp"]
+        self.vclamp = self.general_settings["vclamp"]
         self.current_name = current_name
-        self.other_vars_to_record = other_vars_to_record
+        self.other_vars_to_record = other_vars_to_record or []
         self.ispk = None
 
         if slider_vars:
@@ -94,25 +130,26 @@ class SynapseTuner:
                 # If the key is missing from synaptic_props, get the value using getattr
                 if key not in self.synaptic_props:
                     try:
-                        # Get the alternative value from getattr dynamically
                         self._set_up_cell()
                         self._set_up_synapse()
                         value = getattr(self.syn, key)
-                        # print(value)
                         self.slider_vars[key] = value
                     except AttributeError as e:
                         print(f"Error accessing '{key}' in syn {self.syn}: {e}")
-
         else:
             self.slider_vars = self.synaptic_props
 
-        h.tstop = general_settings["tstart"] + general_settings["tdur"]
-        h.dt = general_settings["dt"]  # Time step (resolution) of the simulation in ms
+        h.tstop = self.general_settings["tstart"] + self.general_settings["tdur"]
+        h.dt = self.general_settings["dt"]  # Time step (resolution) of the simulation in ms
         h.steps_per_ms = 1 / h.dt
-        h.celsius = general_settings["celsius"]
+        h.celsius = self.general_settings["celsius"]
 
         # get some stuff set up we need for both SingleEvent and Interactive Tuner
-        self._set_up_cell()
+        # Only set up cell if hoc_cell was not provided
+        if self.hoc_cell is None:
+            self._set_up_cell()
+        else:
+            self.cell = self.hoc_cell
         self._set_up_synapse()
 
         self.nstim = h.NetStim()
@@ -137,7 +174,7 @@ class SynapseTuner:
 
         self._set_up_recorders()
 
-    def _update_spec_syn_param(self, json_folder_path):
+    def _update_spec_syn_param(self, json_folder_path: str) -> None:
         """
         Update specific synaptic parameters using JSON files located in the specified folder.
 
@@ -146,6 +183,8 @@ class SynapseTuner:
         json_folder_path : str
             Path to folder containing JSON files, where each JSON file corresponds to a connection type.
         """
+        if not self.conn_type_settings:
+            return
         for conn_type, settings in self.conn_type_settings.items():
             json_file_path = os.path.join(json_folder_path, f"{conn_type}.json")
             if os.path.exists(json_file_path):
@@ -155,13 +194,17 @@ class SynapseTuner:
             else:
                 print(f"JSON file for {conn_type} not found.")
 
-    def _set_up_cell(self):
+    def _set_up_cell(self) -> None:
         """
         Set up the neuron cell based on the specified connection settings.
+        This method is only called when hoc_cell is not provided.
         """
-        self.cell = getattr(h, self.conn["spec_settings"]["post_cell"])()
+        if self.hoc_cell is None:
+            self.cell = getattr(h, self.conn["spec_settings"]["post_cell"])()
+        else:
+            self.cell = self.hoc_cell
 
-    def _set_up_synapse(self):
+    def _set_up_synapse(self) -> None:
         """
         Set up the synapse on the target cell according to the synaptic parameters in `conn_type_settings`.
 
@@ -176,7 +219,7 @@ class SynapseTuner:
             )
         )
         for key, value in self.conn["spec_syn_param"].items():
-            if isinstance(value, (int, float)):  # Only create sliders for numeric values
+            if isinstance(value, (int, float)):
                 if hasattr(self.syn, key):
                     setattr(self.syn, key, value)
                 else:
@@ -184,7 +227,7 @@ class SynapseTuner:
                         f"Warning: {key} cannot be assigned as it does not exist in the synapse. Check your mod file or spec_syn_param."
                     )
 
-    def _set_up_recorders(self):
+    def _set_up_recorders(self) -> None:
         """
         Set up recording vectors to capture simulation data.
 
@@ -952,11 +995,12 @@ class SynapseTuner:
 class GapJunctionTuner:
     def __init__(
         self,
-        mechanisms_dir: str = None,
-        templates_dir: str = None,
-        config: str = None,
-        general_settings: dict = None,
-        conn_type_settings: dict = None,
+        mechanisms_dir: Optional[str] = None,
+        templates_dir: Optional[str] = None,
+        config: Optional[str] = None,
+        general_settings: Optional[dict] = None,
+        conn_type_settings: Optional[dict] = None,
+        hoc_cell: Optional[object] = None,
     ):
         """
         Initialize the GapJunctionTuner class.
@@ -973,34 +1017,49 @@ class GapJunctionTuner:
             General settings dictionary including parameters like simulation time step, duration, and temperature.
         conn_type_settings : dict
             A dictionary containing connection-specific settings for gap junctions.
+        hoc_cell : object, optional
+            An already loaded NEURON cell object. If provided, template loading and cell creation will be skipped.
         """
-        if config is None and (mechanisms_dir is None or templates_dir is None):
-            raise ValueError(
-                "Either a config file or both mechanisms_dir and templates_dir must be provided."
-            )
+        self.hoc_cell = hoc_cell
 
-        if config is None:
-            neuron.load_mechanisms(mechanisms_dir)
-            h.load_file(templates_dir)
-        else:
-            # this will load both mechs and templates
-            load_templates_from_config(config)
+        if hoc_cell is None:
+            if config is None and (mechanisms_dir is None or templates_dir is None):
+                raise ValueError(
+                    "Either a config file, both mechanisms_dir and templates_dir, or a hoc_cell must be provided."
+                )
+
+            if config is None:
+                neuron.load_mechanisms(mechanisms_dir)
+                h.load_file(templates_dir)
+            else:
+                # this will load both mechs and templates
+                load_templates_from_config(config)
 
-        self.general_settings = general_settings
+        # Use default general settings if not provided, merge with user-provided
+        if general_settings is None:
+            self.general_settings: dict = DEFAULT_GAP_JUNCTION_GENERAL_SETTINGS.copy()
+        else:
+            self.general_settings = {**DEFAULT_GAP_JUNCTION_GENERAL_SETTINGS, **general_settings}
         self.conn_type_settings = conn_type_settings
 
-        h.tstop = general_settings["tstart"] + general_settings["tdur"] + 100.0
-        h.dt = general_settings["dt"]  # Time step (resolution) of the simulation in ms
+        h.tstop = self.general_settings["tstart"] + self.general_settings["tdur"] + 100.0
+        h.dt = self.general_settings["dt"]  # Time step (resolution) of the simulation in ms
         h.steps_per_ms = 1 / h.dt
-        h.celsius = general_settings["celsius"]
-
-        self.cell_name = conn_type_settings["cell"]
+        h.celsius = self.general_settings["celsius"]
 
         # set up gap junctions
         pc = h.ParallelContext()
 
-        self.cell1 = getattr(h, self.cell_name)()
-        self.cell2 = getattr(h, self.cell_name)()
+        # Use provided hoc_cell or create new cells
+        if self.hoc_cell is not None:
+            self.cell1 = self.hoc_cell
+            # For gap junctions, we need two cells, so create a second one if using hoc_cell
+            self.cell_name = conn_type_settings["cell"]
+            self.cell2 = getattr(h, self.cell_name)()
+        else:
+            self.cell_name = conn_type_settings["cell"]
+            self.cell1 = getattr(h, self.cell_name)()
+            self.cell2 = getattr(h, self.cell_name)()
 
         self.icl = h.IClamp(self.cell1.soma[0](0.5))
         self.icl.delay = self.general_settings["tstart"]
@@ -1247,6 +1306,10 @@ class SynapseOptimizer:
             - max_amplitude: maximum synaptic response amplitude
             - rise_time: time for synaptic response to rise from 20% to 80% of peak
             - decay_time: time constant of synaptic response decay
+            - latency: synaptic response latency
+            - half_width: synaptic response half-width
+            - baseline: baseline current
+            - amp: peak amplitude from syn_props
         """
         # Set these to 0 for when we return the dict
         induction = 0
@@ -1255,11 +1318,22 @@ class SynapseOptimizer:
         amp = 0
         rise_time = 0
         decay_time = 0
+        latency = 0
+        half_width = 0
+        baseline = 0
+        syn_amp = 0
 
         if self.run_single_event:
             self.tuner.SingleEvent(plot_and_print=False)
-            rise_time = self.tuner.rise_time
-            decay_time = self.tuner.decay_time
+            # Use the attributes set by SingleEvent method
+            rise_time = getattr(self.tuner, "rise_time", 0)
+            decay_time = getattr(self.tuner, "decay_time", 0)
+            # Get additional syn_props directly
+            syn_props = self.tuner._get_syn_prop()
+            latency = syn_props.get("latency", 0)
+            half_width = syn_props.get("half_width", 0)
+            baseline = syn_props.get("baseline", 0)
+            syn_amp = syn_props.get("amp", 0)
 
         if self.run_train_input:
             self.tuner._simulate_model(self.train_frequency, self.train_delay)
@@ -1276,6 +1350,10 @@ class SynapseOptimizer:
             "max_amplitude": float(amp),
             "rise_time": float(rise_time),
             "decay_time": float(decay_time),
+            "latency": float(latency),
+            "half_width": float(half_width),
+            "baseline": float(baseline),
+            "amp": float(syn_amp),
         }
 
     def _default_cost_function(

bmtool/util/util.py

@@ -16,6 +16,9 @@ import numpy as np
 import pandas as pd
 from neuron import h
 
+from typing import Dict, List, Optional, Union, Any
+from pathlib import Path
+
 # from bmtk.utils.io.cell_vars import CellVarsFile
 # from bmtk.analyzer.cell_vars import _get_cell_report
 # from bmtk.analyzer.io_tools import load_config
@@ -345,95 +348,366 @@ def load_edges(edges_file, edge_types_file):
     return edges  # return (population, edges_df)
 
 
-def load_edges_from_paths(edge_paths):  # network_dir='network'):
+def load_edges_from_paths(
+    file_paths: str, 
+    verbose: bool = False
+) -> Dict[str, pd.DataFrame]:
     """
-    Returns: A dictionary of connections with filenames (minus _edges.h5) as keys
-
-    edge_paths must be in the format in a circuit config file:
-        [
-            {
-            "edges_file":"filepath", (csv)
-            "edge_types_file":"filepath" (h5)
-            },...
-        ]
-    util.load_edges_from_paths([{"edges_file":"network/hippocampus_hippocampus_edges.h5","edge_types_file":"network/hippocampus_hippocampus_edge_types.csv"}])
+    Load multiple SONATA edge files into a dictionary of DataFrames.
+    
+    This function reads a configuration file containing multiple edge file pairs
+    (CSV + H5) and loads each pair into a separate DataFrame. Each DataFrame
+    contains merged edge connectivity and metadata information.
+    
+    Parameters:
+    -----------
+    file_paths str
+        Expected structure:  [{'edge_types_file': ..., 'edges_file': ...}, ...]
+    verbose : bool, default False
+        If True, print detailed information about the loading process
+        
+    Returns:
+    --------
+    Dict[str, pd.DataFrame]
+        Dictionary mapping edge dataset names to DataFrames.
+        Keys are derived from CSV filenames (without '_edge_types.csv' suffix).
+        Values are DataFrames containing merged edge data.
+        
+    Notes:
+    ------
+    - If loading fails for any dataset, an empty DataFrame is stored for that key
+    - Dataset names are extracted from CSV filenames for readability
+    - All edge data follows SONATA format specifications
     """
-    import h5py
-    import pandas as pd
-    # edges_regex = "_edges.h5"
-    # edge_types_regex = "_edge_types.csv"
-
-    # edges_h5_fpaths = glob.glob(os.path.join(network_dir,'*'+edges_regex))
-    # edge_types_fpaths = glob.glob(os.path.join(network_dir,'*'+edge_types_regex))
-
-    # connections = [re.findall('^[A-Za-z0-9]+_[A-Za-z0-9][^_]+', os.path.basename(n))[0] for n in edges_h5_fpaths]
-    edges_dict = {}
-
-    def get_edge_table(edges_file, edge_types_file, population=None):
-        # dataframe where each row is an edge type
-        cm_df = pd.read_csv(edge_types_file, sep=" ")
-        cm_df.set_index("edge_type_id", inplace=True)
-
-        with h5py.File(edges_file, "r") as connections_h5:
-            if population is None:
-                if len(connections_h5["/edges"]) > 1:
-                    raise Exception(
-                        "Multiple populations in edges file. Not currently implemented, should not be hard to do, contact Tyler"
-                    )
-                else:
-                    population = list(connections_h5["/edges"])[0]
-            conn_grp = connections_h5["/edges"][population]
-
-            # dataframe where each row is an edge
-            c_df = pd.DataFrame(
-                {key: conn_grp[key] for key in ("edge_type_id", "source_node_id", "target_node_id")}
-            )
+    
+    # Load configuration and extract edge file information
+   
+    edge_files_list = file_paths
+    
+    if verbose:
+        print(f"Loading {len(edge_files_list)} edge datasets from config: {config}")
+    
+    edges_dict: Dict[str, pd.DataFrame] = {}
+    
+    # Process each edge file pair
+    for i, file_info in enumerate(edge_files_list):
+        csv_path = file_info['edge_types_file']
+        h5_path = file_info['edges_file']
+        
+        # Generate meaningful key from h5 file
+        with h5py.File(h5_path, "r") as connections_h5:
+            key = list(connections_h5["/edges"])[0]
+            
+        if verbose:
+            print(f"\n{'='*60}")
+            print(f"Loading edge set {i+1}/{len(edge_files_list)}: {key}")
+            print(f"{'='*60}")
+        
+        try:
+            # Load individual edge dataset
+            df = load_sonata_edges_to_dataframe(csv_path, h5_path, verbose=verbose)
+            edges_dict[key] = df
+            
+            if verbose:
+                print(f"\nSuccessfully loaded {key}: {df.shape[0]} edges, {df.shape[1]} columns")
+                
+                # Show important columns for verification
+                important_cols = ['edge_type_id', 'source_node_id', 'target_node_id', 
+                                'model_template', 'afferent_section_id']
+                available_important = [col for col in important_cols if col in df.columns]
+                print(f"Key columns available: {available_important}")
+                
+                # Show model template information if available
+                if 'model_template' in df.columns:
+                    unique_templates = df['model_template'].dropna().unique()
+                    print(f"Model templates: {unique_templates}")
+                    
+        except Exception as e:
+            if verbose:
+                print(f"Error loading {key}: {str(e)}")
+            edges_dict[key] = pd.DataFrame()  # Store empty DataFrame on error
+    
+    if verbose:
+        print(f"\n{'='*60}")
+        print("LOADING SUMMARY")
+        print(f"{'='*60}")
+        for key, df in edges_dict.items():
+            if not df.empty:
+                print(f"{key}: {df.shape[0]} edges, {df.shape[1]} columns")
+                if 'model_template' in df.columns:
+                    templates = df['model_template'].dropna().unique()
+                    print(f"  Model templates: {list(templates)}")
+                if 'afferent_section_id' in df.columns:
+                    print(f"  afferent_section_id available")
+            else:
+                print(f"{key}: No data loaded (error occurred)")
+    
+    return edges_dict
 
-            c_df.reset_index(inplace=True)
-            c_df.rename(columns={"index": "edge_id"}, inplace=True)
-            c_df.set_index("edge_type_id", inplace=True)
 
-            # add edge type properties to df of edges
-            edges_df = pd.merge(
-                left=c_df, right=cm_df, how="left", left_index=True, right_index=True
+def load_sonata_edges_to_dataframe(
+    csv_path: Union[str, Path], 
+    h5_path: Union[str, Path], 
+    verbose: bool = False
+) -> pd.DataFrame:
+    """
+    Load SONATA format edge data into a pandas DataFrame.
+    
+    This function combines edge connectivity data from HDF5 files with edge type 
+    metadata from CSV files according to the SONATA specification. H5 attributes 
+    take precedence over CSV attributes when both exist for the same column.
+    
+    Parameters:
+    -----------
+    csv_path : Union[str, Path]
+        Path to the edge types CSV file (space-delimited)
+    h5_path : Union[str, Path]
+        Path to the edges HDF5 file containing connectivity data
+    verbose : bool, default False
+        If True, print detailed information about the loading process
+        
+    Returns:
+    --------
+    pd.DataFrame
+        Combined edge data with columns from both H5 and CSV sources.
+        Columns include:
+        - edge_id: Sequential edge identifier
+        - population: Edge population name
+        - edge_type_id: Edge type identifier for merging with CSV
+        - source_node_id, target_node_id: Connected node IDs
+        - source_population, target_population: Node population names
+        - edge_group_id, edge_group_index: Edge grouping information
+        - Additional attributes from H5 edge groups
+        - Edge type metadata from CSV (model_template, afferent_section_id, etc.)
+        
+    Notes:
+    ------
+    - CSV file must be space-delimited and contain 'edge_type_id' column
+    - H5 file must follow SONATA edge format with required datasets
+    - When both H5 and CSV contain the same attribute, H5 version is kept
+    - Dynamics parameters are flattened with 'dynamics_params/' prefix
+    """
+    
+    # Load edge types CSV (space-delimited)
+    edge_types_df = pd.read_csv(csv_path, sep=' ')
+    
+    if verbose:
+        print(f"Loaded edge types from: {csv_path}")
+        print(f"  Columns: {edge_types_df.columns.tolist()}")
+        print(f"  Shape: {edge_types_df.shape}")
+        print(f"  Unique edge_type_ids: {sorted(edge_types_df['edge_type_id'].unique())}")
+    
+    # Open HDF5 file and process edge data
+    with h5py.File(h5_path, 'r') as h5f:
+        
+        # Navigate to edges group
+        edges_group = h5f['edges']
+        populations = list(edges_group.keys())
+        
+        if verbose:
+            print(f"\nProcessing H5 file: {h5_path}")
+            print(f"  Edge populations: {populations}")
+        
+        # Process each population (typically one per file)
+        all_edges_data: List[pd.DataFrame] = []
+        
+        for pop_name in populations:
+            pop_group = edges_group[pop_name]
+            
+            if verbose:
+                print(f"\n  Processing population: {pop_name}")
+            
+            # Read required SONATA edge datasets
+            edge_type_ids = pop_group['edge_type_id'][:]
+            source_node_ids = pop_group['source_node_id'][:]
+            target_node_ids = pop_group['target_node_id'][:]
+            edge_group_ids = pop_group['edge_group_id'][:]
+            edge_group_indices = pop_group['edge_group_index'][:]
+            
+            if verbose:
+                print(f"    Unique edge_type_ids in H5: {sorted(np.unique(edge_type_ids))}")
+                print(f"    Number of edges: {len(edge_type_ids)}")
+            
+            # Extract node population attributes with proper string handling
+            source_pop = pop_group['source_node_id'].attrs.get('node_population', '')
+            target_pop = pop_group['target_node_id'].attrs.get('node_population', '')
+            
+            # Handle both string and bytes cases for population names
+            if isinstance(source_pop, bytes):
+                source_pop = source_pop.decode()
+            if isinstance(target_pop, bytes):
+                target_pop = target_pop.decode()
+            
+            n_edges = len(edge_type_ids)
+            
+            # Create base DataFrame with core edge attributes
+            edges_df = pd.DataFrame({
+                'edge_id': np.arange(n_edges),  # Sequential edge identifiers
+                'population': pop_name,
+                'edge_type_id': edge_type_ids,
+                'source_node_id': source_node_ids,
+                'target_node_id': target_node_ids,
+                'source_population': source_pop,
+                'target_population': target_pop,
+                'edge_group_id': edge_group_ids,
+                'edge_group_index': edge_group_indices
+            })
+            
+            # Process edge groups to extract additional attributes
+            edge_group_names = [
+                key for key in pop_group.keys() 
+                if key not in ['edge_type_id', 'source_node_id', 'target_node_id', 
+                              'edge_group_id', 'edge_group_index', 'indices']
+            ]
+            
+            # Track which columns come from H5 for precedence handling
+            edge_attributes: Dict[str, np.ndarray] = {}
+            h5_column_names: set = set()
+            
+            # Process each edge group to extract group-specific attributes
+            for group_name in edge_group_names:
+                group = pop_group[group_name]
+                group_id = int(group_name)
+                
+                # Identify edges belonging to this group
+                group_mask = edge_group_ids == group_id
+                group_indices = edge_group_indices[group_mask]
+                
+                # Extract all attributes from this edge group
+                for attr_name in group.keys():
+                    if attr_name == 'dynamics_params': # sonata says this exist but i have yet to see it
+                        # Handle nested dynamics parameters
+                        dynamics_group = group[attr_name]
+                        for param_name in dynamics_group.keys():
+                            param_data = dynamics_group[param_name][:]
+                            full_attr_name = f"dynamics_params/{param_name}"
+                            
+                            # Initialize attribute array if not exists
+                            if full_attr_name not in edge_attributes:
+                                edge_attributes[full_attr_name] = np.full(n_edges, np.nan, dtype=object)
+                            
+                            # Assign data to appropriate edges
+                            edge_attributes[full_attr_name][group_mask] = param_data[group_indices]
+                            h5_column_names.add(full_attr_name)
+                    else:
+                        # Handle regular attributes with proper string decoding
+                        attr_data = group[attr_name][:]
+                        
+                        # Handle string attributes properly
+                        if attr_data.dtype.kind in ['S', 'U']:  # String or Unicode
+                            # Decode bytes to strings if necessary
+                            if attr_data.dtype.kind == 'S':
+                                attr_data = np.array([
+                                    s.decode() if isinstance(s, bytes) else s 
+                                    for s in attr_data
+                                ])
+                        
+                        # Initialize attribute array with appropriate default
+                        if attr_name not in edge_attributes:
+                            if attr_data.dtype.kind in ['S', 'U']:
+                                edge_attributes[attr_name] = np.full(n_edges, '', dtype=object)
+                            else:
+                                edge_attributes[attr_name] = np.full(n_edges, np.nan, dtype=object)
+                        
+                        # Assign data to appropriate edges
+                        edge_attributes[attr_name][group_mask] = attr_data[group_indices]
+                        h5_column_names.add(attr_name)
+            
+            # Add all H5 attributes to the DataFrame
+            for attr_name, attr_values in edge_attributes.items():
+                edges_df[attr_name] = attr_values
+            
+            all_edges_data.append(edges_df)
+    
+    # Combine all populations into single DataFrame
+    if all_edges_data:
+        combined_edges_df = pd.concat(all_edges_data, ignore_index=True)
+    else:
+        combined_edges_df = pd.DataFrame()
+    
+    if verbose:
+        print(f"\n  H5 columns: {sorted(h5_column_names)}")
+        print(f"  CSV columns: {edge_types_df.columns.tolist()}")
+        print(f"  Combined edges before merge: {combined_edges_df.shape}")
+    
+    # Merge H5 data with CSV edge type metadata
+    if not combined_edges_df.empty and not edge_types_df.empty:
+        # Determine merge columns (typically just edge_type_id)
+        merge_cols = []
+        if 'edge_type_id' in combined_edges_df.columns and 'edge_type_id' in edge_types_df.columns:
+            merge_cols.append('edge_type_id')
+        
+        if 'population' in edge_types_df.columns and 'population' in combined_edges_df.columns:
+            merge_cols.append('population')
+        
+        if merge_cols:
+            if verbose:
+                # Debug merge compatibility
+                h5_edge_types = set(combined_edges_df['edge_type_id'].unique())
+                csv_edge_types = set(edge_types_df['edge_type_id'].unique())
+                overlap = h5_edge_types.intersection(csv_edge_types)
+                
+                print(f"\n  Merging on columns: {merge_cols}")
+                print(f"  Edge types in H5: {sorted(h5_edge_types)}")
+                print(f"  Edge types in CSV: {sorted(csv_edge_types)}")
+                print(f"  Overlap: {sorted(overlap)}")
+            
+            # Perform left join to preserve all H5 edges
+            final_df = combined_edges_df.merge(
+                edge_types_df, 
+                on=merge_cols, 
+                how='left', 
+                suffixes=('', '_csv')
             )
-
-            # extra properties of individual edges (see SONATA Data format)
-            if conn_grp.get("0"):
-                edge_group_id = conn_grp["edge_group_id"][()]
-                edge_group_index = conn_grp["edge_group_index"][()]
-                n_group = edge_group_id.max() + 1
-                prop_dtype = {}
-                for group_id in range(n_group):
-                    group = conn_grp[str(group_id)]
-                    idx = edge_group_id == group_id
-                    for prop in group:
-                        # create new column with NaN if property does not exist
-                        if prop not in edges_df:
-                            edges_df[prop] = np.nan
-                        edges_df.loc[idx, prop] = tuple(group[prop][edge_group_index[idx]])
-                        prop_dtype[prop] = group[prop].dtype
-                # convert to original data type if possible
-                for prop, dtype in prop_dtype.items():
-                    edges_df[prop] = edges_df[prop].astype(dtype, errors="ignore")
-
-        return population, edges_df
-
-    # for edges_dict, conn_models_file, conns_file in zip(connections, edge_types_fpaths, edges_h5_fpaths):
-    #    connections_dict[connection] = get_connection_table(conn_models_file,conns_file)
-    try:
-        for nodes in edge_paths:
-            edges_file = nodes["edges_file"]
-            edge_types_file = nodes["edge_types_file"]
-            region_name, region = get_edge_table(edges_file, edge_types_file)
-            edges_dict[region_name] = region
-    except Exception as e:
-        print(repr(e))
-        print("Hint: Are you loading the correct simulation config file?")
-        print("Command Line: bmtool plot --config yourconfig.json <rest of command>")
-        print("Python: bmplot.connection_matrix(config='yourconfig.json')")
-
-    return edges_dict
+            
+            if verbose:
+                print(f"  After merge: {final_df.shape}")
+            
+            # Apply H5 precedence rule: H5 attributes override CSV attributes
+            for col in edge_types_df.columns:
+                if col in merge_cols:
+                    continue  # Skip merge columns
+                    
+                csv_col_name = f"{col}_csv" if f"{col}_csv" in final_df.columns else col
+                
+                # If column exists in H5, handle row-by-row precedence
+                if col in h5_column_names:
+                    if csv_col_name in final_df.columns and csv_col_name != col:
+                        # Row-by-row logic: use CSV value only where H5 is NaN
+                        mask_h5_nan = final_df[col].isna()
+                        final_df.loc[mask_h5_nan, col] = final_df.loc[mask_h5_nan, csv_col_name]
+                        
+                        if verbose and mask_h5_nan.any():
+                            n_filled = mask_h5_nan.sum()
+                            print(f"    Column '{col}': filled {n_filled} NaN values from CSV")
+                        
+                        # Remove CSV version after filling NaN values
+                        final_df = final_df.drop(columns=[csv_col_name])
+                    elif verbose:
+                        print(f"    Column '{col}' exists in H5 only")
+                # If column only in CSV, rename if needed
+                elif csv_col_name in final_df.columns and csv_col_name != col:
+                    final_df = final_df.rename(columns={csv_col_name: col})
+                    if verbose:
+                        print(f"    Added column '{col}' from CSV")
+        else:
+            if verbose:
+                print("  No common merge columns found. Using H5 data only.")
+            final_df = combined_edges_df
+    else:
+        final_df = combined_edges_df
+    
+    if verbose:
+        print(f"\nFinal DataFrame: {final_df.shape}")
+        print(f"Final columns: {final_df.columns.tolist()}")
+    
+    # Set edge_type_id as the index to match old function
+    if 'edge_type_id' in final_df.columns:
+        final_df = final_df.set_index('edge_type_id')
+        if verbose:
+            print(f"Set edge_type_id as index. Final shape: {final_df.shape}")
+    
+    return final_df
 
 
 def load_mechanisms_from_config(config=None):
@@ -604,7 +878,6 @@ def relation_matrix(
             if relation_func:
                 source_nodes = nodes[source].add_prefix("source_")
                 target_nodes = nodes[target].add_prefix("target_")
-
                 c_edges = pd.merge(
                     left=edges[e_name],
                     right=source_nodes,
@@ -856,7 +1129,8 @@ def percent_connections(
         cons = edges[(edges[source_id_type] == source_id) & (edges[target_id_type] == target_id)]
         if not include_gap:
             try:
-                cons = cons[~cons["is_gap_junction"]]
+                gaps = cons["is_gap_junction"]==True
+                cons = cons[~gaps]
             except:
                 raise Exception("no gap junctions found to drop from connections")
 
@@ -1003,7 +1277,7 @@ def gap_junction_connections(
         # print(cons)
 
         try:
-            cons = cons[cons["is_gap_junction"]]
+            cons = cons[cons["is_gap_junction"]==True]
         except:
             raise Exception("no gap junctions found to drop from connections")
         mean = cons["target_node_id"].value_counts().mean()

{bmtool-0.7.2.1.dist-info → bmtool-0.7.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bmtool
-Version: 0.7.2.1
+Version: 0.7.4
 Summary: BMTool
 Home-page: https://github.com/cyneuro/bmtool
 Download-URL: 
@@ -36,6 +36,7 @@ Requires-Dist: requests
 Requires-Dist: pyyaml
 Requires-Dist: PyWavelets
 Requires-Dist: numba
+Requires-Dist: tqdm
 Provides-Extra: dev
 Requires-Dist: ruff>=0.1.0; extra == "dev"
 Requires-Dist: pytest>=7.0.0; extra == "dev"

{bmtool-0.7.2.1.dist-info → bmtool-0.7.4.dist-info}/RECORD

@@ -6,16 +6,16 @@ bmtool/graphs.py,sha256=gBTzI6c2BBK49dWGcfWh9c56TAooyn-KaiEy0Im1HcI,6717
 bmtool/manage.py,sha256=lsgRejp02P-x6QpA7SXcyXdalPhRmypoviIA2uAitQs,608
 bmtool/plot_commands.py,sha256=Dxm_RaT4CtHnfsltTtUopJ4KVbfhxtktEB_b7bFEXII,12716
 bmtool/singlecell.py,sha256=I2yolbAnNC8qpnRkNdnDCLidNW7CktmBuRrcowMZJ3A,45041
-bmtool/synapses.py,sha256=y8UJAqO1jpZY-mY9gVVMN8Dj1r9jD2fI1nAaNQeQfz4,66148
+bmtool/synapses.py,sha256=-kg_TJoqXStIgE5iHpJWpXU6VRKT0YyIVpTw8frNbxA,69653
 bmtool/analysis/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 bmtool/analysis/entrainment.py,sha256=NQloQtVpEWjDzmkZwMWVcm3hSjErHBZfQl1mrBVoIE8,25321
 bmtool/analysis/lfp.py,sha256=S2JvxkjcK3-EH93wCrhqNSFY6cX7fOq74pz64ibHKrc,26556
 bmtool/analysis/netcon_reports.py,sha256=VnPZNKPaQA7oh1q9cIatsqQudm4cOtzNtbGPXoiDCD0,2909
 bmtool/analysis/spikes.py,sha256=3n-xmyEZ7w6CKEND7-aKOAvdDg0lwDuPI5sMdOuPwa0,24637
 bmtool/bmplot/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-bmtool/bmplot/connections.py,sha256=L0UGOOsD4Iqxv492uSChpsFHIC_Ntz8X51ZfE_AJcP8,58216
+bmtool/bmplot/connections.py,sha256=DwnnMp5cUwZC3OqsJqvgTPr4lBzc0sNWYRPmwij0WRk,59337
 bmtool/bmplot/entrainment.py,sha256=BrBMerqyiG2YWAO_OEFv7OJf3yeFz3l9jUt4NamluLc,32837
-bmtool/bmplot/lfp.py,sha256=SNpbWGOUnYEgnkeBw5S--aPN5mIGD22Gw2Pwus0_lvY,2034
+bmtool/bmplot/lfp.py,sha256=7JLozQQJ19ty0ZNyfhkuJAr_K8_pVP9C0flVJd_YXaY,2027
 bmtool/bmplot/netcon_reports.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 bmtool/bmplot/spikes.py,sha256=odzCSMbFRHp9qthSGQ0WzMWUwNQ7R1Z6gLT6VPF_o5Q,15326
 bmtool/debug/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -23,12 +23,12 @@ bmtool/debug/commands.py,sha256=VV00f6q5gzZI503vUPeG40ABLLen0bw_k4-EX-H5WZE,580
 bmtool/debug/debug.py,sha256=9yUFvA4_Bl-x9s29quIEG3pY-S8hNJF3RKBfRBHCl28,208
 bmtool/util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 bmtool/util/commands.py,sha256=Nn-R-4e9g8ZhSPZvTkr38xeKRPfEMANB9Lugppj82UI,68564
-bmtool/util/util.py,sha256=S8sAXwDiISGAqnSXRIgFqxqCRzL5YcxAqP1UGxGA5Z4,62906
+bmtool/util/util.py,sha256=TAWdGd0tDuouS-JiusMs8WwP7kQpWHPr1nu0XG01TBQ,75056
 bmtool/util/neuron/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 bmtool/util/neuron/celltuner.py,sha256=lokRLUM1rsdSYBYrNbLBBo39j14mm8TBNVNRnSlhHCk,94868
-bmtool-0.7.2.1.dist-info/licenses/LICENSE,sha256=qrXg2jj6kz5d0EnN11hllcQt2fcWVNumx0xNbV05nyM,1068
-bmtool-0.7.2.1.dist-info/METADATA,sha256=rEXYmlS4RZxoXEK_2fV57B1c0CSNA38Eh4JmzQ1YGQ8,3577
-bmtool-0.7.2.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-bmtool-0.7.2.1.dist-info/entry_points.txt,sha256=0-BHZ6nUnh0twWw9SXNTiRmKjDnb1VO2DfG_-oprhAc,45
-bmtool-0.7.2.1.dist-info/top_level.txt,sha256=gpd2Sj-L9tWbuJEd5E8C8S8XkNm5yUE76klUYcM-eWM,7
-bmtool-0.7.2.1.dist-info/RECORD,,
+bmtool-0.7.4.dist-info/licenses/LICENSE,sha256=qrXg2jj6kz5d0EnN11hllcQt2fcWVNumx0xNbV05nyM,1068
+bmtool-0.7.4.dist-info/METADATA,sha256=126Cn17YBt6twoBgMXYXK10kOWUXRhnLnGmPr7z7k_4,3595
+bmtool-0.7.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+bmtool-0.7.4.dist-info/entry_points.txt,sha256=0-BHZ6nUnh0twWw9SXNTiRmKjDnb1VO2DfG_-oprhAc,45
+bmtool-0.7.4.dist-info/top_level.txt,sha256=gpd2Sj-L9tWbuJEd5E8C8S8XkNm5yUE76klUYcM-eWM,7
+bmtool-0.7.4.dist-info/RECORD,,