aspect-stable 0.1.0__py3-none-any.whl

aspect/__init__.py

@@ -0,0 +1,12 @@
+import logging
+from .io import cfg, load_cfg, load_model
+from .workflow import SpectrumDetector, model, CHOICE_DM, TIME_DM
+from .plots import decision_matrix_plot
+from .trainer import components_trainer
+
+# Creating the lime logger
+_logger = logging.getLogger("aspect")
+_logger.setLevel(logging.INFO)
+
+# Invert the dictionary of categories number
+cfg['number_shape'] =  {v: k for k, v in cfg['shape_number'].items()}

aspect/aspect.toml

@@ -0,0 +1,113 @@
+[shape_number]
+undefined = 0
+white-noise = 1
+continuum = 2
+emission = 3
+cosmic-ray = 4
+pixel-line = 5
+broad = 6
+doublet = 7
+peak = 8
+absorption = 9
+dead-pixel = 10
+
+[colors]
+undefined = 'black'
+white-noise = '#C41E3A'     # Red
+continuum = '#F48CBA'       # Pink
+emission = '#00FF98'        # Spring Green
+cosmic-ray= '#FFF468'       # Yellow
+pixel-line = '#0070DD'      # Blue
+broad = '#A330C9'           # Dark magenta
+doublet = '#3FC7EB'         # Light blue
+peak = '#C69B6D'            # Tan
+absorption = '#FF7C0A'      # Orange
+dead-pixel = '#8788EE'      # Purple
+
+[decision_matrices]
+
+#                   Orange      Blue
+decision_colors = ['#be6530', '#72a7c2']
+
+choice_labels = ['1st option', '2nd option']
+choice =   [[2,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1], #undefined
+            [0,  2,  1,  0,  0,  0,  0,  0,  0,  0,  0], #white-noise
+            [0,  0,  2,  1,  1,  1,  1,  1,  1,  1,  1], #continuum
+            [0,  1,  0,  2,  0,  0,  0,  0,  0,  0,  0], #emission
+            [0,  1,  0,  1,  2,  0,  0,  0,  0,  0,  0], #cosmic-ray
+            [0,  1,  0,  1,  0,  2,  0,  0,  0,  0,  0], #pixel-line
+            [0,  1,  0,  1,  0,  0,  2,  0,  0,  0,  0], #broad
+            [0,  1,  0,  1,  0,  0,  0,  2,  0,  0,  0], #doublet
+            [0,  1,  0,  1,  0,  0,  0,  0,  2,  0,  0], #peak
+            [0,  1,  0,  0,  0,  0,  0,  0,  0,  2,  0], #absorption
+            [0,  1,  0,  0,  0,  0,  0,  0,  0,  0,  2]] #dead-pixel
+#            u   w   c   e   c   p   b   d   p   a   d
+#            n   h   o   m   o   i   r   o   e   b   e
+#            d   i   n   i   s   x   o   u   a   s   a
+#            e   t   t   s   m   e   a   b   k   o   d
+#            f   e   i   s   i   l   d   l       r   -
+#            i   -   n   i   c   -       e       p   p
+#            n   n   u   o   -   l       t       t   i
+#            e   o   u   n   r   i               i   x
+#            d   i   m       a   n               o   e
+#                s           y   e               n   l
+#                e
+
+time_labels = ['Current detection', 'Past detection']
+time =     [[0,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1], #undefined
+            [0,  0,  1,  1,  1,  1,  1,  1,  1,  1,  1], #white-noise
+            [0,  0,  0,  1,  1,  1,  1,  1,  1,  1,  1], #continuum
+            [0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0], #emission
+            [0,  0,  0,  1,  0,  0,  0,  1,  0,  0,  0], #cosmic-ray
+            [0,  0,  0,  1,  0,  0,  0,  1,  0,  0,  0], #pixel-line
+            [0,  0,  0,  1,  0,  0,  0,  1,  0,  0,  0], #broad
+            [0,  0,  0,  1,  0,  0,  0,  0,  0,  0,  0], #doublet
+            [0,  0,  0,  1,  1,  1,  1,  1,  0,  0,  0], #peak
+            [0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0], #absorption
+            [0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0]] #dead-pixel
+#            u   w   c   e   c   p   b   d   p   a   d
+#            n   h   o   m   o   i   r   o   e   b   e
+#            d   i   n   i   s   x   o   u   a   s   a
+#            e   t   t   s   m   e   a   b   k   o   d
+#            f   e   i   s   i   l   d   l       r   -
+#            i   -   n   i   c   -       e       p   p
+#            n   n   u   o   -   l       t       t   i
+#            e   o   u   n   r   i               i   x
+#            d   i   m       a   n               o   e
+#                s           y   e               n   l
+#                e
+
+
+
+#choice =   [[0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0], #undefined
+#            [0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0], #white-noise
+#            [0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0], #continuum
+#            [0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0], #emission
+#            [0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0], #cosmic-ray
+#            [0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0], #pixel-line
+#            [0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0], #broad
+#            [0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0], #doublet
+#            [0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0], #peak
+#            [0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0], #absorption
+#            [0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0]] #dead-pixel
+##            u   w   c   e   c   p   b   d   p   a   d
+##            n   h   o   m   o   i   r   o   e   b   e
+##            d   i   n   i   s   x   o   u   a   s   a
+##            e   t   t   s   m   e   a   b   k   o   d
+##            f   e   i   s   i   l   d   l       r   -
+##            i   -   n   i   c   -       e       p   p
+##            n   n   u   o   -   l       t       t   i
+##            e   o   u   n   r   i               i   x
+##            d   i   m       a   n               o   e
+##                s           y   e               n   l
+##                e
+
+[plots_format]
+dm."figure.dpi" = 300
+dm."figure.figsize" = [5, 5]
+dm."axes.titlesize" = 9
+dm."axes.labelsize" = 14
+dm."legend.fontsize" = 7
+dm."xtick.labelsize" = 12
+dm."ytick.labelsize" = 12
+dm."font.size" = 5

aspect/io.py

@@ -0,0 +1,97 @@
+from joblib import load as jload
+import numpy as np
+
+from pathlib import Path
+
+try:
+    import tomllib
+except ModuleNotFoundError:
+    import tomli as tomllib
+
+# Specify the files location
+_ASPECT_FOLDER = Path(__file__).parent
+_MODEL_FOLDER = _ASPECT_FOLDER/'models'
+
+# Configuration file
+_CONF_FILE = _ASPECT_FOLDER/'aspect.toml'
+
+class Aspect_Error(Exception):
+    """Aspect exception function"""
+
+# Read lime configuration file
+with open(_CONF_FILE, mode="rb") as fp:
+    cfg = tomllib.load(fp)
+
+# Default feature detection model
+# DEFAULT_MODEL_ADDRESS = _MODEL_FOLDER/'training_multi_sample_v4_min-max_8categories_v4_175000points_angleSample_numpy_array_model.joblib'
+# DEFAULT_MODEL_ADDRESS = _MODEL_FOLDER/'aspect_min-max_mediumbox_v1_model.joblib'
+DEFAULT_MODEL_ADDRESS = _MODEL_FOLDER/'aspect_min-max_mediumbox_v3_model.joblib'
+
+
+def read_trained_model(file_address):
+
+    # Read trained model
+    model = jload(file_address)
+
+    # Read lime configuration file
+    cfg_address = Path(file_address).parent/f'{file_address.stem}.toml'
+    with open(cfg_address, mode="rb") as cm:
+        cfg_model = tomllib.load(cm)
+
+    return model, cfg_model
+
+
+def check_lisa(model1D, model2D, setup_cfg):
+
+    if model1D is None:
+        coeffs1D = np.array(setup_cfg['linear']['model1D_coeffs']), np.array(setup_cfg['linear']['model1D_intercept'])
+    else:
+        model1D_job = jload(model1D)
+        coeffs1D = np.squeeze(model1D_job.coef_), np.squeeze(model1D_job.intercept_)
+
+    if model2D is None:
+        coeffs2D = np.array(setup_cfg['linear']['model2D_coeffs']), np.array(setup_cfg['linear']['model2D_intercept'])
+    else:
+        model2D_job = jload(model2D)
+        coeffs2D = np.squeeze(model2D_job.coef_), np.squeeze(model2D_job.intercept_)
+
+    return coeffs1D, coeffs2D
+
+
+# Function to load configuration file
+def load_cfg(file_address, fit_cfg_suffix='_line_fitting'):
+
+    """
+
+    This function reads a configuration file with the `toml format <https://toml.io/en/>`_.
+
+    :param file_address: Input configuration file address.
+    :type file_address: str, pathlib.Path
+
+    :param fit_cfg_suffix: Suffix for LiMe configuration sections. The default value is "_line_fitting".
+    :type fit_cfg_suffix:  str
+
+    :return: Parsed configuration data
+    :type: dict
+
+    """
+
+    file_path = Path(file_address)
+
+    # Open the file
+    if file_path.is_file():
+
+        # Toml file
+        with open(file_path, mode="rb") as fp:
+            cfg = tomllib.load(fp)
+
+    else:
+        raise Aspect_Error(f'The configuration file was not found at: {file_address}')
+
+    return cfg
+
+def load_model(file_address):
+
+    ml_function = jload(file_address)
+
+    return ml_function

aspect/models/aspect_min-max_mediumbox_v3_model.toml

@@ -0,0 +1,27 @@
+[resuts]
+f1 = "0.9093169891178652"
+precision = "0.9128093921636236"
+Recall = "0.9096525248222435"
+confusion_matrix = [ [ "0.13127522178049186", "0.008862660852373147", "0.0001668970416429516", "0.0009157940233741447", "0.0005691617061157067", "0.00023964703415398178", "0.0008302057968905798",], [ "0.030512202741390895", "0.10520076858227383", "0.0031239702666501197", "0.003385014357424993", "0.0005477646494948155", "8.558822648356493e-06", "7.702940383520843e-05",], [ "0.0001027058717802779", "0.005986896442525366", "0.12498448713394986", "0.011036601805055696", "0.0007446175704070149", "0.0", "0.0",], [ "0.00024392644547816003", "0.005537558253486651", "0.0003637499625551509", "0.13671435357352243", "0.0", "0.0", "0.0",], [ "0.0007874116836487973", "8.986763780774317e-05", "0.00017973527561548634", "0.0", "0.1417982942266462", "0.0", "0.0",], [ "0.0013865292690337517", "0.0004964117136046765", "0.0", "0.0", "0.0", "0.1296747219452492", "0.01129764589583057",], [ "0.0010698528310445616", "0.0013009410425501868", "0.0", "0.0", "0.0", "0.0004835734796321418", "0.1400052208818155",],]
+fit_time = "0.0"
+
+[properties]
+box_size = 12
+sample_size = 300000
+test_sample_size_fraction = 0.1
+categories = [ "white-noise", "continuum", "cosmic-ray", "emission", "doublet", "dead-pixel", "absorption",]
+scale = "min-max"
+
+[properties.estimator]
+module = "sklearn.ensemble"
+class = "RandomForestClassifier"
+
+[properties.estimator_params]
+random_state = 42
+n_estimators = 60
+max_depth = 8
+max_features = "sqrt"
+verbose = 0
+n_jobs = 15
+min_samples_split = 2000
+min_samples_leaf = 2000

aspect/plots.py

@@ -0,0 +1,274 @@
+import numpy as np
+from matplotlib import pyplot as plt, gridspec, colors, rc_context
+from .io import Aspect_Error, cfg
+from lime.plots import theme
+from .tools import detection_function, stratify_sample
+
+
+
+def decision_matrix_plot(matrix_arr, output_address=None, categories=None, exclude_diagonal=True, show_categories=False):
+
+    # Try to find from database if None provided
+    matrix_name = None
+    if isinstance(matrix_arr, str):
+        if matrix_arr in cfg['decision_matrices']:
+            matrix_name = str(matrix_arr)
+            matrix_arr = np.array(cfg['decision_matrices'][matrix_arr])
+        else:
+            raise Aspect_Error(f'Decision matrix "{matrix_arr}" not found in configuration file please')
+        categories = list(cfg['shape_number'].keys())
+
+    # Number of categories
+    n_categories = matrix_arr.shape[0]
+
+    # Set the diagonal to a distinct value (e.g., -1) to differentiate it
+    if exclude_diagonal:
+        np.fill_diagonal(matrix_arr, -1)
+
+    # Figure format
+    decision_colors = cfg['decision_matrices']['decision_colors']
+    axes_labels = None if matrix_name is None else cfg['decision_matrices'][f'{matrix_name}_labels']
+
+    # Start the figure
+    with rc_context(cfg['plots_format']['dm']):
+
+        # Define colors for values
+        cmap = colors.ListedColormap(['white', decision_colors[0], decision_colors[1]])
+        bounds = [-1.5, -0.5, 0.5, 1.5]
+        norm = colors.BoundaryNorm(bounds, cmap.N)
+
+        # Adjusting the plot by adding gridlines to the subplots
+        fig = plt.figure(figsize=(10, 8))
+        gs = gridspec.GridSpec(n_categories, 2, width_ratios=[n_categories, 1], wspace=0.05)
+
+        # Plot the decision matrix on the left (column 0 of the GridSpec)
+        ax_matrix = fig.add_subplot(gs[:, 0])
+        ax_matrix.matshow(matrix_arr, cmap=cmap, norm=norm)
+
+        # Customize matrix ticks and labels
+        ax_matrix.set_xticks(range(n_categories))
+        ax_matrix.set_yticks(range(n_categories))
+
+        ax_matrix.set_xticklabels(categories, rotation=90)
+        ax_matrix.set_yticklabels(categories)
+
+        # # Move x-axis labels to the bottom
+        # ax_matrix.xaxis.set_ticks_position('bottom')
+        # ax_matrix.xaxis.set_label_position('bottom')
+
+        # Add black gridlines to separate each square
+        ax_matrix.set_xticks(np.arange(-.5, n_categories, 1), minor=True)
+        ax_matrix.set_yticks(np.arange(-.5, n_categories, 1), minor=True)
+        ax_matrix.grid(which="minor", color="black", linestyle='-', linewidth=2)
+        ax_matrix.tick_params(which="minor", size=0)
+
+        if axes_labels is not None:
+            ax_matrix.set_ylabel(axes_labels[0], color=decision_colors[0])
+            ax_matrix.set_xlabel(axes_labels[1], color=decision_colors[1])
+
+        # Add individual plots on the right side (column 1 of the GridSpec) for each category
+        if show_categories:
+
+            for i, category in enumerate(categories):
+
+                ax_plot = fig.add_subplot(gs[i, 1])
+
+                # Example placeholder data for each category plot
+                x = np.linspace(0, 10, 100)
+                y = np.sin(x + i)  # Example sine wave that varies by row
+                ax_plot.step(x, y, color=cfg['colors'][category])
+
+                # Add gridlines to the subplots
+                ax_plot.grid(True, which='both', linestyle='--', linewidth=0.5)
+
+                # Hide y-axis ticks for these small plots for a cleaner look
+                ax_plot.yaxis.set_ticks([])
+                ax_plot.xaxis.set_ticks([])
+
+        # Output the result
+        if output_address is None:
+            plt.show()
+        else:
+            fig.savefig(output_address, bbox_inches='tight')
+
+    return
+
+
+def scatter_plot(fig, ax, x_arr, y_arr, labels_arr, feature_list, color_dict, alpha=0.5, idx_target=None,
+                 detection_range=None, ratio_color=None):
+
+    for feature in feature_list:
+        idcs_class = labels_arr == feature
+        x_feature = x_arr[idcs_class]
+        y_feature = y_arr[idcs_class]
+        label = f'{feature} ({y_feature.size})'
+        color_points = color_dict[feature] if ratio_color is None else ratio_color
+        scatter = ax.scatter(x_feature, y_feature, label=label, c=color_points, alpha=alpha, edgecolor='none')
+
+        # if ratio_color is not None:
+        #     fig.colorbar(scatter, ax=ax)
+
+    if idx_target is not None:
+        ax.scatter(x_arr[idx_target], y_arr[idx_target], marker='x', label='selection', color='black')
+
+    if detection_range is not None:
+        ax.plot(detection_range, detection_function(detection_range))
+
+    return
+
+def parse_fig_cfg(fig_cfg=None, ax_diag=None, ax_line=None, dtype=None):
+
+    # Input configuration updates default
+    fig_cfg = fig_cfg if fig_cfg is not None else {'axes.labelsize': 10, 'axes.titlesize': 10,
+                                                'figure.figsize': (12, 6), 'hatch.linewidth': 0.3, 'legend.fontsize': 8}
+    fig_cfg = theme.fig_defaults(fig_cfg)
+
+    if dtype == 'classifier':
+        ax_diag = {} if ax_diag is None else ax_diag
+        ax_diag = {'xlabel': r'$\frac{\sigma_{gas}}{\Delta\lambda_{inst}} = \sigma_{pixels}$ (Gaussian sigma in pixels)',
+                   'ylabel': r'$\frac{A_{gas}}{\sigma_{noise}}$ (Signal-to-noise)',
+                   **ax_diag}
+
+        ax_line = {} if ax_line is None else ax_line
+        ax_line = {'xlabel': 'Feature Number', 'ylabel': 'value', **ax_line}
+
+    if dtype == 'doublet':
+        ax_diag = {} if ax_diag is None else ax_diag
+        ax_diag = {'xlabel': r'$\frac{\sigma_{gas}}{\Delta\lambda_{inst}} = \sigma_{pixels}$ (Gaussian sigma in pixels)',
+                   'ylabel': r'$S_{pixels}$ (pixels)',
+                   **ax_diag}
+
+        ax_line = {} if ax_line is None else ax_line
+        ax_line = {'xlabel': 'Feature Number', 'ylabel': 'value', **ax_line}
+
+    return {'fig': fig_cfg, 'ax1': ax_diag, 'ax2': ax_line}
+
+
+def ax_wording(ax, ax_cfg=None, legend_cfg=None, yscale=None):
+
+    ax.update(ax_cfg)
+
+    if legend_cfg is not None:
+        ax.legend(**legend_cfg)
+
+    if yscale is not None:
+        ax.set_yscale(yscale)
+
+    return
+
+
+class CheckSample:
+
+    def __init__(self, in_data_arr, in_pred_arr, fig_cfg=None, ax_diag=None, ax_line=None, base=10000, sample_size=None,
+                 categories=None, column_labels='shape_class', dtype='classifier'):
+
+        # Stratify the selection aiming for same number of points
+        data_arr, pred_arr = stratify_sample(in_data_arr, in_pred_arr, sample_size, categories, randomize=True)
+
+        self.y_base = base
+        self.x_coords = data_arr[:, 1]
+        self.y_coords = data_arr[:, 0]
+        self.id_arr = pred_arr
+        self.classes = np.sort(np.unique(self.id_arr))
+        self.data_df = data_arr[:, 2:]
+        self.wave_range = np.arange(self.data_df.shape[1])
+        self.dtype = dtype
+        self.y_scale = 'log' if self.dtype == 'classifier' else 'linear'
+        self.ratio_color = None
+
+        if self.dtype == 'classifier':
+            self.y_coords_log = np.log10(self.y_coords) / np.log10(self.y_base)
+        else:
+            self.y_coords_log = self.y_coords
+
+        if self.dtype == 'doublet':
+            self.ratio_color = self.y_coords/self.x_coords
+
+        self.idx_current = None
+        self.color_dict = cfg['colors']
+
+        self.fig_format = parse_fig_cfg(fig_cfg, ax_diag, ax_line, self.dtype)
+        self._fig, self._ax1, self._ax2 = None, None, None
+
+        self.detection_range = np.linspace(data_arr[:,1].min(), data_arr[:,1].max(), 50) if self.dtype == 'classifier' else None
+
+        return
+
+    def show(self):
+
+        # Generate the figure
+        with rc_context(self.fig_format['fig']):
+
+            # Create the figure
+            self._fig, (self._ax1, self._ax2) = plt.subplots(1, 2)
+
+            # Diagnostic plot
+            scatter_plot(self._fig, self._ax1, self.x_coords, self.y_coords, self.id_arr, self.classes, self.color_dict,
+                         idx_target=self.idx_current, detection_range=self.detection_range, ratio_color=self.ratio_color)
+            ax_wording(self._ax1, self.fig_format['ax1'], legend_cfg={'loc': 'lower center', 'ncol':2, 'framealpha':0.95},
+                       yscale=self.y_scale)
+
+            # Line plot
+            self.index_target()
+            self.line_plot()
+            ax_wording(self._ax2, self.fig_format['ax2'])
+
+            # Interactive widget
+            self._fig.canvas.mpl_connect('button_press_event', self._on_click)
+
+            # Display the plot
+            plt.tight_layout()
+            plt.show()
+
+        return
+
+    def _on_click(self, event):
+
+        if event.inaxes == self._ax1 and event.button == 1:
+
+            if self.y_scale == 'log':
+                user_point = (event.xdata, np.log10(event.ydata) / np.log10(self.y_base))
+            else:
+                user_point = (event.xdata, event.ydata)
+
+
+            # Get index point
+            self.index_target(user_point)
+
+            # Replot the figures
+            self._ax1.clear()
+            scatter_plot(self._fig, self._ax1, self.x_coords, self.y_coords, self.id_arr, self.classes, self.color_dict,
+                         idx_target=self.idx_current, detection_range=self.detection_range, ratio_color=self.ratio_color)
+            ax_wording(self._ax1, self.fig_format['ax1'], legend_cfg={'loc': 'lower center', 'ncol':2, 'framealpha':0.95},
+                       yscale=self.y_scale)
+
+            self._ax2.clear()
+            # self._fig.clear()
+
+            self.line_plot()
+            plt.tight_layout()
+            self._fig.canvas.draw()
+
+        return
+
+    def index_target(self, mouse_coords=None):
+
+        # If no selection use first point
+        if mouse_coords is None:
+            self.idx_current = 0
+            print(f'Reseting location')
+
+        else:
+            distances = np.sqrt((self.x_coords - mouse_coords[0]) ** 2 + (self.y_coords_log - mouse_coords[1]) ** 2)
+            self.idx_current = np.argmin(distances)
+            print('Click on:', mouse_coords)#, self.ratio_color[self.idx_current])
+
+        return
+
+    def line_plot(self):
+
+        feature = self.id_arr[self.idx_current]
+        self._ax2.step(self.wave_range, self.data_df[self.idx_current, :], label=feature,
+                      color=self.color_dict[feature], where='mid')
+
+        return

aspect/tools.py

@@ -0,0 +1,153 @@
+import logging
+import numpy as np
+from .io import Aspect_Error
+
+# Log variable
+_logger = logging.getLogger('aspect')
+np.random.seed(42)
+
+
+def monte_carlo_expansion(flux_array, err_array, n_mc, for_loop=True):
+
+
+    # # Get the noise scale for the selections
+    # if noise_scale is None:
+    #     noise_scale = self._spec.err_flux if self._spec.err_flux is not None else self._spec.cont_std
+    #
+    #     if noise_scale is None:
+    #         _logger.warning(f"No flux uncertainty provided for the line detection. There won't be a confidence value"
+    #                         f" for the predictions.")
+    #         self.n_mc = 1
+
+    # Scale array depending on the scale
+    # noise_scale = err_array if np.isscalar(err_array) else err_array[..., None]
+    # noise_scale = err_array
+
+    # Add random noise matrix
+    # noise_array = np.random.normal(0, err_array, size=(n_mc, flux_array.size))
+    # mc_flux = flux_array[:, :, np.newaxis] + noise_array
+
+    if for_loop:
+        mc_flux = flux_array + np.random.normal(0, err_array, size=(n_mc, flux_array.size))
+
+    else:
+        noise_scale = err_array if np.isscalar(err_array) else err_array[..., None]
+        noise_matrix_shape = (flux_array.shape[0], flux_array.shape[1], n_mc)
+
+        noise_array = np.random.normal(0, noise_scale, size=noise_matrix_shape)
+        mc_flux = flux_array[:, :, np.newaxis] + noise_array
+
+    return mc_flux
+
+
+def scale_min_max(data, axis=None):
+
+    data_min_array = data.min(axis=axis, keepdims=True)
+    data_max_array = data.max(axis=axis, keepdims=True)
+    data_norm = (data - data_min_array) / (data_max_array - data_min_array)
+
+    return data_norm
+
+
+def scale_log(data, log_base, axis=None):
+
+    data_min_array = data.min(axis=axis, keepdims=True)
+
+    y_cont = data - data_min_array + 1
+    data_norm = np.emath.logn(log_base, y_cont)
+
+    return data_norm
+
+
+def scale_log_min_max(data, log_base, axis=None):
+
+    data_min_array = data.min(axis=axis, keepdims=True)
+    data_cont = data - data_min_array + 1
+    log_data = np.emath.logn(log_base, data_cont)
+    log_min_array, log_max_array = log_data.min(axis=axis, keepdims=True), log_data.max(axis=axis, keepdims=True)
+    data_norm = (log_data - log_min_array) / (log_max_array - log_min_array)
+
+    return data_norm
+
+
+def feature_scaling(data, transformation, log_base=None, axis=1):
+
+    match transformation:
+        case 'min-max':
+            return scale_min_max(data, axis=axis)
+        case 'log':
+            return scale_log(data, log_base=log_base, axis=axis)
+        case 'log-min-max':
+            return scale_log_min_max(data, log_base=log_base, axis=axis)
+        case _:
+            raise Aspect_Error(f'Input scaling: "{transformation}" is not recognized')
+
+
+def white_noise_scale(flux_arr):
+
+    min, max = flux_arr.min(), flux_arr.max()
+
+    diff = max - min if max != 0 else np.abs(max-min)
+
+                # 1 White noise, 2 continuum
+    output_type = 1 if diff > 10 else 2
+
+    return output_type
+
+
+def detection_function(x_ratio):
+
+    # Original
+    # 2.5 + 1/np.square(x_ratio - 0.1) + 0.5 * np.square(x_ratio)
+
+    return 0.5 * np.power(x_ratio, 2) - 0.5 * x_ratio + 5
+
+
+def cosmic_ray_function(x_ratio, res_ratio_check=True):
+
+    # Resolution ration
+    if res_ratio_check:
+        output = np.exp(0.5 * np.power(x_ratio, -2))
+
+    # Intensity ratio
+    else:
+        output = 1/np.sqrt(2 * np.log(x_ratio))
+
+    return output
+
+
+def stratify_sample(x_arr, y_arr, n_samples=None, categories=None, randomize=True):
+
+    # Inspect input sample
+    unique_categories, counts = np.unique(y_arr, return_counts=True)
+    min_count = min(counts)
+
+    # Use all categories and the minimum number of counts if not provided
+    n_samples = n_samples if n_samples is not None else min_count
+    categories = categories if categories is not None else unique_categories
+
+    # Check input sample size is below category
+    if n_samples > min_count:
+        _logger.warning(f'The input sample minimun size category ({unique_categories[counts==min_count]} = {min_count})'
+                        f' is less than the requested input size ({n_samples}). The minimum count will be used instead.')
+        n_samples = min_count
+
+    # Empty mask for the target categories
+    selection_mask = np.zeros(y_arr.size, dtype=bool)
+
+    # Mark indices for each category
+    print(f'\nInput sample has {y_arr.shape[0]} entries:')
+    for j, category in enumerate(categories):
+        print(f'- {category}: {counts[j]}')
+        category_indices = np.where(y_arr == category)[0]
+        sampled_indices = np.random.choice(category_indices, n_samples, replace=False)
+        selection_mask[sampled_indices] = True
+    print(f'Cropping to {n_samples} entries per category')
+
+    selection_mask = np.nonzero(selection_mask)[0]
+
+    if randomize:
+        np.random.shuffle(selection_mask)
+
+    return x_arr[selection_mask, :], y_arr[selection_mask]
+

aspect/trainer.py

@@ -0,0 +1,102 @@
+import importlib
+import numpy as np
+import joblib
+import toml
+from sklearn.model_selection import cross_val_score, cross_val_predict
+from sklearn.metrics import confusion_matrix
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.model_selection import StratifiedShuffleSplit
+from sklearn.metrics import confusion_matrix, precision_score, recall_score, f1_score
+from time import time
+from pathlib import Path
+from .io import cfg as aspect_cfg
+
+
+def get_training_test_sets(x_arr, y_arr, test_fraction, random_state=None):
+
+    # Split into training and testing:
+    print(f'\nSplitting sample with categories:')
+    print(np.unique(y_arr))
+    sss = StratifiedShuffleSplit(n_splits=1, train_size=int(y_arr.size * (1 - test_fraction)),
+                                 test_size=int(y_arr.size * test_fraction), random_state=random_state)
+
+    for train_index, test_index in sss.split(x_arr, y_arr):
+        X_train, X_test = x_arr[train_index, 2:], x_arr[test_index, 2:]
+        y_train, y_test = y_arr[train_index], y_arr[test_index]
+
+    # Convert strings to integers
+    y_train = np.vectorize(aspect_cfg['shape_number'].get)(y_train)
+    y_test = np.vectorize(aspect_cfg['shape_number'].get)(y_test)
+
+    return X_train, y_train, X_test, y_test
+
+
+def components_trainer(model_label, x_arr, y_arr, fit_cfg, list_labels, output_folder=None, test_fraction=0.1,
+                       random_state=None):
+
+    # Preparing the estimator:
+    print(f'\nLoading estimator: {fit_cfg["estimator"]["class"]}')
+    estimator = getattr(importlib.import_module(fit_cfg['estimator']["module"]), fit_cfg['estimator']["class"])
+    estimator_params = fit_cfg.get('estimator_params', {})
+
+    # Split into training and testing:
+    print(f'\nSplitting sample with categories:')
+    X_train, y_train, X_test, y_test = get_training_test_sets(x_arr, y_arr, test_fraction, random_state)
+
+    # Run the training
+    print(f'\nTraining: {y_train.size/len(fit_cfg["categories"]):.0f} * {len(fit_cfg["categories"])} = {y_train.size}  points ({model_label})')
+    print(f'- Settings: {fit_cfg["estimator_params"]}\n')
+    start_time = time()
+    ml_function = estimator(**estimator_params)
+    ml_function.fit(X_train, y_train)
+    end_time = np.round((time()-start_time)/60, 2)
+    print(f'- completed ({end_time} minutes)')
+
+    # Save the trained model and configuration
+    output_folder = Path(output_folder)/'results'
+    output_folder.mkdir(parents=True, exist_ok=True)
+
+    model_address = output_folder/f'{model_label}.joblib'
+    joblib.dump(ml_function, model_address)
+
+    # Run initial diagnostics
+    print(f'\nReloading model from: {model_address}')
+    start_time = time()
+    ml_function = joblib.load(model_address)
+    fit_time = np.round((time()-start_time)/60, 3)
+    print(f'- completed ({fit_time} minutes)')
+
+    print(f'\nRuning prediction on test set ({y_test.size} points)')
+    start_time = time()
+    y_pred = ml_function.predict(X_test)
+    print(f'- completed ({(time()-start_time)/60:0.2f} minutes)')
+
+    # Testing confussion matrix
+    print(f'\nConfusion matrix in test set ({y_test.size} points)')
+    start_time = time()
+    conf_matrix_test = confusion_matrix(y_test, y_pred, normalize="all")
+    print(f'- completed ({(time()-start_time)/60:0.2f} minutes)')
+
+    # Precision, recall and f1:
+    print(f'\nF1, Precision and recall diagnostics ({y_test.size} points)')
+    start_time = time()
+    pres = precision_score(y_test, y_pred, average='macro')
+    recall = recall_score(y_test, y_pred, average='macro')
+    f1 = f1_score(y_test, y_pred, average='macro')
+    print(f'- completed ({(time()-start_time)/60:0.2f} minutes)')
+
+    print(f'\nModel outputs')
+    print(f'- F1: \n {f1}')
+    print(f'- Precision: \n {pres}')
+    print(f'- Recall: \n {recall}')
+    print(f'- Testing confusion matrix: \n {conf_matrix_test}')
+    print(f'- Fitting time: \n {fit_time}')
+
+    # Save results into a TOML file
+    toml_path = output_folder/f'{model_label}.toml'
+    output_dict = {'resuts': {'f1':f1, 'precision':pres, 'Recall':recall, 'confusion_matrix':conf_matrix_test,
+                              'fit_time': fit_time}, 'properties': fit_cfg,}
+    with open(toml_path, 'w') as f:
+        toml.dump(output_dict, f)
+
+    return

aspect/workflow.py

@@ -0,0 +1,299 @@
+import numpy as np
+from time import time
+from .io import read_trained_model, DEFAULT_MODEL_ADDRESS, cfg, Aspect_Error
+from .tools import monte_carlo_expansion, feature_scaling, white_noise_scale
+from matplotlib import pyplot as plt
+
+CHOICE_DM = np.array(cfg['decision_matrices']['choice'])
+TIME_DM = np.array(cfg['decision_matrices']['time'])
+
+# TODO Larger box overwrites small
+# TODO COMPLEX overwrites simple
+
+def unpack_spec_flux(spectrum, rest_wl_lim):
+
+    # Extract the mask if masked array
+    mask_check = np.ma.isMaskedArray(spectrum.flux)
+    pixel_mask = spectrum.flux.mask if mask_check else np.zeros(spectrum.flux.size).astype(bool)
+
+    # Limit to region if requested
+    if rest_wl_lim is not None:
+        wave_rest = spectrum.wave_rest if not mask_check else spectrum.wave_rest.data
+        pixel_mask = pixel_mask |  ~((wave_rest > rest_wl_lim[0]) & (wave_rest < rest_wl_lim[1]))
+
+    # Extract flux and error arrays and invert the mask for location of the valid data indeces
+    pixel_mask = ~pixel_mask
+    flux_arr = spectrum.flux[pixel_mask] if not mask_check else spectrum.flux.data[pixel_mask]
+    err_arr = spectrum.err_flux[pixel_mask] if not mask_check else spectrum.err_flux.data[pixel_mask]
+    idcs_data_mask = np.flatnonzero(pixel_mask)
+
+    return flux_arr, err_arr, idcs_data_mask
+
+
+def enbox_spectrum(input_flux, box_size, range_box):
+
+    # Use only the true entries from the mask
+    # flux_array = input_flux if not np.ma.isMaskedArray(input_flux) else input_flux.data[~input_flux.mask]
+
+    # Reshape to the detection interval
+    n_intervals = input_flux.size - box_size + 1
+    input_flux = input_flux[np.arange(n_intervals)[:, None] + range_box]
+
+    # # Remove nan entries
+    # idcs_nan_rows = np.isnan(input_flux).any(axis=1)
+    # flux_array = input_flux[~idcs_nan_rows, :]
+
+    return input_flux
+
+
+def detection_spectrum_prechecks(y_arr, box_size, idcs_data):
+
+    valid = True
+
+    # Box bigger than spectrum or all entries are masked
+    if (y_arr.size < box_size) or (idcs_data.size < box_size):
+        valid = False
+
+    return valid
+
+
+class ModelManager:
+
+    def __init__(self, model_address=None, n_jobs=None, verbose=0):
+
+        self.cfg = None
+        self.detection_model = None
+        self.b_pixels_arr = None
+        self.scale = None
+        self.log_base = None
+
+        self.categories_str = None
+        self.feature_number_dict = None
+        self.number_feature_dict = None
+        self.n_categories = None
+
+        # Default values
+        model_address = DEFAULT_MODEL_ADDRESS if model_address is None else model_address
+
+        # Load the model
+        self.predictor, self.cfg = read_trained_model(model_address)
+
+        # Specify cores (default 4)
+        n_jobs = 4
+        self.predictor.n_jobs = n_jobs  # Use 4 cores
+        self.predictor.verbose = verbose  # No output message
+
+        # Array with the boxes size
+        self.b_pixels_arr = np.atleast_1d(self.cfg['properties']['box_size'])
+        self.b_pixels_range = np.atleast_2d(np.arange(self.b_pixels_arr [0]))
+
+        # Scaling properties
+        self.scale = self.cfg['properties']['scale']
+        self.log_base = self.cfg['properties'].get('log_base')
+        self.categories_str = np.array(self.cfg['properties']['categories'])
+        self.feature_number_dict = cfg['shape_number']
+        self.number_feature_dict = {v: k for k, v in self.feature_number_dict.items()}
+
+        self.n_categories = len(self.feature_number_dict)
+
+        return
+
+    def reload_model(self, model_address=None, n_jobs=None):
+
+        # Call the constructor again
+        self.__init__(model_address, n_jobs)
+
+        return
+
+
+# Create object with default model
+model = ModelManager()
+
+
+class SpectrumDetector:
+
+    def __init__(self, spectrum, model_address=None):
+
+        self._spec = spectrum
+        self.narrow_detect = None
+        self.box_width = None
+        self.range_box = None
+        self.n_mc = 100
+        self.detection_min = 40
+        self.white_noise_maximum = 50
+
+        self.line_1d_pred = None
+        self.line_2d_pred = None
+        self.line_pred = None
+
+        self.features = None
+
+        # Read the detection model
+        if model_address is None:
+            self.model = model
+
+        # Arrays to store the data
+        self.seg_flux = None
+        self.seg_err = None
+
+        self.seg_pred = None
+        self.seg_conf = None
+
+        self.pred_arr = None
+        self.conf_arr = None
+
+        return
+
+    def detection(self, feature_list=None, bands=None, exclude_continuum=True, show_steps=False, rest_wl_lim=None):
+
+        # Support variables
+        box_size = self.model.b_pixels_arr[0]
+        box_range = self.model.b_pixels_range[0]
+
+        # Remove masks from flux and uncertainty
+        y_arr, err_arr, idcs_data = unpack_spec_flux(self._spec, rest_wl_lim)
+
+        # Check the validity of the spectrum
+        if detection_spectrum_prechecks(y_arr, box_size, idcs_data):
+
+            # Empty containers
+            self.pred_arr = np.zeros(self._spec.flux.size, dtype=np.int64)
+            self.conf_arr = np.zeros(self._spec.flux.size, dtype=np.int64)
+
+            self.seg_pred = np.zeros(box_size, dtype=np.int64)
+            self.seg_conf = np.zeros(box_size, dtype=np.int64)
+
+            # Reshape spectrum to box size
+            y_enbox = enbox_spectrum(y_arr, box_size, box_range)
+            err_enbox = enbox_spectrum(err_arr, box_size, box_range)
+
+            # MC expansion
+            y_enbox = monte_carlo_expansion(y_enbox, err_enbox, self.n_mc, for_loop=False)
+
+            # Scaling
+            y_norm = feature_scaling(y_enbox, 'min-max', 1)
+
+            # Run the prediction
+            y_reshaped = y_norm.transpose(0, 2, 1).reshape(-1, box_size)
+            y_pred = self.model.predictor.predict(y_reshaped)
+            y_pred = y_pred.reshape(-1, 100)
+
+            # Get the count of types detected on Monte-Carlo
+            counts_categories = np.apply_along_axis(np.bincount, axis=1, arr=y_pred, minlength=self.model.n_categories)
+
+            # Exclude white-noise regions from review:
+            if exclude_continuum:
+                idcs_detection = np.flatnonzero(counts_categories[:, 1] < self.white_noise_maximum)
+            else:
+                idcs_detection = np.arange(y_arr.size - box_size)
+
+            for idx in idcs_detection:
+
+                # Get segment arrays
+                self.seg_pred[:] = self.pred_arr[idcs_data][idx:idx + box_size]
+                self.seg_conf[:] = self.conf_arr[idcs_data][idx:idx + box_size]
+
+                # Count
+                counts = counts_categories[idx, :]
+                idcs_categories = counts > self.detection_min
+
+                # Choose detection
+                out_type, out_confidence = self.detection_evaluation(counts, idcs_categories)
+
+                # Check with previous detection
+                idcs_pred, new_pred, new_conf = self.detection_revision(idx, box_size, out_type, out_confidence)
+
+                # Only pass if more than half
+                half_check = idcs_pred[6:].sum() > 5
+                if half_check:
+                    idcs_pred = np.flatnonzero(idcs_pred)
+                    self.seg_pred[idcs_pred] = new_pred[idcs_pred]
+                    self.seg_conf[idcs_pred] = new_conf[idcs_pred]
+                else:
+                    self.seg_pred[:] = self.pred_arr[idcs_data][idx:idx + box_size]
+                    self.seg_conf[:] = self.conf_arr[idcs_data][idx:idx + box_size]
+
+                if show_steps:
+                    self.plot_steps(y_norm[idx, :], idx, counts, idcs_categories, out_type, out_confidence,
+                                    self.pred_arr[idcs_data][idx:idx + box_size], self.conf_arr[idcs_data][idx:idx + box_size],
+                                    idcs_pred, new_pred, new_conf)
+
+                # Assign new categories and confidence
+                self.pred_arr[idcs_data[idx:idx + box_size]] = self.seg_pred[:]
+                self.conf_arr[idcs_data[idx:idx + box_size]] = self.seg_conf[:]
+
+
+
+
+    def detection_evaluation(self, counts_categories, idcs_categories):
+
+        n_detections = idcs_categories.sum()
+
+        match n_detections:
+
+            # Undefined
+            case 0:
+                return 0, 0
+
+            # One detection
+            case 1:
+                return np.argmax(idcs_categories), counts_categories[idcs_categories][0]
+
+            # Two detections
+            case 2:
+                category_candidates = np.flatnonzero(idcs_categories)
+                idx_output = CHOICE_DM[category_candidates[0], category_candidates[1]]
+                output_type, output_count = category_candidates[idx_output], counts_categories[idcs_categories][idx_output]
+                return output_type, output_count
+
+            # Three detections
+            case _:
+                raise Aspect_Error(f'Number of detections: "{n_detections}" is not recognized')
+
+
+    def detection_revision(self, idx, box_size, new_type, new_confidence):
+
+        new_pred, new_conf = np.full(box_size, new_type), np.full(box_size, new_confidence)
+        idcs_pred = TIME_DM[self.seg_pred, new_pred]
+        # idcs_pred = np.nonzero(idcs_pred)
+
+        return idcs_pred, new_pred, new_conf
+
+
+    def transform_category(self, input_category, segment_flux):
+
+        match input_category:
+
+            # White noise scale
+            case 1:
+                return white_noise_scale(segment_flux)
+
+            case _:
+                return input_category
+
+    def plot_steps(self, y_norm, idx, counts, idcs_categories, out_type, out_confidence, old_pred, old_conf,
+                   idcs_pred, new_pred, new_conf):
+
+        x_arr = self._spec.wave_rest if not np.ma.isMaskedArray(self._spec.wave_rest) else self._spec.wave_rest.data[~self._spec.wave_rest.mask]
+        x_sect = x_arr[idx:idx+y_norm.shape[0]]
+        print(f'Idx "{idx}"; counts: {counts}; Output: {model.number_feature_dict[out_type]} ({out_type})')
+
+        colors_old = [cfg['colors'][model.number_feature_dict[val]] for val in old_pred]
+        colors_new = [cfg['colors'][model.number_feature_dict[val]] for val in self.seg_pred]
+
+        fig, ax = plt.subplots()
+        color_detection = cfg['colors'][model.number_feature_dict[out_type]]
+        ax.step(x_sect, y_norm[:,0], where='mid', color=color_detection, label='Out detection')
+        ax.scatter(x_sect, np.zeros(x_sect.size), color=colors_old, label='Old prediction')
+        ax.scatter(x_sect, np.ones(x_sect.size), color=colors_new, label='New prediction')
+        ax.set_xlabel(r'Wavelength $(\AA)$')
+
+        ax_secondary = ax.twinx()  # Creates a twin y-axis on the right
+        ax_secondary.set_ylim(ax.get_ylim())  # Match the primary y-axis limits
+        ax_secondary.set_yticks([0, 0.5, 1])  # Custom tick positions
+        ax_secondary.set_yticklabels(['Previous\nClassification', 'Present\nClassification', 'Output\nClassification'])
+
+        plt.tight_layout()
+        plt.show()
+
+        return

aspect_stable-0.1.0.dist-info/METADATA

@@ -0,0 +1,41 @@
+Metadata-Version: 2.1
+Name: aspect-stable
+Version: 0.1.0
+Summary: Automatic SPEctra Components Tagging
+Author-email: Vital Fernández <vgf@umich.edu>
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.7
+Requires-Python: >=3.10
+Description-Content-Type: text/x-rst
+Requires-Dist: numpy~=1.2
+Requires-Dist: joblib~=1.3
+Requires-Dist: matplotlib~=3.7
+Requires-Dist: sklearn~=1.5
+Requires-Dist: tomli>=2.0.0; python_version < "3.11"
+Provides-Extra: tests
+Requires-Dist: pytest~=7.4; extra == "tests"
+Requires-Dist: pytest-cov~=4.1; extra == "tests"
+Requires-Dist: pytest-mpl~=0.16; extra == "tests"
+Provides-Extra: docs
+Requires-Dist: nbsphinx~=0.9; extra == "docs"
+Requires-Dist: ipympl~=0.9; extra == "docs"
+Requires-Dist: sphinx-rtd-theme~=1.0; extra == "docs"
+
+
+This library provides a set of tools to identify features in astronomical spectra
+
+Installation
+============
+
+The library can be installed directly from its PyPi_ project page running the command:
+
+``pip install aspect-stable``
+
+Development
+===========
+
+ASPECT is currently on an alpha release. Any comment/issue/request can be added as an issue on the github page.
+Please commit to dev branch.
+
+.. _PyPi: https://pypi.org/project/aspect-stable/

aspect_stable-0.1.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+aspect/__init__.py,sha256=347qWVlQ7-cQ7qEIpeNcjakEHrMxkxHd3khM4dncAZk,415
+aspect/aspect.toml,sha256=PRTDA8_2oolApOvm0sxFGnm2UKu5aG17TFNDG9NbrGw,4940
+aspect/changelog.txt,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+aspect/io.py,sha256=O8y3g8NimOX1-6FJty1_Ky3x1rpTynX7ITKIwvNioss,2686
+aspect/plots.py,sha256=HRhGXhQYSxeSPlYstDlO7XESoBSZqSFanx5JRzCfX18,10359
+aspect/tools.py,sha256=NWG5lpbpxEyNI9J23yI_v3QorkB3JaQZ9BfiQli85GQ,4912
+aspect/trainer.py,sha256=a8ftyVSbqouB1vrcXHbFJWXKz3buj9pOPEJmBSp7NIg,4254
+aspect/workflow.py,sha256=2E0ZpAWH1i-arC0bDZWL-X7k5A9VkESmBmUGZ34zprE,10996
+aspect/models/aspect_min-max_mediumbox_v3_model.joblib,sha256=CPVKObt44quRwAMSurYAR1DQ6egtdaZgPHI4Ev2Poks,2137409
+aspect/models/aspect_min-max_mediumbox_v3_model.toml,sha256=Mvp2LDbo1iGA7IsntMYE3iz9UCXk9cCJEyIzwhnWXfw,1585
+aspect_stable-0.1.0.dist-info/METADATA,sha256=cT5Q1wtx3_arl2hz3tMRySua1uVY-NF9QE4AU5LBs5A,1304
+aspect_stable-0.1.0.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+aspect_stable-0.1.0.dist-info/top_level.txt,sha256=RknseycB7xet8TKNMnAzNEdvIcRwUWffKDxYxGimWO8,7
+aspect_stable-0.1.0.dist-info/RECORD,,

aspect_stable-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (75.6.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

aspect_stable-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+aspect