AeroViz 0.1.3b0__py3-none-any.whl → 0.1.4__py3-none-any.whl

AeroViz/plot/templates/koschmieder.py

@@ -1,4 +1,3 @@
-from typing import Literal
 
 import matplotlib.pyplot as plt
 import numpy as np
@@ -11,146 +10,83 @@ from AeroViz.plot.utils import *
 __all__ = ['koschmieder']
 
 
-@set_figure(fs=12)
+@set_figure
 def koschmieder(df: pd.DataFrame,
-                y: Literal['Vis_Naked', 'Vis_LPV'],
-                function: Literal['log', 'reciprocal'] = 'log',
+                vis: str,
+                ext: list[str],
                 ax: Axes | None = None,
-                **kwargs) -> tuple[Figure, Axes]:
-    # x = Visibility, y = Extinction, log-log fit!!
-    def _log_fit(x, y, func=lambda x, a: -x + a):
-        x_log = np.log(x)
-        y_log = np.log(y)
+                **kwargs
+                ) -> tuple[Figure, Axes]:
+    """
+    Plot Koschmieder relationship between Visibility and Extinction.
 
+    x = Visibility, y = Extinction, log-log fit!!
+    """
+    def _log_fit(x, y, func=lambda x, a: -x + a):
+        x_log, y_log = np.log(x), np.log(y)
         popt, pcov = curve_fit(func, x_log, y_log)
 
-        residuals = y_log - func(x_log, *popt)
-        ss_res = np.sum(residuals ** 2)
-        ss_total = np.sum((y_log - np.mean(y_log)) ** 2)
-        r_squared = 1 - (ss_res / ss_total)
-        print(f'Const_Log = {popt[0].round(3)}')
-        print(f'Const = {np.exp(popt)[0].round(3)}')
-        print(f'R^2 = {r_squared.round(3)}')
         return np.exp(popt)[0], pcov
 
-    def _reciprocal_fit(x, y, func=lambda x, a, b: a / (x ** b)):
-        popt, pcov = curve_fit(func, x, y)
-
-        residuals = y - func(x, *popt)
-        ss_res = np.sum(residuals ** 2)
-        ss_total = np.sum((y - np.mean(y)) ** 2)
-        r_squared = 1 - (ss_res / ss_total)
-        print(f'Const = {popt.round(3)}')
-        print(f'  R^2 = {r_squared.round(3)}')
-        return popt, pcov
-
     fig, ax = plt.subplots(**kwargs.get('fig_kws', {})) if ax is None else (ax.get_figure(), ax)
 
-    _df1 = df[['Extinction', 'ExtinctionByGas', y]].dropna().copy()
-    _df2 = df[['total_ext_dry', 'ExtinctionByGas', y]].dropna().copy()
-
-    x_data1 = _df1[y]
-    y_data1 = _df1['Extinction'] + _df1['ExtinctionByGas']
-
-    x_data2 = _df2[y]
-    y_data2 = _df2['total_ext_dry'] + _df2['ExtinctionByGas']
-
-    para_coeff = []
     boxcolors = ['#3f83bf', '#a5bf6b']
+    scattercolor = ['blue', 'green']
+    arts = []
+    labels = []
 
-    for i, (df_, x_data, y_data) in enumerate(zip([_df1, _df2], [x_data1, x_data2], [y_data1, y_data2])):
-        df_['Total_Ext'] = y_data
-
-        if y == 'Vis_Naked':
-            df_grp = df_.groupby(f'{y}')
-
-            vals, median_vals, vis = [], [], []
-            for j, (name, subdf) in enumerate(df_grp):
-                if len(subdf['Total_Ext'].dropna()) > 20:
-                    vis.append('{:.0f}'.format(name))
-                    vals.append(subdf['Total_Ext'].dropna().values)
-                    median_vals.append(subdf['Total_Ext'].dropna().median())
+    for i, ext_col in enumerate(ext):
+        _df = df[[ext_col, vis]].dropna().copy()
+        x_data = _df[vis]
+        y_data = _df[ext_col]
 
-            plt.boxplot(vals, labels=vis, positions=np.array(vis, dtype='int'), widths=0.4,
-                        showfliers=False, showmeans=True, meanline=False, patch_artist=True,
-                        boxprops=dict(facecolor=boxcolors[i], alpha=.7),
-                        meanprops=dict(marker='o', markerfacecolor='white', markeredgecolor='k', markersize=4),
-                        medianprops=dict(color='#000000', ls='-'))
+        bins = np.linspace(0, 50, 25)
+        wid = (bins + (bins[1] - bins[0]) / 2)[0:-1]
 
-            plt.scatter(x_data, y_data, marker='.', s=10, facecolor='white', edgecolor=boxcolors[i], alpha=0.1)
+        _df[f'{vis}_bins'] = pd.cut(x_data, bins=bins, labels=wid)
 
-        if y == 'Vis_LPV':
-            bins = np.linspace(0, 70, 36)
-            wid = (bins + (bins[1] - bins[0]) / 2)[0:-1]
+        grouped = _df.groupby(f'{vis}_bins', observed=False)
 
-            df_[f'{x_data.name}' + '_bins'] = pd.cut(x=x_data, bins=bins, labels=wid)
+        vis_labels, vals, median_vals = [], [], []
+        for _, subdf in grouped:
+            if len(subdf[ext_col].dropna()) > 3:
+                vis_labels.append(subdf[vis].mean())
+                vals.append(subdf[ext_col].dropna().values)
+                median_vals.append(subdf[ext_col].mean())
 
-            grouped = df_.groupby(f'{x_data.name}' + '_bins', observed=False)
+        plt.boxplot(vals, labels=vis_labels, positions=np.array(vis_labels, dtype='float'),
+                    widths=(bins[1] - bins[0]) / 2.5,
+                    showfliers=False, showmeans=True, meanline=False, patch_artist=True,
+                    boxprops=dict(facecolor=boxcolors[i], alpha=.7),
+                    meanprops=dict(marker='o', markerfacecolor='white', markeredgecolor='k', markersize=4),
+                    medianprops=dict(color='#000000', ls='-'))
 
-            vals, median_vals, vis = [], [], []
-            for j, (name, subdf) in enumerate(grouped):
-                if len(subdf['Total_Ext'].dropna()) > 20:
-                    vis.append('{:.1f}'.format(name))
-                    vals.append(subdf['Total_Ext'].dropna().values)
-                    median_vals.append(subdf['Total_Ext'].dropna().mean())
-
-            plt.boxplot(vals, labels=vis, positions=np.array(vis, dtype='float'), widths=(bins[1] - bins[0]) / 2.5,
-                        showfliers=False, showmeans=True, meanline=False, patch_artist=True,
-                        boxprops=dict(facecolor=boxcolors[i], alpha=.7),
-                        meanprops=dict(marker='o', markerfacecolor='white', markeredgecolor='k', markersize=4),
-                        medianprops=dict(color='#000000', ls='-'))
-
-            plt.scatter(x_data, y_data, marker='.', s=10, facecolor='white', edgecolor=boxcolors[i], alpha=0.1)
+        plt.scatter(x_data, y_data, marker='.', s=10, facecolor='white', edgecolor=boxcolors[i], alpha=0.1)
 
         # fit curve
-        _x = np.array(vis, dtype='float')
-        _y = np.array(median_vals, dtype='float')
-
-        if function == 'log':
-            func = lambda x, a: a / x
-            coeff, pcov = _log_fit(_x, _y)
-
-        else:
-            func = lambda x, a, b: a / (x ** b)
-            coeff, pcov = _reciprocal_fit(_x, _y)
-
-        para_coeff.append(coeff)
+        coeff, _ = _log_fit(np.array(vis_labels, dtype='float'), np.array(median_vals, dtype='float'))
 
-    # Plot lines (ref & Measurement)
-    x_fit = np.linspace(0.1, 70, 1000)
+        # Plot lines (ref & Measurement)
+        x_fit = np.linspace(0.1, 50, 1000)
 
-    if function == 'log':
-        line1, = ax.plot(x_fit, func(x_fit, para_coeff[0]), c='b', lw=3)
-        line2, = ax.plot(x_fit, func(x_fit, para_coeff[1]), c='g', lw=3)
+        func = lambda x, a: a / x
+        line, = ax.plot(x_fit, func(x_fit, coeff), c=scattercolor[i], lw=3,
+                        label=f'Vis (km) = {round(coeff)} / Ext')
 
-        labels = ['Vis (km) = ' + f'{round(para_coeff[0])}' + ' / Ext (Dry Extinction)',
-                  'Vis (km) = ' + f'{round(para_coeff[1])}' + ' / Ext (Amb Extinction)']
+        arts.append(line)
+        labels.append(f'Vis (km) = {round(coeff)} / Ext')
 
-    else:
-        x_fit = np.linspace(0.1, 70, 1000)
-        line1, = ax.plot(x_fit, func(x_fit, *para_coeff[0]), c='b', lw=3)
-        line2, = ax.plot(x_fit, func(x_fit, *para_coeff[1]), c='g', lw=3)
+    ax.legend(handles=arts, labels=labels, loc='upper right', prop=dict(weight='bold'), bbox_to_anchor=(0.99, 0.99))
 
-        labels = [f'Ext = ' + '{:.0f} / Vis ^ {:.3f}'.format(*para_coeff[0]) + ' (Dry Extinction)',
-                  f'Ext = ' + '{:.0f} / Vis ^ {:.3f}'.format(*para_coeff[1]) + ' (Amb Extinction)']
+    ax.set(xlabel=kwargs.get('xlim', 'Visibility (km)'),
+           ylabel=kwargs.get('xlim', 'Extinction (1/Mm)'),
+           title=kwargs.get('ylim', 'Koschmieder relationship'),
+           xlim=kwargs.get('xlim', (0, 30)),
+           ylim=kwargs.get('ylim', (0, 500))
+           )
 
-    plt.legend(handles=[line1, line2], labels=labels, loc='upper right', prop=dict(size=10, weight='bold'),
-               bbox_to_anchor=(0.99, 0.99))
-
-    plt.xticks(ticks=np.array(range(0, 51, 5)), labels=np.array(range(0, 51, 5)))
-    plt.xlim(0, 50)
-    plt.ylim(0, 700)
-    plt.title(r'$\bf Koschmieder\ relationship$')
-    plt.xlabel(f'{y} (km)')
-    plt.ylabel(r'$\bf Extinction\ coefficient\ (1/Mm)$')
+    plt.xticks(ticks=np.array(range(0, 31, 5)), labels=np.array(range(0, 31, 5)))
 
     plt.show()
 
     return fig, ax
-
-
-if __name__ == '__main__':
-    from AeroViz.tools import DataBase
-
-    koschmieder(DataBase(), 'Vis_LPV', 'log')
-# koschmieder(DataBase, 'Vis_Naked', 'reciprocal')

AeroViz/plot/templates/metal_heatmap.py

@@ -2,41 +2,139 @@ import matplotlib.pyplot as plt
 import numpy as np
 import seaborn as sns
 from matplotlib.pyplot import Figure, Axes
-from pandas import DataFrame, date_range
+from pandas import DataFrame, date_range, concat
 from sklearn.preprocessing import StandardScaler
 
 from AeroViz.plot.utils import *
 
+__all__ = ['metal_heatmaps', 'process_data_with_two_df']
+
+
+def process_data(df, detected_limit=True, outlier_threshold=5, smoothing_window=6, fill_method='MDL'):
+    # Fill missing values based on the specified method
+    df = fill_missing_values(df.copy(), method=fill_method)
 
-def process_data(df):
-    # detected_limit = 0.0001
-    df = df.where(df >= 0.0001, np.nan)
     # Normalize the data
-    df = DataFrame(StandardScaler().fit_transform(df), index=df.index, columns=df.columns)
+    df = normalize_data(df)
+
     # Remove outliers
-    df = df[(np.abs(df) < 6)]
-    # Interpolate the missing values
+    df = remove_outliers(df, threshold=outlier_threshold)
+
+    # Interpolate missing values
     df = df.interpolate(method='linear')
+
     # Smooth the data
-    df = df.rolling(window=3, min_periods=1).mean()
+    df = smooth_data(df, window=smoothing_window)
 
     return df
 
 
+def process_data_with_two_df(df, df2, outlier_threshold=5, smoothing_window=6, fill_method='MDL'):
+    # Shift the first DataFrame by 30 minutes
+    df = df.shift(freq='30min')
+
+    # Fill missing values for both DataFrames
+    df = fill_missing_values(df.copy(), method=fill_method)
+    df2 = fill_missing_values(df2.copy(), method=fill_method)
+
+    # Normalize both DataFrames together
+    df, df2 = normalize_and_split(df, df2)
+
+    # Shift the first DataFrame back by 30 minutes
+    df = df.shift(freq='-30min')
+
+    # Remove outliers for both DataFrames
+    df = remove_outliers(df, threshold=outlier_threshold)
+    df2 = remove_outliers(df2, threshold=outlier_threshold)
+
+    # Interpolate missing values
+    df = df.interpolate(method='linear')
+    df2 = df2.interpolate(method='linear')
+
+    # Smooth the data
+    df = smooth_data(df, window=smoothing_window)
+    df2 = smooth_data(df2, window=smoothing_window)
+
+    return df, df2
+
+
+def fill_missing_values(df, method='MDL'):
+    if method == 'interpolate':
+        return df.interpolate(method='linear')
+    else:
+        return fill_with_mdl(df)
+
+
+def fill_with_mdl(df):
+    # Minimum detection limit (MDL) dictionary
+    MDL = {
+        'Al': 100, 'Si': 18, 'P': 5.2, 'S': 3.2,
+        'Cl': 1.7, 'K': 1.2, 'Ca': 0.3, 'Ti': 1.6,
+        'V': 0.12, 'Cr': 0.12, 'Mn': 0.14, 'Fe': 0.17,
+        'Co': 0.14, 'Ni': 0.096, 'Cu': 0.079, 'Zn': 0.067,
+        'Ga': 0.059, 'Ge': 0.056, 'As': 0.063, 'Se': 0.081,
+        'Br': 0.1, 'Rb': 0.19, 'Sr': 0.22, 'Y': 0.28,
+        'Zr': 0.33, 'Nb': 0.41, 'Mo': 0.48, 'Pd': 2.2,
+        'Ag': 1.9, 'Cd': 2.5, 'In': 3.1, 'Sn': 4.1,
+        'Sb': 5.2, 'Te': 0.6, 'I': 0.49, 'Cs': 0.37,
+        'Ba': 0.39, 'La': 0.36, 'Ce': 0.3, 'Pt': 0.12,
+        'Au': 0.1, 'Hg': 0.12, 'Tl': 0.12, 'Pb': 0.13,
+        'Bi': 0.13
+    }
+
+    # Replace values below MDL with 5/6 * MDL
+    for element, threshold in MDL.items():
+        if element in df.columns:
+            df.loc[:, element] = df[element].where(df[element] >= threshold, 5 / 6 * threshold)
+
+    return df
+
+
+def normalize_data(df):
+    # Standardize the data (z-score normalization)
+    return DataFrame(StandardScaler().fit_transform(df), index=df.index, columns=df.columns)
+
+
+def remove_outliers(df, threshold=5):
+    # Remove rows where any column value exceeds the threshold
+    return df[(np.abs(df) < threshold)]
+
+
+def smooth_data(df, window=6):
+    # Apply rolling mean to smooth the data
+    return df.rolling(window=window, min_periods=1).mean()
+
+
+def normalize_and_split(df, df2):
+    # Concatenate DataFrames for combined normalization
+    combined_df = concat([df, df2])
+    normalized_combined_df = normalize_data(combined_df)
+
+    # Split the normalized DataFrame back into df and df2
+    df = normalized_combined_df.loc[df.index]
+    df2 = normalized_combined_df.loc[df2.index]
+
+    return df, df2
+
+
 @set_figure(figsize=(12, 3), fs=6)
-def metal_heatmaps(df, major_freq='24h', minor_freq='12h', ax: Axes | None = None, title=None, **kwargs
+def metal_heatmaps(df,
+                   process=True,
+                   major_freq='24h',
+                   minor_freq='12h',
+                   cmap='jet',
+                   ax: Axes | None = None,
+                   **kwargs
                    ) -> tuple[Figure, Axes]:
-    items = ['Al', 'Zr', 'Si', 'Ca', 'Ti', 'Mn', 'Fe', 'V', 'Cl', 'K',
-             'Sr', 'Ba', 'Bi', 'Pd', 'Sn', 'Cr', 'W', 'Cu', 'Zn',
-             'As', 'Co', 'Se', 'Br', 'Cd', 'Sb', 'In', 'Pb', 'Ni']
-
-    df = df[items]
+    if process:
+        df = process_data(df)
 
     fig, ax = plt.subplots(**kwargs.get('fig_kws', {})) if ax is None else (ax.get_figure(), ax)
 
-    sns.heatmap(df.T, vmax=3, cmap="jet", xticklabels=False, yticklabels=True,
-                cbar_kws={'label': 'Z score'})
+    sns.heatmap(df.T, vmin=None, vmax=3, cmap=cmap, xticklabels=False, yticklabels=True,
+                cbar_kws={'label': 'Z score', "pad": 0.02})
     ax.grid(color='gray', linestyle='-', linewidth=0.3)
+
     # Set x-tick positions and labels
     major_tick = date_range(start=df.index[0], end=df.index[-1], freq=major_freq)
     minor_tick = date_range(start=df.index[0], end=df.index[-1], freq=minor_freq)
@@ -47,9 +145,9 @@ def metal_heatmaps(df, major_freq='24h', minor_freq='12h', ax: Axes | None = Non
     ax.set_xticklabels(major_tick.strftime('%F'))
     ax.tick_params(axis='y', rotation=0)
 
-    ax.set_title(f"{title}", fontsize=10)
     ax.set(xlabel='',
            ylabel='',
+           title=kwargs.get('title', None)
            )
 
     plt.show()

AeroViz/plot/timeseries/__init__.py

@@ -1 +1,2 @@
+from .template import *
 from .timeseries import *

AeroViz/plot/timeseries/template.py

@@ -0,0 +1,47 @@
+import matplotlib.pyplot as plt
+from matplotlib.pyplot import Figure, Axes
+from pandas import DataFrame
+
+from AeroViz.plot.timeseries import timeseries
+
+
+def timeseries_template(df: DataFrame) -> tuple[Figure, Axes]:
+    fig, ax = plt.subplots(5, 1, figsize=(len(df.index) * 0.01, 4))
+    (ax1, ax2, ax3, ax4, ax5) = ax
+
+    timeseries(df,
+               y=['Extinction', 'Scattering', 'Absorption'],
+               rolling=30,
+               ax=ax1,
+               ylabel='Coefficient',
+               ylim=[0., None],
+               set_xaxis_visible=False,
+               legend_ncol=3,
+               )
+
+    # Temp, RH
+    timeseries(df,
+               y='AT',
+               y2='RH',
+               rolling=30,
+               ax=ax2,
+               ax_plot_kws=dict(color='r'),
+               ax2_plot_kws=dict(color='b'),
+               ylim=[10, 30],
+               ylim2=[20, 100],
+               set_xaxis_visible=False,
+               legend_ncol=2,
+               )
+
+    timeseries(df, y='WS', color='WD', style='scatter', ax=ax3, scatter_kws=dict(cmap='hsv'),
+               cbar_kws=dict(ticks=[0, 90, 180, 270, 360]),
+               ylim=[0, None], set_xaxis_visible=False)
+
+    timeseries(df, y='VC', color='PBLH', style='bar', ax=ax4, bar_kws=dict(cmap='Blues'), set_xaxis_visible=False,
+               ylim=[0, 5000])
+
+    timeseries(df, y='PM25', color='PM1/PM25', style='scatter', ax=ax5, ylim=[0, None])
+
+    plt.show()
+
+    return fig, ax