nkululeko 0.77.13__py3-none-any.whl → 0.78.0__py3-none-any.whl

nkululeko/constants.py

@@ -1,2 +1,2 @@
-VERSION="0.77.13"
+VERSION="0.78.0"
 SAMPLING_RATE = 16000

nkululeko/experiment.py

@@ -577,6 +577,9 @@ class Experiment:
                 scale_feats,
             )
             self.feats_train, self.feats_test = self.scaler_feats.scale()
+            # store versions
+            self.util.save_to_store(self.feats_train, "feats_train_scaled")
+            self.util.save_to_store(self.feats_test, "feats_test_scaled")
 
     def init_runmanager(self):
         """Initialize the manager object for the runs."""

nkululeko/plots.py

@@ -379,51 +379,115 @@ class Plots:
             )
 
     def scatter_plot(self, feats, label_df, label, dimred_type):
+        dim_num = int(self.util.config_val("EXPL", "scatter.dim", 2))
         fig_dir = self.util.get_path("fig_dir") + "../"  # one up because of the runs
         sample_selection = self.util.config_val("EXPL", "sample_selection", "all")
-        filename = (
-            f"{label}_{self.util.get_feattype_name()}_{sample_selection}_{dimred_type}"
-        )
+        filename = f"{label}_{self.util.get_feattype_name()}_{sample_selection}_{dimred_type}_{str(dim_num)}d"
         filename = f"{fig_dir}{filename}.{self.format}"
         self.util.debug(f"computing {dimred_type}, this might take a while...")
         data = None
         labels = label_df[label]
         if dimred_type == "tsne":
-            data = self.getTsne(feats)
-        elif dimred_type == "umap":
-            import umap
+            data = self.getTsne(feats, dim_num)
+        else:
+            if dimred_type == "umap":
+                import umap
+
+                y = umap.UMAP(
+                    n_neighbors=10,
+                    random_state=0,
+                    n_components=dim_num,
+                ).fit_transform(feats.values)
+            elif dimred_type == "pca":
+                from sklearn.decomposition import PCA
+                from sklearn.preprocessing import StandardScaler
 
-            y_umap = umap.UMAP(
-                n_neighbors=10,
-                random_state=0,
-            ).fit_transform(feats.values)
+                scaler = StandardScaler()
+                pca = PCA(n_components=dim_num)
+                y = pca.fit_transform(scaler.fit_transform(feats.values))
+            else:
+                self.util.error(
+                    f"no such dimensionality reduction function: {dimred_type}"
+                )
+            if dim_num == 2:
+                columns = ["Dim_1", "Dim_2"]
+            elif dim_num == 3:
+                columns = ["Dim_1", "Dim_2", "Dim_3"]
+            else:
+                self.util.error(f"wrong dimension number: {dim_num}")
             data = pd.DataFrame(
-                y_umap,
+                y,
                 feats.index,
-                columns=["Dim_1", "Dim_2"],
+                columns=columns,
             )
-        elif dimred_type == "pca":
-            from sklearn.decomposition import PCA
-            from sklearn.preprocessing import StandardScaler
 
-            scaler = StandardScaler()
-            pca = PCA(n_components=2)
-            y_pca = pca.fit_transform(scaler.fit_transform(feats.values))
-            data = pd.DataFrame(
-                y_pca,
-                feats.index,
-                columns=["Dim_1", "Dim_2"],
+        if dim_num == 2:
+            plot_data = np.vstack((data.T, labels)).T
+            plot_df = pd.DataFrame(data=plot_data, columns=("Dim_1", "Dim_2", "label"))
+            # plt.tight_layout()
+            ax = (
+                sns.FacetGrid(plot_df, hue="label", height=6)
+                .map(plt.scatter, "Dim_1", "Dim_2")
+                .add_legend()
+            )
+        elif dim_num == 3:
+            from mpl_toolkits.mplot3d import Axes3D
+            from sklearn.preprocessing import LabelEncoder
+
+            le = LabelEncoder()
+
+            labels_e = le.fit_transform(labels)
+            plot_data = np.vstack((data.T, labels_e)).T
+            plot_df = pd.DataFrame(
+                data=plot_data, columns=("Dim_1", "Dim_2", "Dim_3", "label")
             )
+            # plt.tight_layout()
+            # axes instance
+            fig = plt.figure(figsize=(6, 6))
+            ax = Axes3D(fig, auto_add_to_figure=False)
+            fig.add_axes(ax)
+            # get colormap from seaborn
+            # cmap = ListedColormap(sns.color_palette("hsv", 256).as_hex())
+            color_dict = {
+                0: "red",
+                1: "blue",
+                2: "green",
+                3: "yellow",
+                4: "purple",
+                5: "#ff69b4",
+                6: "black",
+                7: "cyan",
+                8: "magenta",
+                9: "#faebd7",
+                10: "#2e8b57",
+                11: "#eeefff",
+                12: "#da70d6",
+                13: "#ff7f50",
+                14: "#cd853f",
+                15: "#bc8f8f",
+                16: "#5f9ea0",
+                17: "#daa520",
+            }
+            # plot
+            # make the numbers bigger so they can be used as distinguishable colors
+            labels_ex = [color_dict[xi] for xi in labels_e]
+            sc = ax.scatter(
+                plot_df.Dim_1,
+                plot_df.Dim_2,
+                plot_df.Dim_3,
+                s=40,
+                c=labels_ex,
+                marker="o",
+                # cmap=cmap,
+                alpha=1,
+            )
+            ax.set_xlabel("Dim_1")
+            ax.set_ylabel("Dim_2")
+            ax.set_zlabel("Dim_3")
+            # legend
+            plt.legend(*sc.legend_elements(), bbox_to_anchor=(1.05, 1), loc=2)
         else:
-            self.util.error(f"no such dimensionality reduction function: {dimred_type}")
-        plot_data = np.vstack((data.T, labels)).T
-        plot_df = pd.DataFrame(data=plot_data, columns=("Dim_1", "Dim_2", "label"))
-        # plt.tight_layout()
-        ax = (
-            sns.FacetGrid(plot_df, hue="label", height=6)
-            .map(plt.scatter, "Dim_1", "Dim_2")
-            .add_legend()
-        )
+            self.util.error(f"wrong dimension number: {dim_num}")
         fig = ax.figure
         plt.savefig(filename)
         fig.clear()
@@ -437,35 +501,10 @@ class Plots:
             )
         )
 
-    def plotTsne(self, feats, labels, filename, perplexity=30, learning_rate=200):
-        """Make a TSNE plot to see whether features are useful for classification"""
-        fig_dir = self.util.get_path("fig_dir") + "../"  # one up because of the runs
-        filename = f"{fig_dir}{filename}.{self.format}"
-        self.util.debug(f"plotting tsne to {filename}, this might take a while...")
-        model = TSNE(
-            n_components=2,
-            random_state=0,
-            perplexity=perplexity,
-            learning_rate=learning_rate,
-        )
-        tsne_data = model.fit_transform(feats)
-        tsne_data_labs = np.vstack((tsne_data.T, labels)).T
-        tsne_df = pd.DataFrame(data=tsne_data_labs, columns=("Dim_1", "Dim_2", "label"))
-        # plt.tight_layout()
-        ax = (
-            sns.FacetGrid(tsne_df, hue="label", height=6)
-            .map(plt.scatter, "Dim_1", "Dim_2")
-            .add_legend()
-        )
-        fig = ax.figure
-        plt.savefig(filename)
-        fig.clear()
-        plt.close(fig)
-
-    def getTsne(self, feats, perplexity=30, learning_rate=200):
+    def getTsne(self, feats, dim_num, perplexity=30, learning_rate=200):
         """Make a TSNE plot to see whether features are useful for classification"""
         model = TSNE(
-            n_components=2,
+            n_components=dim_num,
             random_state=0,
             perplexity=perplexity,
             learning_rate=learning_rate,

nkululeko/scaler.py

@@ -95,10 +95,12 @@ class Scaler:
             b2 = np.quantile(self.feats_train[c], 0.66)
             feats_bin_train[c] = self._bin(self.feats_train[c].values, b1, b2).values
             feats_bin_test[c] = self._bin(self.feats_test[c].values, b1, b2).values
+        self.feats_train = feats_bin_train
+        self.feats_test = feats_bin_test
 
     def _bin(self, series, b1, b2):
         bins = [-1000000, b1, b2, 1000000]
-        labels = ["low", "middle", "high"]
+        labels = [0, 0.5, 1]
         result = np.digitize(series, bins) - 1
         result = pd.Series(result)
         for i, l in enumerate(labels):

nkululeko/utils/util.py

@@ -307,7 +307,13 @@ class Util:
         elif format == "csv":
             return audformat.utils.read_csv(name)
         else:
-            self.error(f"unkown store format: {format}")
+            self.error(f"unknown store format: {format}")
+
+    def save_to_store(self, df, name):
+        store = self.get_path("store")
+        store_format = self.config_val("FEATS", "store_format", "pkl")
+        storage = f"{store}{name}.{store_format}"
+        self.write_store(df, storage, store_format)
 
     def copy_flags(self, df_source, df_target):
         if hasattr(df_source, "is_labeled"):

{nkululeko-0.77.13.dist-info → nkululeko-0.78.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nkululeko
-Version: 0.77.13
+Version: 0.78.0
 Summary: Machine learning audio prediction experiments based on templates
 Home-page: https://github.com/felixbur/nkululeko
 Author: Felix Burkhardt
@@ -313,6 +313,15 @@ F. Burkhardt, Johannes Wagner, Hagen Wierstorf, Florian Eyben and Björn Schulle
 Changelog
 =========
 
+Version 0.78.0
+--------------
+* added 3-d scatter plots
+* removed epoch-plots if epoch_num=1
+
+Version 0.77.14
+--------------
+* fixed bug preventing bin scaling to work
+
 Version 0.77.13
 --------------
 * added bins scaler

{nkululeko-0.77.13.dist-info → nkululeko-0.78.0.dist-info}/RECORD

@@ -2,10 +2,10 @@ nkululeko/__init__.py,sha256=62f8HiEzJ8rG2QlTFJXUCMpvuH3fKI33DoJSj33mscc,63
 nkululeko/aug_train.py,sha256=uUCqtAGhT6P1dyrPpwT409wEhxfzsxpCiq_v_zdmfOI,3103
 nkululeko/augment.py,sha256=sHWG4Jmb4BjnaaXXnRYMP7Jkk0qlaZ2ohsArP8uW_d8,3003
 nkululeko/cacheddataset.py,sha256=lIJ6hUo5LoxSrzXtWV8mzwO7wRtUETWnOQ4ws2XfL1E,969
-nkululeko/constants.py,sha256=3c07uj-DuO1BAfszagYr4S2y4QcHzfmPYXiRjCaZ3C4,40
+nkululeko/constants.py,sha256=svWlKgLGI6BGzQ_UC7BqFvjh7pXevsQxvV9VqPg-n2s,39
 nkululeko/demo.py,sha256=FFR8qHMCY8kKKRWDTa8xA7A8mWeYalRKYNtV5rjGg44,1915
 nkululeko/demo_predictor.py,sha256=j4ySWWcIxW7ZAIBH86m9BfRFokzrUNkRD6fpsvAQGTw,2629
-nkululeko/experiment.py,sha256=g-GdcvcWyQKj70VutyFPuzSxmsD-Tk_-vY-5FM7dszs,28644
+nkululeko/experiment.py,sha256=znrMTpGqNeBQWD0F7sUHzQyRjCxsRH9t3V_8NhAtzrg,28823
 nkululeko/explore.py,sha256=pfciOL66e0T4Bk0RTkwDyE6pK_baSUdjMo2Ybpst3L4,2202
 nkululeko/export.py,sha256=mHeEAAmtZuxdyebLlbSzPrHSi9OMgJHbk35d3DTxRBc,4632
 nkululeko/feature_extractor.py,sha256=tKv1b1-o7xNMgBavTR8VY8_H5HKoJEnnosS-KcjmOEU,7281
@@ -15,13 +15,13 @@ nkululeko/glob_conf.py,sha256=6MZe83QCgHD_zB79Sl9m6UJQASwfqJlyb-1nqrQ_6Iw,381
 nkululeko/modelrunner.py,sha256=TQW08f72-GjBEIFTE3_8B8qMCWvTJUqJ1fveygmYnXI,9278
 nkululeko/multidb.py,sha256=5F62YhUKwi__a_fhcovyOCOcydTH48nM3JJ3oZ47Tjg,3852
 nkululeko/nkululeko.py,sha256=Rm6L9iQx7KWO7jJW1pdwMgYfRruapfSoYE9TE-MaD2s,1851
-nkululeko/plots.py,sha256=tW9VlhD1K6FRM15rtYYJiHO7YFhbJpOXVy8SHu7UrqI,21687
+nkululeko/plots.py,sha256=K88ZRPFGX_r03BT742H06Dde20xZYdltv7dxjgUiAFA,23025
 nkululeko/predict.py,sha256=dRXX-sQVESa7cNi_56S6UkUOa_pV1g_K4xYtYVM1SJs,1876
 nkululeko/reporter.py,sha256=wrtWbU_UKDwhKQNMi7Q_Ix5N_UTzTagRwZikgUGk1c8,11606
 nkululeko/resample.py,sha256=C2S3aOTwlx5iYah_hs0JARHBC8Cq4Z5cH_mnDMb5RKk,2185
 nkululeko/result.py,sha256=HeiOrrqf9W2yxMryN33zgEmQejNWRxNmm589AYt6-eM,499
 nkululeko/runmanager.py,sha256=WoZ4cRyHV7kAhtt-55RAAj6D3iXqEvvEMm6Ll4YkV2E,7192
-nkululeko/scaler.py,sha256=gA4ziPMx_qIubtk97v-vrqzlP54tbXvxpna5g63GQCg,4031
+nkululeko/scaler.py,sha256=4nkIqoajkIkuTPK0Z02ifMN_awl6fP_i-GBYdoGYgGM,4101
 nkululeko/segment.py,sha256=DfJYZsCEH41gwKyjpMgv8kWUzfVkmC0wWTbgHOL4i4g,4787
 nkululeko/syllable_nuclei.py,sha256=Sky-C__MeUDaxqHnDl2TGLLYOYvsahD35TUjWGeG31k,10047
 nkululeko/test.py,sha256=cRtOn_d3Fh2kZmnT4nnQeGzZTRtpr5jRhowykOwunME,1421
@@ -99,9 +99,9 @@ nkululeko/segmenting/seg_silero.py,sha256=lLytS38KzARS17omwv8VBw-zz60RVSXGSvZ5Ev
 nkululeko/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nkululeko/utils/files.py,sha256=UiGAtZRWYjHSvlmPaTMtzyNNGE6qaLaxQkybctS7iRM,4021
 nkululeko/utils/stats.py,sha256=29otJpUp1VqbtDKmlLkPPzBmVfTFiHZ70rUdR4860rM,2788
-nkululeko/utils/util.py,sha256=xYLHCWcKYF-DlSYYDGHd9EXnvZPB_bYdE6BXCq5Twgo,11985
-nkululeko-0.77.13.dist-info/LICENSE,sha256=0zGP5B_W35yAcGfHPS18Q2B8UhvLRY3dQq1MhpsJU_U,1076
-nkululeko-0.77.13.dist-info/METADATA,sha256=CU3Dp3UQziQ5j4cYGbwG7lRhe_Acfj7ThrYlSY04lG0,31461
-nkululeko-0.77.13.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-nkululeko-0.77.13.dist-info/top_level.txt,sha256=DPFNNSHPjUeVKj44dVANAjuVGRCC3MusJ08lc2a8xFA,10
-nkululeko-0.77.13.dist-info/RECORD,,
+nkululeko/utils/util.py,sha256=Ibwco798iDhmamO7SjVStfxOl-DNBdzkKl2HniYgoNE,12238
+nkululeko-0.78.0.dist-info/LICENSE,sha256=0zGP5B_W35yAcGfHPS18Q2B8UhvLRY3dQq1MhpsJU_U,1076
+nkululeko-0.78.0.dist-info/METADATA,sha256=Vq5Rc0-yJHX1IDRVElWFgoTqCwVyqfS6QSXoVKMJeOI,31629
+nkululeko-0.78.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+nkululeko-0.78.0.dist-info/top_level.txt,sha256=DPFNNSHPjUeVKj44dVANAjuVGRCC3MusJ08lc2a8xFA,10
+nkululeko-0.78.0.dist-info/RECORD,,