geocif 0.1.32__tar.gz → 0.1.34__tar.gz

{geocif-0.1.32/geocif.egg-info → geocif-0.1.34}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: geocif
-Version: 0.1.32
+Version: 0.1.34
 Summary: Models to visualize and forecast crop conditions and yields
 Home-page: https://ritviksahajpal.github.io/yield_forecasting/
 Author: Ritvik Sahajpal

{geocif-0.1.32 → geocif-0.1.34}/geocif/analysis.py

@@ -162,8 +162,8 @@ class Geoanalysis:
             return pd.DataFrame(), pd.DataFrame()
 
         df_metrics = self._compute_metrics(df)
-        df_metrics = self._process_metrics(df_metrics)
-        self._plot_metrics(df_metrics)
+        #df_metrics = self._process_metrics(df_metrics)
+        #self._plot_metrics(df_metrics)
 
         df_regional_metrics_by_year = self._compute_regional_metrics(
             df, by="Harvest Year"
@@ -172,9 +172,9 @@ class Geoanalysis:
             df_regional_metrics_by_year
         )
         df_regional_metrics = self._average_mape(df_regional_metrics_by_year)
-
+        breakpoint()
         self._store_results(
-            df_metrics, df_regional_metrics, df_regional_metrics_by_year
+            None, df_regional_metrics, df_regional_metrics_by_year
         )
 
         df_national_yield = self._compute_national_yield(df)
@@ -195,7 +195,7 @@ class Geoanalysis:
             .apply(self.annual_metrics)
             .reset_index()
         )
-
+        breakpoint()
         return df_metrics.pivot_table(
             index=["Country", "Model", "Harvest Year", "Stage Name", "Stage Range"],
             columns="level_5",

{geocif-0.1.32 → geocif-0.1.34}/geocif/cei/indices.py

@@ -465,7 +465,7 @@ class CEIs:
 
         extended_stages_list = []
         if self.method in ["phenological_stages", "fraction_season", "full_season"]:
-            extended_stages_list = [stages]
+            extended_stages_list = stages
         elif self.method in ["dekad_r", "biweekly_r", "monthly_r"]:
             # reverse stages
             stages = stages[::-1]
@@ -566,10 +566,10 @@ class CEIs:
 
         """
         if self.method in ["phenological_stages", "fraction_season"]:
-            mask = df_harvest_year_region[col].isin(stages)
+            mask = df_harvest_year_region[col].isin([stages])
             df_time_period = df_harvest_year_region[mask]
 
-            mask = df_all_years[col].isin(stages)
+            mask = df_all_years[col].isin([stages])
             df_base_period = df_all_years[mask]
         elif self.method in [
             "dekad",
@@ -605,6 +605,10 @@ class CEIs:
         Returns:
 
         """
+        # If stage is not a list then convert it to a list
+        if not isinstance(stage, list):
+            stage = [stage]
+
         columns = [
             "Description",
             "CEI",
@@ -721,6 +725,10 @@ class CEIs:
         :param index_details:
         :return:
         """
+        # If stage is not a list then convert it to a list
+        if not isinstance(stage, list):
+            stage = [stage]
+
         df = df[df["bounds"] == 1]
         # Exclude lat, lon, time, bounds and time_bounds columns
         df = df.drop(columns=["lat", "lon", "time", "bounds", "time_bounds"])

{geocif-0.1.32 → geocif-0.1.34}/geocif/geocif.py

@@ -222,6 +222,10 @@ class Geocif:
             self.logger.info(f"Selected features: {self.selected_features}")
 
             """ Update model to include conformal estimates """
+            if "lat" not in self.selected_features:
+                self.selected_features.append("lat")
+            if "lon" not in self.selected_features:
+                self.selected_features.append("lon")
             X_train = df_region[self.selected_features + self.cat_features]
             dir_output = (
                 self.dir_analysis
@@ -275,6 +279,12 @@ class Geocif:
                         verbose=False,
                         # callbacks=[TQDMCallback(self.best_hyperparams["iterations"])],
                     )
+                elif self.model_name == "geospaNN":
+                    self.model.fit(
+                        X_train,
+                        y_train,
+                        # callbacks=[TQDMCallback(self.best_hyperparams["iterations"])],
+                    )
                 elif self.model_name == "merf":
                     Z_train = np.ones((len(X_train), 1))
                     clusters_train = df_region["Region"]
@@ -341,6 +351,25 @@ class Geocif:
                     X_test, Z_test, clusters_test.astype("object")
                 )
                 best_hyperparameters = self.model.fe_model.get_params().copy()
+            elif self.model_name == "geospaNN":
+                import torch
+                import geospaNN
+
+                # Remove any categorical features
+                X_test = X_test.drop(columns=self.cat_features)
+                X = torch.from_numpy(X_test.to_numpy()).float()
+                coord = torch.from_numpy(self.df_test[['lon', 'lat']].to_numpy()).float()
+
+                p = X.shape[1]
+                n = X.shape[0]
+                nn = 5
+
+                data = geospaNN.make_graph(X, Y, coord, nn)
+
+                # remove categorical features from df_train
+                data_train = df_region[self.selected_features + self.cat_features + [self.target]]
+                w_train = data_train.y - self.estimate(data_train.x)
+
             else:
                 y_pred = self.model.predict(X_test)
                 best_hyperparameters = self.model.get_params().copy()
@@ -458,9 +487,10 @@ class Geocif:
             "Harvest Year",
             "Stage Name",
         ]
-        df.index = df.apply(
-            lambda row: "_".join([str(row[col]) for col in index_columns]), axis=1
-        )
+        try:
+            df.index = df.apply(lambda row: "_".join([str(row[col]) for col in index_columns]), axis=1)
+        except Exception as e:
+            breakpoint()
 
         # name the index level
         df.index.set_names(["Index"], inplace=True)
@@ -527,6 +557,10 @@ class Geocif:
         if self.use_outlook_as_feature:
             self.feature_names.append("FCST")
 
+        # Add lat and lon to feature names
+        self.feature_names.append("lat")
+        self.feature_names.append("lon")
+
         self.selected_features = []
 
     def loop_ml(self, stages, dict_selected_features, dict_best_cei):
@@ -782,6 +816,13 @@ class Geocif:
             how="outer",
         )
 
+        # Add a lat and lon column to self.dg_country
+        self.dg_country["lat"] = self.dg_country.centroid.y
+        self.dg_country["lon"] = self.dg_country.centroid.x
+
+        # Add lat and lon columns to df by merging on Country Region column
+        df = df.merge(self.dg_country[["Country Region", "lat", "lon"]].drop_duplicates(), on="Country Region", how="left")
+
         dict_kwargs = {}
         dict_kwargs["all_stages"] = self.all_stages
         dict_kwargs["target_col"] = self.target

{geocif-0.1.32 → geocif-0.1.34}/geocif/indices_runner.py

@@ -165,11 +165,12 @@ class cei_runner(base.BaseGeo):
         combinations = [
             i
             for i in combinations
-            if "angola_maize" in i[3] or "lesotho_maize" in i[3] or
-            #                 "namibia" in i[2] or
-            #                 "united_republic_of_tanzania" in i[2] or
+            if "angola_maize" in i[3] or
+               "lesotho_maize" in i[3] or
+            #   "namibia_" in i[2] or
+                "united_republic_of_tanzania_maize" in i[3] or
             "zambia_maize" in i[3] or "zimbabwe_maize" in i[3] or
-            #                 "south_africa" in i[2] or
+            "south_africa_maize" in i[3] or
             "mozambique_maize" in i[3]
         ]
         #                 "malawi" in i[2]]

geocif-0.1.34/geocif/indices_runner_v2.py

@@ -0,0 +1,208 @@
+import itertools
+import warnings
+from multiprocessing import Pool, cpu_count
+from pathlib import Path
+
+import arrow as ar
+import pandas as pd
+from tqdm import tqdm
+
+warnings.filterwarnings("ignore")
+
+from .cei import indices
+from geoprepare import base
+
+
+def remove_duplicates(lst):
+    """
+
+    :param lst:
+    :return:
+    """
+    return list(set([i for i in lst]))
+
+
+def get_admin_zone(country, dg_shp):
+    admin_zone = "admin_1"
+    country = country.title().replace(" ", "_")
+
+    # Read in shapefile
+    dg_country = dg_shp[dg_shp["ADMIN0"] == country]
+
+    # Is the ADMIN2 column all None? If so, return admin_1 else return admin_2
+    if dg_country.empty:
+        admin_zone = "admin_1"
+    elif not dg_country["ADMIN2"].isna().all():
+        admin_zone = "admin_2"
+
+    return admin_zone
+
+
+class cei_runner(base.BaseGeo):
+    def __init__(self, path_config_file):
+        super().__init__(path_config_file)
+
+        # Parse configuration files
+        self.parse_config()
+
+        self.dir_input = Path(self.parser.get("PATHS", "dir_input"))
+        self.base_dir = Path(self.parser.get("PATHS", "dir_crop_inputs"))
+        self.do_parallel = self.parser.getboolean("DEFAULT", "do_parallel")
+
+    def collect_files(self):
+        """
+        1. Collect all the files which contain EO information
+        2. Exclude files from the `processed` directory if it is already in
+        processed_include_fall directory
+        3. Create a dataframe that contains the following columns:
+            - directory: name of directory where file is located
+            - path: full path to file
+            - filename: name of file
+        :return: Return the dataframe created above
+        """
+        import geopandas as gp
+
+        dg_shp = gp.read_file(
+            self.dir_input
+            / "Global_Datasets"
+            / "Regions"
+            / "Shps"
+            / "adm_shapefile.shp",
+            engine="pyogrio",
+        )
+
+        # Collect all the files which contain EO information
+        df_files = pd.DataFrame(columns=["directory", "path", "filename", "admin_zone"])
+        for filepath in self.base_dir.rglob("*.csv"):
+            country = filepath.parents[0].name
+
+            admin_zone = get_admin_zone(country, dg_shp)
+
+            # If country is not in cc.COUNTRIES then skip
+            # HACK: Skip korea for now, as it is giving errors
+            if country == "republic_of_korea":
+                continue
+
+            # Get name of directory one level up
+            process_type = filepath.parents[1].name
+
+            # Get name of file
+            filename = filepath.name
+
+            # Add to dataframe
+            df_files.loc[len(df_files)] = [process_type, filepath, filename, admin_zone]
+
+        # Exclude those rows where directory is processed and file is already in
+        # processed_include_fall directory
+        no_fall = df_files["directory"] == "processed"
+        include_fall = df_files[df_files["directory"] == "processed_include_fall"][
+            "filename"
+        ]
+
+        df_files = df_files[~(no_fall & (df_files["filename"].isin(include_fall)))]
+
+        return df_files
+
+    def process_combinations(self, df, method):
+        """
+        Create a list of tuples of the following:
+            - directory: name of directory where file is located
+            - path: full path to file
+            - filename: name of file
+            - method: whether to compute indices for phenological stages or not
+        This tuple will be used as input to the `process` function
+        :param df:
+        :param method:
+        :return:
+        """
+        combinations = []
+
+        for index, row in tqdm(df.iterrows()):
+            combinations.extend(
+                list(
+                    itertools.product([row[0]], [row[1]], [row[2]], [row[3]], [method])
+                )
+            )
+
+        combinations = remove_duplicates(combinations)
+
+        return combinations
+
+    def main(self, method):
+        """
+
+        :param method:
+        :return:
+        """
+        # Create a dataframe of the files to be analyzed
+        df_files = self.collect_files()
+
+        combinations = self.process_combinations(df_files, method)
+
+        # Add an element to the tuple to indicate the season
+        # Last element is redo flag which is True if the analysis is to be redone
+        # and False otherwise. Analysis is always redone for the current year
+        # and last year whether file exists or not
+        combinations = [
+            (
+                self.parser,
+                status,
+                path,
+                filename,
+                admin_zone,
+                category,
+                year,
+                "ndvi",
+                False,  # redo
+            )
+            for year in range(2001, ar.utcnow().year + 1)
+            for status, path, filename, admin_zone, category in combinations
+        ]
+
+        # Only keep those entries in combinations where the third elemt is
+        # mozambique, south_africa, angola or dem_people's_rep_of_korea
+        # This is done to test the code for these countries
+        #combinations = [
+        #    i
+        #    for i in combinations
+        #    if "ethiopia_maize_s1" in i[3]
+        #]
+        #                 "malawi" in i[2]]
+
+        if True:
+            num_cpu = int(cpu_count() * 0.8)
+            with Pool(num_cpu) as p:
+                for i, _ in enumerate(p.imap_unordered(indices.process, combinations)):
+                    pass
+        else:
+            # Use the code below if you want to test without parallelization or
+            # if you want to debug by using pdb
+            pbar = tqdm(combinations)
+            for i, val in enumerate(pbar):
+                pbar.set_description(
+                    f"Main loop {combinations[i][2]} {combinations[i][5]}"
+                )
+                indices.process(val)
+
+
+def run(path_config_files=[]):
+    """
+
+    Args:
+        path_config_files:
+
+    Returns:
+
+    """
+    """ Check dictionary keys to have no spaces"""
+    indices.validate_index_definitions()
+
+    for method in [
+        "phenological_stages",  # "dekad_r"  # "dekad_r"
+    ]:  # , "full_season", "phenological_stages", "fraction_season"]:
+        obj = cei_runner(path_config_files)
+        obj.main(method)
+
+
+if __name__ == "__main__":
+    run()

{geocif-0.1.32 → geocif-0.1.34}/geocif/ml/correlations.py

@@ -260,6 +260,9 @@ def all_correlated_feature_by_time(df, **kwargs):
         ):
             df_corr = _all_correlated_feature_by_time(group, **kwargs)
 
+            # Remove columns with more than 50% NaN values
+            df_corr = df_corr.dropna(thresh=len(df_corr) / 2, axis=1)
+
             if not df_corr.empty:
                 df_tmp = df_corr[df_corr.columns[(df_corr.mean() > 0.1)]]
                 dict_selected_features[region_id] = df_tmp.columns

geocif-0.1.34/geocif/ml/correlations_backup.py

@@ -0,0 +1,412 @@
+import os
+
+import matplotlib.pyplot as plt
+import palettable as pal
+import pandas as pd
+import seaborn as sns
+from tqdm import tqdm
+
+from geocif import utils
+from geocif.ml import embedding
+from geocif.ml import stages
+
+
+def most_correlated_feature_by_time(df_train, simulation_stages, target_col):
+    """
+
+    Args:
+        df_train:
+        simulation_stages:
+        target_col:
+
+    Returns:
+
+    """
+    frames = []
+
+    stages = [simulation_stages[: idx + 1] for idx in range(len(simulation_stages))]
+
+    # Only select columns that have been observed till the current stage
+    for stage in tqdm(stages, leave=False, desc="Compute most correlated feature"):
+        current_feature_set = [
+            col for col in df_train.columns if col.endswith(f"_{stage[-1]}")
+        ]
+
+        # Get the most correlated feature for each region
+        top_feature_by_region, counter = embedding.get_top_correlated_features(
+            df_train[current_feature_set + ["Region"]],
+            df_train[target_col],
+        )
+
+        # Create a dataframe with the most common top feature and number of occurrences over timestep
+        _feature = counter.most_common(1)[0][0]
+        # Loop through top_feature_by_region and find the average score for _feature
+        # Calculate the average score for 'DTR_36'
+        _feature_scores = [
+            value[1][0]
+            for key, value in top_feature_by_region.items()
+            if _feature in value[0]
+        ]
+        average_score = sum(_feature_scores) / len(_feature_scores)
+        _feature = utils.remove_last_part(_feature)
+
+        df = pd.DataFrame(
+            {
+                "Stage": [stage[-1]],
+                "Date": [utils.dict_growth_stages[stage[-1]]],
+                "Feature with Highest Correlation": [counter.most_common(1)[0][0]],
+                "Feature Category": [_feature],
+                "Score": [average_score],
+                # "Type": [ci.dict_indices[_feature][0]],
+                "Number of Occurrences": [counter.most_common(1)[0][1]],
+                # "Current Feature Set": [current_feature_set],
+            }
+        )
+        frames.append(df)
+
+    df_most_corr_feature_by_time = pd.concat(frames)
+
+
+def plot_feature_corr_by_time(df, **kwargs):
+    country = kwargs.get("country")
+    crop = kwargs.get("crop")
+    dir_output = kwargs.get("dir_output")
+    forecast_season = kwargs.get("forecast_season")
+    national_correlation = kwargs.get("national_correlation")
+    group_by = kwargs.get("groupby")
+
+    # Setup the figure and gridspec
+    fig = plt.figure(figsize=(10, 5))
+    gs = fig.add_gridspec(
+        3, 2, height_ratios=[6, 5, 1], width_ratios=[5, 1.5], hspace=0.6, wspace=0.0
+    )
+
+    # Assign subplots
+    ax_heatmap = fig.add_subplot(gs[0:2, 0])
+    ax_map = fig.add_subplot(gs[0, 1])
+    cbar_ax = fig.add_subplot(gs[2, 0])
+    ax4 = fig.add_subplot(gs[2, 1])
+
+    # Transpose and reverse the columns of the dataframe
+    #breakpoint()
+    ## Only select foll. columns:
+
+    df = df[
+        [
+            "TG",
+            "TG10p",
+            "DTR",
+            "vDTR",
+            "R99p",
+            "RX5day",
+            "MEAN_ESI4WK",
+        ]
+    ]
+    df_transpose = df.T
+    df = df_transpose[df_transpose.columns[::-1]]
+
+    # Split column names and only use value before space
+    df.columns = df.columns.str.split(" ").str[0]
+    # In row names, replace ESI4WK by ES
+    df.index = df.index.str.replace("MEAN_ESI4WK", "ZScore_ES")
+    df.index = df.index.str.replace("R99p", "MEAN_SM")
+    df.index = df.index.str.replace("RX5day", "AUC_SM")
+    # Remove the last row
+    # Select the first, third and fifth column
+    df = df[["Dec", "Feb", "Apr"]]
+    # Rename Dec to Planting - Early Vegetative
+    # Rename Feb to Early Vegetative - Senescence
+    # Rename Apr to Senescence - Harvest
+    df.columns = ["Planting - Early Vegetative", "Early Vegetative - Senescence", "Senescence - Harvest"]
+    ax_heatmap = sns.heatmap(
+        df,
+        ax=ax_heatmap,
+        annot=True,
+        cmap=pal.cartocolors.diverging.Earth_5.get_mpl_colormap(),
+        fmt=".2f",
+        square=False,
+        linewidths=0.5,
+        linecolor="white",
+        cbar_ax=cbar_ax,
+        cbar_kws={"orientation": "horizontal"},  # , "shrink": 0.5},
+        annot_kws={"size": 6},
+        xticklabels=True,
+        yticklabels=True,
+    )
+    ax_heatmap.tick_params(left=False, bottom=False)
+
+    # Plot the map using GeoPandas
+    dg_country = kwargs.get("dg_country")
+
+    ax_map = dg_country.plot(
+        ax=ax_map,
+        color="white",
+        edgecolor="black",
+        linewidth=1.0,
+        facecolor=None,
+        legend=False,
+    )
+
+    if not national_correlation:
+        id = kwargs["region_id"]
+        dg_region = dg_country[dg_country[group_by] == id]
+        ax_map = dg_region.plot(
+            ax=ax_map, color="blue", edgecolor="blue", linewidth=1.0, legend=False
+        )
+        # Set title with color blue
+        ax_map.set_title(f"Region: {id}", color="blue")
+
+    # No colorbar for the map
+    ax_map.axis("off")
+    # Remove borders
+    ax_map.spines["top"].set_visible(False)
+    ax_map.spines["right"].set_visible(False)
+    ax_map.spines["bottom"].set_visible(False)
+    ax_map.spines["left"].set_visible(False)
+    # ax4 should not be visible
+    ax4.axis("off")
+
+    # Add colorbar label
+    # cbar_ax.set_xlabel("Correlation Coefficient", labelpad=3, size="small")
+    cbar_ax.set_title("Correlation Coefficient", loc="left", size="small")
+    ax_heatmap.set_xticklabels(
+        ax_heatmap.get_xticklabels(), size="x-small", rotation=0, fontsize=7
+    )
+    ax_heatmap.set_yticklabels(ax_heatmap.get_yticklabels(), size="x-small", fontsize=7)
+    ax_heatmap.set_xlabel("")
+    ax_heatmap.set_ylabel(" ")
+    # Reduce font size of ticks of colorbar
+    cbar_ax.tick_params(axis="both", which="major", labelsize=6)
+
+    _country = country.title().replace("_", " ")
+    _crop = crop.title().replace("_", " ")
+    if not national_correlation:
+        fname = f"{country}_{crop}_{id}_corr_feature_by_time.png"
+    else:
+        fname = f"{country}_{crop}_corr_feature_by_time.png"
+    ax_heatmap.set_title(f"{_country}\n{_crop}")
+
+    # plt.tight_layout()
+    os.makedirs(dir_output, exist_ok=True)
+    plt.savefig(dir_output / fname, dpi=250)
+    plt.close()
+
+
+def _all_correlated_feature_by_time(df, **kwargs):
+    """
+
+    Args:
+        df:
+        **kwargs:
+
+    Returns:
+
+    """
+    frames = []
+    all_stages = kwargs.get("all_stages")
+    target_col = kwargs.get("target_col")
+    method = kwargs.get("method")
+
+    longest_stage = max(all_stages, key=len)
+
+    # Split the original string into a list of its parts
+    longest_stage = longest_stage.split("_")
+
+    # Generate the list of strings as described by the user, removing one element from the start each time
+    stages_features = ["_".join(longest_stage[i:]) for i in range(len(longest_stage))]
+
+    # Drop columns with no yield information
+    df = df.dropna(subset=[target_col])
+
+    # Only select columns that have been observed till the current stage
+    pbar = tqdm(stages_features, total=len(stages_features), leave=False)
+    for stage in pbar:
+        pbar.set_description(f"Calculating correlations")
+        pbar.update()
+
+        stage_name = stages.get_stage_information_dict(f"GD4_{stage}", method)[
+            "Stage Name"
+        ]
+        # starting_stage = stage_name.split("-")[0]
+        current_feature_set = [col for col in df.columns if stage_name in col]
+
+        # Get the most correlated feature for each region
+        df_tmp = embedding.get_all_features_correlation(
+            df[current_feature_set + ["Region"]], df[target_col], method
+        )
+
+        frames.append(df_tmp)
+
+    df_results = pd.concat(frames)
+    if not df_results.empty:
+        # Exclude Region column
+        df_results = df_results.drop(columns="Region")
+        # Groupby Dekad and compute mean of all columns apart from Region
+        df_results = df_results.groupby(method).mean()
+
+        all_stage_names = []
+        for stage in stages_features:
+            _tmp = stages.get_stage_information_dict(f"GD4_{stage}", method)[
+                "Stage Name"
+            ]
+            all_stage_names.append(_tmp)
+
+        df_results = df_results.reindex(all_stage_names)
+
+        # Drop rows with all NaN values
+        df_results = df_results.dropna(how="all")
+
+        # Split the index based on - and only keep the first element
+        df_results.index = df_results.index.str.split("-").str[0]
+
+        return df_results
+    else:
+        return pd.DataFrame()
+
+
+def all_correlated_feature_by_time(df, **kwargs):
+    """
+
+    Args:
+        df:
+        **kwargs:
+
+    Returns:
+
+    """
+    THRESHOLD = 0.1
+    national_correlation = kwargs.get("national_correlation")
+    group_by = kwargs.get("groupby")
+    combined_dict = kwargs.get("combined_dict")
+
+    dict_selected_features = {}
+    dict_best_cei = {}
+
+    if not national_correlation:
+        groups = df.groupby(group_by)
+        for region_id, group in tqdm(
+            groups, desc=f"Compute all correlated feature by {group_by}", leave=False
+        ):
+            df_corr = _all_correlated_feature_by_time(group, **kwargs)
+
+            # Remove columns with more than 50% NaN values
+            df_corr = df_corr.dropna(thresh=len(df_corr) / 2, axis=1)
+
+            if not df_corr.empty:
+                df_tmp = df_corr[df_corr.columns[(abs(df_corr.mean()) > THRESHOLD)]]
+                # Add the columns to dict_selected_features along with the absolute mean value
+                absolute_medians = df_tmp.abs().median()
+
+                # Create a DataFrame to display the column names and their absolute median values
+                absolute_median_df = absolute_medians.reset_index()
+                absolute_median_df.columns = ['CEI', 'Median']
+
+                # Add the CEI and Median value to dict_selected_features
+                dict_selected_features[region_id] = absolute_median_df
+
+                df_tmp2 = (
+                    df_tmp.median(axis=0)
+                    .abs()
+                    .sort_values(ascending=False)
+                    .reset_index()
+                )
+                df_tmp2.columns = ["Metric", "Value"]
+                # Add another column based on Type of Metric
+                for idx, row in df_tmp2.iterrows():
+                    df_tmp2.loc[idx, "Type"] = combined_dict[row[0]][0]
+
+                # Compute median of each CEI and sort the dataframe based on the absolute value of the median
+                dict_best_cei[region_id] = (
+                    df_tmp2.groupby("Type")
+                    .max()
+                    .reset_index()
+                    .sort_values("Value", ascending=False)["Metric"]
+                    .values
+                )
+
+                kwargs["region_id"] = region_id
+                plot_feature_corr_by_time(df_tmp, **kwargs)
+                # For each element in dict_best_cei, add the type of the cei
+            else:
+                # HACK
+                df_corr = _all_correlated_feature_by_time(df, **kwargs)
+
+                df_tmp = df_corr[df_corr.columns[(abs(df_corr.mean()) > THRESHOLD)]]
+                # Add the columns to dict_selected_features along with the absolute mean value
+                absolute_medians = df_tmp.abs().median()
+
+                # Create a DataFrame to display the column names and their absolute median values
+                absolute_median_df = absolute_medians.reset_index()
+                absolute_median_df.columns = ['CEI', 'Median']
+
+                # Add the CEI and Median value to dict_selected_features
+                dict_selected_features[region_id] = absolute_median_df
+                dict_best_cei[region_id] = {}
+    else:
+        df_corr = _all_correlated_feature_by_time(df, **kwargs)
+        df_tmp = df_corr[df_corr.columns[(abs(df_corr.mean()) > THRESHOLD)]]
+        # Add the columns to dict_selected_features along with the absolute mean value
+        absolute_medians = df_tmp.abs().median()
+
+        # Create a DataFrame to display the column names and their absolute median values
+        absolute_median_df = absolute_medians.reset_index()
+        absolute_median_df.columns = ['CEI', 'Median']
+
+        # Add the CEI and Median value to dict_selected_features
+        dict_selected_features[0] = absolute_median_df
+
+        plot_feature_corr_by_time(df_corr, **kwargs)
+
+    return dict_selected_features, dict_best_cei
+
+
+def feature_correlation_by_time(**kwargs):
+    raise NotImplementedError()
+
+    frames = []
+    simulation_stages = kwargs.get("simulation_stages")
+    df_train = kwargs.get("df_train")
+    target_col = kwargs.get("target_col")
+
+    stages = [simulation_stages[: idx + 1] for idx in range(len(simulation_stages))]
+
+    # Only select columns that have been observed till the current stage
+    for stage in tqdm(stages, leave=False, desc="Compute feature correlation by time"):
+        current_feature_set = [
+            col for col in df_train.columns if col.endswith(f"_{stage[-1]}")
+        ]
+
+        # Get the most correlated feature for each region
+        top_feature_by_region, counter = embedding.compute_feature_correlations(
+            df_train[current_feature_set + ["Region"]],
+            df_train[target_col],
+            "all",
+        )
+
+        # Create a dataframe with the most common top feature and number of occurrences over timestep
+        _feature = counter.most_common(1)[0][0]
+        # Loop through top_feature_by_region and find the average score for _feature
+        # Calculate the average score for 'DTR_36'
+        _feature_scores = [
+            value[1][0]
+            for key, value in top_feature_by_region.items()
+            if _feature in value[0]
+        ]
+        average_score = sum(_feature_scores) / len(_feature_scores)
+        _feature = utils.remove_last_part(_feature)
+
+        df = pd.DataFrame(
+            {
+                "Stage": [stage[-1]],
+                "Date": [utils.dict_growth_stages[stage[-1]]],
+                "Feature with Highest Correlation": [counter.most_common(1)[0][0]],
+                "Feature Category": [_feature],
+                "Score": [average_score],
+                # "Type": [ci.dict_indices[_feature][0]],
+                "Number of Occurrences": [counter.most_common(1)[0][1]],
+                # "Current Feature Set": [current_feature_set],
+            }
+        )
+        frames.append(df)
+
+    df_corr_feature_by_time = pd.concat(frames)

{geocif-0.1.32 → geocif-0.1.34}/geocif/ml/spatial_autocorrelation.py

@@ -77,7 +77,7 @@ def create_base_weights(merged_df):
     return w_base, dg
 
 
-def create_weights_for_year(dg_country, regions_with_data):
+def create_weights_for_year(dg_country, regions_with_data, year):
     """
 
     Args:
@@ -97,10 +97,8 @@ def create_weights_for_year(dg_country, regions_with_data):
     ]
     if no_neighbors:
         dg = dg.drop(index=no_neighbors[0]).reset_index(drop=True)
-        try:
-            wt = weights.Queen.from_dataframe(dg[["Country Region", "geometry"]])
-        except:
-            breakpoint()
+        wt = weights.Queen.from_dataframe(dg[["Country Region", "geometry"]])
+
     return wt, dg
 
 
@@ -125,6 +123,8 @@ def compute_morans_i(merged_df):
     for year in tqdm(years, desc="Compute Moran's I"):
         year_data = merged_df[merged_df["Harvest Year"] == year]
         regions_with_data = year_data["Country Region"].unique()
+        if len(regions_with_data) < 3:
+            continue
         year_data = year_data[year_data["Country Region"].isin(regions_with_data)]
 
         y = year_data[
@@ -132,9 +132,8 @@ def compute_morans_i(merged_df):
         ].drop_duplicates()
         dg_country = year_data[["Country Region", "geometry"]].drop_duplicates()
 
-        w, x = create_weights_for_year(dg_country, regions_with_data)
+        w, x = create_weights_for_year(dg_country, regions_with_data, year)
         y = y[y["Country Region"].isin(x["Country Region"])]
-
         if len(y) > 1:
             try:
                 mi = esda.Moran(y["Yield (tn per ha)"].values, w, permutations=999)

{geocif-0.1.32 → geocif-0.1.34}/geocif/ml/stages.py

@@ -144,10 +144,13 @@ def select_stages_for_ml(stages_features, method="latest", n=100):
 
     selected_stages = []
     if method == "latest":
+        # Find the longest array in the list of arrays
+        selected_stages = [max(stages_features, key=len)]
+
         # Only select those arrays in the list of arrays that are starting with latest_stage
-        for stage in stages_features:
-            if stage[0] == latest_stage[0]:
-                selected_stages.append(stage)
+        # for stage in stages_features:
+        #     if stage[0] == latest_stage[0]:
+        #         selected_stages.append(stage)
     elif method == "fraction":
         # Filter arrays with exactly 2 elements
         two_element_arrays = []

{geocif-0.1.32 → geocif-0.1.34}/geocif/ml/trainers.py

@@ -289,6 +289,40 @@ def auto_train(
             model = LinearGAM(n_splines=25, spline_order=3).gridsearch(
                 X_train.values, y_train.values, lam=np.logspace(-3, 3, 11)
             )
+        elif model_name == "geospaNN":
+            import torch
+            import geospaNN
+
+            # Remove any categorical features
+            X_train = X_train.drop(columns=cat_features)
+            X = torch.from_numpy(X_train.to_numpy()).float()
+            Y = torch.from_numpy(y_train.to_numpy().reshape(-1)).float()
+
+            coord = torch.from_numpy(df_train[['lon', 'lat']].to_numpy()).float()
+
+            p = X.shape[1]
+            n = X.shape[0]
+            nn = 5
+
+            data = geospaNN.make_graph(X, Y, coord, nn)
+
+            mlp = torch.nn.Sequential(
+                torch.nn.Linear(p, 50),
+                torch.nn.ReLU(),
+                torch.nn.Linear(50, 20),
+                torch.nn.ReLU(),
+                torch.nn.Linear(20, 10),
+                torch.nn.ReLU(),
+                torch.nn.Linear(10, 1),
+            )
+
+            # Split data
+            data_train, data_val, data_test = geospaNN.split_data(X, Y, coord, neighbor_size=nn, test_proportion=0.1)
+            theta0 = geospaNN.theta_update(torch.tensor([1, 1.5, 0.01]), mlp(data_train.x).squeeze() - data_train.y, data_train.pos, neighbor_size=5)
+            model = geospaNN.nngls(p=p, neighbor_size=nn, coord_dimensions=2, mlp=mlp, theta=torch.tensor(theta0))
+            nngls_model = geospaNN.nngls_train(model, lr=0.01, min_delta=0.001)
+            # Log training process
+            training_log = nngls_model.train(data_train, data_val, data_test, Update_init=10, Update_step=10)
         elif model_name == "xgboost":
             raise NotImplementedError
         else:

{geocif-0.1.32 → geocif-0.1.34}/geocif/playground/misc.py

@@ -1,6 +1,76 @@
-import pandas as pd
+import geopandas as gpd
+import pygmt
 import matplotlib.pyplot as plt
-import matplotlib.patches as patches
+from matplotlib.lines import Line2D
+import matplotlib.patches as mpatches
+import os
+filtered_shapefile_path = r"D:\Users\ritvik\projects\GEOGLAM\Input\Global_Datasets\Regions\Shps\filtered_shapefile5.shp"
+
+if not os.path.isfile(filtered_shapefile_path):
+
+    # Load the shapefile using GeoPandas
+    shapefile_path = r"D:\Users\ritvik\projects\GEOGLAM\Input\Global_Datasets\Regions\Shps\adm_shapefile.shp"
+    gdf = gpd.read_file(shapefile_path, engine="pyogrio")
+
+    # Only keep one row per ADMIN0
+    gdf = gdf.drop_duplicates(subset="ADMIN0")
+
+    sh2_path = r"D:\Users\ritvik\projects\GEOGLAM\Input\Global_Datasets\Regions\Shps\Level_1.shp"
+    gdf2 = gpd.read_file(sh2_path, engine="pyogrio")
+
+    # Subset gdf2 to USA, Pakistan and Afghanistan
+    gdf2 = gdf2[gdf2["ADM0_NAME"].isin(["United States of America"])]
+
+    # Exclude Alska and Hawaii from the USA
+    gdf2 = gdf2[~gdf2["ADM1_NAME"].isin(["Alaska", "Hawaii"])]
+
+    # Now combine all the states into one polygon
+    gdf2 = gdf2.dissolve(by="ADM0_NAME")
+    gdf2 = gdf2.reset_index()
+
+    # Rename ADM0_NAME to ADMIN0 for consistency
+    gdf2.rename(columns={"ADM0_NAME": "ADMIN0"}, inplace=True)
+
+    # Only keep ADMIN0 and geometry columns in gdf and gdf2
+    gdf = gdf[["ADMIN0", "geometry"]]
+    gdf2 = gdf2[["ADMIN0", "geometry"]]
+
+    # Merge gdf and gdf2
+    import pandas as pd
+    gdf = pd.concat([gdf, gdf2], ignore_index=True)
+
+    # Save the filtered shapefile as a temporary file
+
+    gdf.to_file(filtered_shapefile_path)
+else:
+    gdf = gpd.read_file(filtered_shapefile_path, engine="pyogrio")
+
+# Create the global map with highlighted countries
+fig = pygmt.Figure()
+
+# Define the region of interest and projection
+# fig.basemap(region="g", projection="R12c/20", frame=True)
+fig.basemap(region=[-135, 60, -35, 53], projection="Q12c", frame=True)
+
+# Use the coast function to draw land and water
+fig.coast(land="lightgray", water="lightcyan")
+
+# Highlight the countries using the filtered shapefile
+fig.plot(data=filtered_shapefile_path, pen="0.35p,black")
+
+# Add hatches to Pakistan and Afghanistan
+gdf_filled = gdf[gdf["ADMIN0"].isin(["Pakistan", "Afghanistan"])]
+for _, row in gdf_filled.iterrows():
+    fill_gdf = gpd.GeoDataFrame([row], columns=gdf.columns)
+    with pygmt.helpers.GMTTempFile() as tmpfile:
+        fill_gdf.to_file(tmpfile.name, driver="GeoJSON")
+        fig.plot(data=tmpfile.name, pen="0.35p,black", fill="black@50+h")
+
+# Save the figure
+fig.savefig("global_choropleth_highlighted_v1.png", dpi=1000)
+
+# Show the figure
+fig.show()
 
 import matplotlib.pyplot as plt
 import cartopy.crs as ccrs

{geocif-0.1.32 → geocif-0.1.34/geocif.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: geocif
-Version: 0.1.32
+Version: 0.1.34
 Summary: Models to visualize and forecast crop conditions and yields
 Home-page: https://ritviksahajpal.github.io/yield_forecasting/
 Author: Ritvik Sahajpal

{geocif-0.1.32 → geocif-0.1.34}/geocif.egg-info/SOURCES.txt

@@ -8,6 +8,7 @@ geocif/__init__.py
 geocif/analysis.py
 geocif/geocif.py
 geocif/indices_runner.py
+geocif/indices_runner_v2.py
 geocif/logger.py
 geocif/utils.py
 geocif.egg-info/PKG-INFO
@@ -31,6 +32,7 @@ geocif/cei/definitions.py
 geocif/cei/indices.py
 geocif/ml/__init__.py
 geocif/ml/correlations.py
+geocif/ml/correlations_backup.py
 geocif/ml/embedding.py
 geocif/ml/feature_engineering.py
 geocif/ml/feature_selection.py

{geocif-0.1.32 → geocif-0.1.34}/setup.py

@@ -50,6 +50,6 @@ setup(
     test_suite="tests",
     tests_require=test_requirements,
     url="https://ritviksahajpal.github.io/yield_forecasting/",
-    version="0.1.32",
+    version="0.1.34",
     zip_safe=False,
 )