geocif 0.1.68__tar.gz → 0.1.70__tar.gz

{geocif-0.1.68/geocif.egg-info → geocif-0.1.70}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: geocif
-Version: 0.1.68
+Version: 0.1.70
 Summary: Models to visualize and forecast crop conditions and yields
 Home-page: https://ritviksahajpal.github.io/yield_forecasting/
 Author: Ritvik Sahajpal

{geocif-0.1.68 → geocif-0.1.70}/geocif/cei/indices.py

@@ -675,7 +675,7 @@ class CEIs:
                 elif "STD" in iname:
                     val = np.nanstd(eo_vals)
                 elif "AUC" in iname:
-                    val = np.trapz(eo_vals, dx=len(eo_vals))
+                    val = np.trapz(eo_vals)
                 elif "H-INDEX" in iname:
                     # Multiply by 10 for h-index to work
                     val = utils.compute_h_index(eo_vals * 10)

{geocif-0.1.68 → geocif-0.1.70}/geocif/geocif.py

@@ -243,11 +243,11 @@ class Geocif:
                     if any(cei in column for cei in self.use_ceis)
                 ]
             else:
-                # self.logger.info(f"Selecting features for {self.country} {self.crop}")
+                self.logger.info(f"Selecting features for {self.country} {self.crop}")
                 selector, _, self.selected_features = fs.select_features(
                     X_train, y_train, method=self.feature_selection
                 )
-                # self.logger.info(f"Selected features: {self.selected_features}")
+                self.logger.info(f"Selected features: {self.selected_features}")
 
             """ Update model to include conformal estimates """
             if "lat" not in self.selected_features and self.include_lat_lon_as_feature:
@@ -306,7 +306,7 @@ class Geocif:
                         X_train,
                         y_train,
                         cat_features=self.cat_features,
-                        verbose=False,
+                        verbose=True,
                     )
                 elif self.model_name in ["ngboost", "oblique", "tabpfn"]:
                     X_train = X_train.drop(
@@ -598,15 +598,15 @@ class Geocif:
             df_region[f"Median {self.target}"].values, 3
         )
 
-        # if f"Median {self.target} (2014-2018)" in df_region.columns:
-        #     df.loc[:, f"Median {self.target} (2014-2018)"] = np.around(
-        #         df_region[f"Median {self.target} (2014-2018)"].values, 3
-        #     )
-        #
-        # if f"Median {self.target} (2013-2017)" in df_region.columns:
-        #     df.loc[:, f"Median {self.target} (2013-2017)"] = np.around(
-        #         df_region[f"Median {self.target} (2013-2017)"].values, 3
-        #     )
+        if f"Median {self.target} (2018-2022)" in df_region.columns:
+            df.loc[:, f"Median {self.target} (2018-2022)"] = np.around(
+                df_region[f"Median {self.target} (2018-2022)"].values, 3
+            )
+
+        if f"Median {self.target} (2013-2017)" in df_region.columns:
+            df.loc[:, f"Median {self.target} (2013-2017)"] = np.around(
+                df_region[f"Median {self.target} (2013-2017)"].values, 3
+            )
 
         if self.estimate_ci:
             if self.estimate_ci_for_all or self.forecast_season == self.today_year:
@@ -820,8 +820,8 @@ class Geocif:
                 + self.statistics_columns
                 + self.feature_names
                 + [f"Median {self.target}"]
-                #+ [f"Median {self.target} (2014-2018)"]
-                #+ [f"Median {self.target} (2013-2017)"]
+                + [f"Median {self.target} (2018-2022)"]
+                + [f"Median {self.target} (2013-2017)"]
                 + ["Region_ID"]
             )
             if self.check_yield_trend:
@@ -997,6 +997,25 @@ class Geocif:
         if self.use_single_time_period_as_feature:
             df = stages.select_single_time_period_features(df)
 
+        # If forecasting for current season, then exclude the latest month data as it will be partial
+        # and will confuse the model
+        if self.forecast_season == self.today_year:
+            current_month = ar.utcnow().month
+
+            # Identify columns where the second chunk equals the current month index
+            cols_to_drop = []
+            for col in df.columns:
+                if "_" in col:
+                    mon = stages.get_stage_information_dict(col, self.method)['Starting Stage']
+
+                    if mon == current_month:
+                        cols_to_drop.append(col)
+
+            # Drop those columns
+
+            df = df.drop(columns=cols_to_drop)
+
+        # Hack: If
         # Change column name
         # e.g. 'vDTR_7_6_5_4_3_2_1_37_36_35_34_33_32_31' to 'vDTR Mar 1-Oct 27'
         df = stages.update_feature_names(df, self.method)
@@ -1011,13 +1030,13 @@ class Geocif:
             df, self.all_seasons_with_yield, self.number_median_years, self.target
         )
 
-        # df = fe.compute_user_median_statistics(
-        #     df, [2014, 2015, 2016, 2017, 2018]
-        # )
-        #
-        # df = fe.compute_user_median_statistics(
-        #     df, [2013, 2014, 2015, 2016, 2017]
-        # )
+        df = fe.compute_user_median_statistics(
+            df, range(2018, 2023)
+        )
+
+        df = fe.compute_user_median_statistics(
+            df, range(2013, 2018)
+        )
 
         if self.median_area_as_feature:
             df = fe.compute_median_statistics(
@@ -1343,7 +1362,11 @@ class Geocif:
         # e.g. _stages = ['13_12_11', '13_12_11_10', '13_12_11_10_9']
         # then self.simulation_stages = [array([13, 12, 11]), array([13, 12, 11, 10]), array([13, 12, 11, 10, 9])]
         # Drop stages in self.all_stages that do not have _ in them
-        self.all_stages = [element for element in self.all_stages if "_" in element]
+        # self.all_stages = [element for element in self.all_stages if "_" in element]
+
+        if self.forecast_season == self.today_year:
+            current_month = ar.utcnow().month
+            self.all_stages = [elem for elem in self.all_stages if not elem.startswith(str(current_month))]
 
         self.simulation_stages = [
             np.array([int(stage) for stage in s.split("_")]) for s in self.all_stages

{geocif-0.1.68 → geocif-0.1.70}/geocif/geocif_runner.py

@@ -26,40 +26,40 @@ def loop_execute(inputs):
     Returns:
 
     """
-    from pycallgraph2 import Config, PyCallGraph, GlobbingFilter
-    from pycallgraph2.output import GraphvizOutput
-
-    graphviz = GraphvizOutput()
-    graphviz.output_file = "geocif_visualization.png"
-    plt.rcParams["figure.dpi"] = 600
-    config = Config(max_depth=5)
-    config.trace_filter = GlobbingFilter(
-        exclude=[
-            "pycallgraph.*",
-        ]
-    )
-
-    with PyCallGraph(output=graphviz, config=config):
-        project_name, country, crop, season, model, logger, parser, index = inputs
-
-        logger.info("=====================================================")
-        logger.info(f"\tStarting GEOCIF: {country} {crop} {season} {model}")
-        logger.info("=====================================================")
-
-        obj = geocif.Geocif(logger=logger,
-                            parser=parser,
-                            project_name=project_name)
-        obj.read_data(country, crop, season)
-
-        # Store config file in database, only execute this for
-        # the first iteration of the loop
-        if index == 0:
-            output.config_to_db(obj.db_path, obj.parser, obj.today)
-
-        # Setup metadata and run ML code
-        obj.setup(season, model)
-        if obj.simulation_stages:
-            obj.execute()
+    # from pycallgraph2 import Config, PyCallGraph, GlobbingFilter
+    # from pycallgraph2.output import GraphvizOutput
+    #
+    # graphviz = GraphvizOutput()
+    # graphviz.output_file = "geocif_visualization.png"
+    # plt.rcParams["figure.dpi"] = 600
+    # config = Config(max_depth=5)
+    # config.trace_filter = GlobbingFilter(
+    #     exclude=[
+    #         "pycallgraph.*",
+    #     ]
+    # )
+    #
+    # with PyCallGraph(output=graphviz, config=config):
+    project_name, country, crop, season, model, logger, parser, index = inputs
+
+    logger.info("=====================================================")
+    logger.info(f"\tStarting GEOCIF: {country} {crop} {season} {model}")
+    logger.info("=====================================================")
+
+    obj = geocif.Geocif(logger=logger,
+                        parser=parser,
+                        project_name=project_name)
+    obj.read_data(country, crop, season)
+
+    # Store config file in database, only execute this for
+    # the first iteration of the loop
+    if index == 0:
+        output.config_to_db(obj.db_path, obj.parser, obj.today)
+
+    # Setup metadata and run ML code
+    obj.setup(season, model)
+    if obj.simulation_stages:
+        obj.execute()
 
 
 def gather_inputs(parser):

geocif-0.1.70/geocif/indices_runner_angola.py

@@ -0,0 +1,212 @@
+import itertools
+import warnings
+from multiprocessing import Pool, cpu_count
+from pathlib import Path
+
+import arrow as ar
+import pandas as pd
+from tqdm import tqdm
+
+warnings.filterwarnings("ignore")
+
+from .cei import indices
+from geoprepare import base
+
+country = "angola"
+
+def remove_duplicates(lst):
+    """
+
+    :param lst:
+    :return:
+    """
+    return list(set([i for i in lst]))
+
+
+def get_admin_zone(country, dg_shp):
+    admin_zone = "admin_1"
+    country = country.title().replace(" ", "_")
+
+    # Read in shapefile
+    dg_country = dg_shp[dg_shp["ADMIN0"] == country]
+
+    # Is the ADMIN2 column all None? If so, return admin_1 else return admin_2
+    if dg_country.empty:
+        admin_zone = "admin_1"
+    elif not dg_country["ADMIN2"].isna().all():
+        admin_zone = "admin_2"
+
+    return admin_zone
+
+
+class cei_runner(base.BaseGeo):
+    def __init__(self, path_config_file):
+        super().__init__(path_config_file)
+
+        # Parse configuration files
+        self.parse_config()
+
+        self.dir_input = Path(self.parser.get("PATHS", "dir_input"))
+        import platform
+        if platform.system() == "Linux":
+            self.base_dir = Path(
+                rf"/gpfs/data1/cmongp1/GEOGLAM/Output/countries/{country}"
+            )
+        else:
+            self.base_dir = Path(
+                rf"D:\Users\ritvik\projects\GEOGLAM\Output\countries\{country}"
+            )  # Path(self.parser.get("PATHS", "dir_crop_inputs"))
+        self.do_parallel = self.parser.getboolean("DEFAULT", "do_parallel")
+
+    def collect_files(self):
+        """
+        1. Collect all the files which contain EO information
+        2. Exclude files from the `processed` directory if it is already in
+        processed_include_fall directory
+        3. Create a dataframe that contains the following columns:
+            - directory: name of directory where file is located
+            - path: full path to file
+            - filename: name of file
+        :return: Return the dataframe created above
+        """
+        import geopandas as gp
+
+        dg_shp = gp.read_file(
+            self.dir_input
+            / "Global_Datasets"
+            / "Regions"
+            / "Shps"
+            / "adm_shapefile.shp",
+            engine="pyogrio",
+        )
+
+        # Collect all the files which contain EO information
+        df_files = pd.DataFrame(columns=["directory", "path", "filename", "admin_zone"])
+        for filepath in self.base_dir.rglob("*.csv"):
+            country = filepath.parents[0].name
+
+            admin_zone = get_admin_zone(country, dg_shp)
+
+            # If country is not in cc.COUNTRIES then skip
+            # HACK: Skip korea for now, as it is giving errors
+            if country == "republic_of_korea":
+                continue
+
+            # Get name of directory one level up
+            process_type = filepath.parents[1].name
+
+            # Get name of file
+            filename = filepath.name
+
+            # Add to dataframe
+            df_files.loc[len(df_files)] = [process_type, filepath, filename, admin_zone]
+
+        # Exclude those rows where directory is processed and file is already in
+        # processed_include_fall directory
+        no_fall = df_files["directory"] == "processed"
+        include_fall = df_files[df_files["directory"] == "processed_include_fall"][
+            "filename"
+        ]
+
+        df_files = df_files[~(no_fall & (df_files["filename"].isin(include_fall)))]
+
+        return df_files
+
+    def process_combinations(self, df, method):
+        """
+        Create a list of tuples of the following:
+            - directory: name of directory where file is located
+            - path: full path to file
+            - filename: name of file
+            - method: whether to compute indices for phenological stages or not
+        This tuple will be used as input to the `process` function
+        :param df:
+        :param method:
+        :return:
+        """
+        combinations = []
+
+        for index, row in tqdm(df.iterrows()):
+            combinations.extend(
+                list(
+                    itertools.product([row[0]], [row[1]], [row[2]], [row[3]], [method])
+                )
+            )
+
+        combinations = remove_duplicates(combinations)
+
+        return combinations
+
+    def main(self, method):
+        """
+
+        :param method:
+        :return:
+        """
+        # Create a dataframe of the files to be analyzed
+        df_files = self.collect_files()
+
+        combinations = self.process_combinations(df_files, method)
+
+        # Add an element to the tuple to indicate the season
+        # Last element is redo flag which is True if the analysis is to be redone
+        # and False otherwise. Analysis is always redone for the current year
+        # and last year whether file exists or not
+        combinations = [
+            (
+                self.parser,
+                status,
+                path,
+                filename,
+                admin_zone,
+                category,
+                year,
+                "ndvi",
+                False,  # redo
+            )
+            for year in range(2001, ar.utcnow().year + 1)
+            for status, path, filename, admin_zone, category in combinations
+        ]
+
+        # Only keep those entries in combinations where the third elemt is
+        # mozambique, south_africa, angola or dem_people's_rep_of_korea
+        # This is done to test the code for these countries
+        combinations = [i for i in combinations if f"{country}_maize_s1" in i[3]]
+
+        if True:
+            num_cpu = int(cpu_count() * 0.9)
+            with Pool(num_cpu) as p:
+                for i, _ in enumerate(p.imap_unordered(indices.process, combinations)):
+                    pass
+        else:
+            # Use the code below if you want to test without parallelization or
+            # if you want to debug by using pdb
+            pbar = tqdm(combinations)
+            for i, val in enumerate(pbar):
+                pbar.set_description(
+                    f"Main loop {combinations[i][2]} {combinations[i][5]}"
+                )
+                indices.process(val)
+
+
+def run(path_config_files=[]):
+    """
+
+    Args:
+        path_config_files:
+
+    Returns:
+
+    """
+    """ Check dictionary keys to have no spaces"""
+    indices.validate_index_definitions()
+
+    for method in [
+        "monthly_r",  # "dekad_r"  # "dekad_r"
+    ]:  # , "full_season", "phenological_stages", "fraction_season"]:
+        obj = cei_runner(path_config_files)
+        obj.main(method)
+
+
+if __name__ == "__main__":
+    run()

{geocif-0.1.68 → geocif-0.1.70}/geocif/indices_runner_south_africa.py

@@ -162,7 +162,7 @@ class cei_runner(base.BaseGeo):
                 category,
                 year,
                 "ndvi",
-                False,  # redo
+                True,  # redo
             )
             for year in range(2001, ar.utcnow().year + 1)
             for status, path, filename, admin_zone, category in combinations
@@ -174,7 +174,7 @@ class cei_runner(base.BaseGeo):
         combinations = [i for i in combinations if f"{country}_maize_s1" in i[3]]
 
         if True:
-            num_cpu = int(cpu_count() * 0.1)
+            num_cpu = int(cpu_count() * 0.8)
             with Pool(num_cpu) as p:
                 for i, _ in enumerate(p.imap_unordered(indices.process, combinations)):
                     pass

{geocif-0.1.68 → geocif-0.1.70}/geocif/ml/correlations.py

@@ -295,13 +295,16 @@ def all_correlated_feature_by_time(df, **kwargs):
                     df_tmp2.loc[idx, "Type"] = combined_dict[row[0]][0]
 
                 # Compute median of each CEI and sort the dataframe based on the absolute value of the median
-                dict_best_cei[region_id] = (
-                    df_tmp2.groupby("Type")
-                    .max()
-                    .reset_index()
-                    .sort_values("Value", ascending=False)["Metric"]
-                    .values
-                )
+                try:
+                    dict_best_cei[region_id] = (
+                        df_tmp2.groupby("Type")
+                        .max()
+                        .reset_index()
+                        .sort_values("Value", ascending=False)["Metric"]
+                        .values
+                    )
+                except:
+                    breakpoint()
 
                 kwargs["region_id"] = region_id
                 _region_names = ", ".join([str(x) for x in group['Region'].unique()])

{geocif-0.1.68 → geocif-0.1.70}/geocif/ml/embedding.py

@@ -25,29 +25,32 @@ def _compute_correlations(X, y):
     feature_correlations = {}
 
     for feature in X.columns:
-        # Ignore columns that are object or categorical type
+        # Ignore object or categorical type columns
         if X[feature].dtypes.name in ["object", "category"]:
             continue
 
         f_series = X[feature]
 
-        # Ignore NaN values in either y and f_series
-        mask = ~ (np.isnan(y) | np.isnan(f_series))
-        y = y[mask]
-        f_series = f_series[mask]
+        # Ignore NaN values in either y or f_series
+        mask = ~(np.isnan(y) | np.isnan(f_series))
+        y_filtered = y[mask]
+        f_series_filtered = f_series[mask]
 
-        if np.std(f_series) == 0 or np.std(y) == 0:
+        # Handle cases where std is zero
+        if np.std(f_series_filtered) == 0 or np.std(y_filtered) == 0:
             feature_correlations[feature] = np.nan
         else:
             try:
-                r = pearsonr(y, f_series)[0]
+                r = pearsonr(y_filtered, f_series_filtered)[0]
                 feature_correlations[feature] = round(r, 3)
-            except:
+            except Exception as e:
+                print(f"Error computing correlation for {feature}: {e}")
                 feature_correlations[feature] = np.nan
 
     return feature_correlations
 
 
+
 def find_most_common_top_feature(top_feature_by_region):
     """
     Find the most common top feature and number of occurences

{geocif-0.1.68 → geocif-0.1.70}/geocif/ml/feature_engineering.py

@@ -39,10 +39,10 @@ def compute_last_year_yield(df, target_col="Yield (tn per ha)"):
 
     return df
 
-
 def compute_closest_years(all_years, harvest_year, number_lag_years):
     """
-    Finds the years closest to a given harvest year, excluding the harvest year itself.
+    Finds the historical years closest to a given harvest year,
+    excluding any future year (harvest_year itself and beyond).
 
     Args:
         all_years (array-like): List or array of all years to consider.
@@ -50,7 +50,8 @@ def compute_closest_years(all_years, harvest_year, number_lag_years):
         number_lag_years (int): Number of closest years to return.
 
     Returns:
-        list: Years closest to the given harvest year.
+        list: The historical years closest to the given harvest year.
+              Returns an empty list if no historical years exist.
     """
     # Exclude the harvest year before computation to simplify logic
     filtered_years = [year for year in all_years if year != harvest_year]
@@ -96,7 +97,7 @@ def compute_median_statistics(
             mask = (group["Harvest Year"].isin(closest_years)) & (
                 group["Region"] == region
             )
-            median_yield = group.loc[mask, target_col].median()
+            median_yield = group.loc[mask, target_col].mean()
             df.loc[
                 (df["Region"] == region) & (df["Harvest Year"] == harvest_year),
                 f"Median {target_col}",
@@ -186,7 +187,7 @@ def compute_lag_yield(
                 else:
                     # Add median yield
                     mask_group_median = group["Harvest Year"].isin(closest_years)
-                    median_yield = group.loc[mask_group_median, target_col].median()
+                    median_yield = group.loc[mask_group_median, target_col].mean()
 
                     df.loc[mask_region, col] = median_yield
 

{geocif-0.1.68 → geocif-0.1.70}/geocif/ml/trainers.py

@@ -278,7 +278,7 @@ def auto_train(
                 "loss_function": loss_function,
                 "early_stopping_rounds": 20,
                 "random_seed": seed,
-                "verbose": False,
+                "verbose": True,
             }
 
             if model_name == "catboost":