geocif 0.1.34__tar.gz → 0.1.36__tar.gz

{geocif-0.1.34/geocif.egg-info → geocif-0.1.36}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: geocif
-Version: 0.1.34
+Version: 0.1.36
 Summary: Models to visualize and forecast crop conditions and yields
 Home-page: https://ritviksahajpal.github.io/yield_forecasting/
 Author: Ritvik Sahajpal

{geocif-0.1.34 → geocif-0.1.36}/geocif/analysis.py

@@ -162,8 +162,8 @@ class Geoanalysis:
             return pd.DataFrame(), pd.DataFrame()
 
         df_metrics = self._compute_metrics(df)
-        #df_metrics = self._process_metrics(df_metrics)
-        #self._plot_metrics(df_metrics)
+        # df_metrics = self._process_metrics(df_metrics)
+        # self._plot_metrics(df_metrics)
 
         df_regional_metrics_by_year = self._compute_regional_metrics(
             df, by="Harvest Year"
@@ -173,9 +173,7 @@ class Geoanalysis:
         )
         df_regional_metrics = self._average_mape(df_regional_metrics_by_year)
         breakpoint()
-        self._store_results(
-            None, df_regional_metrics, df_regional_metrics_by_year
-        )
+        self._store_results(None, df_regional_metrics, df_regional_metrics_by_year)
 
         df_national_yield = self._compute_national_yield(df)
         self._plot_national_yield(df_national_yield)

{geocif-0.1.34 → geocif-0.1.36}/geocif/cei/indices.py

@@ -393,6 +393,7 @@ class CEIs:
             / self.admin_zone
             / self.country
         )
+
         os.makedirs(self.dir_output, exist_ok=True)
         os.makedirs(self.dir_intermediate, exist_ok=True)
 

geocif-0.1.36/geocif/experiments.py

@@ -0,0 +1,121 @@
+import configparser
+from pathlib import Path
+
+import matplotlib.pyplot as plt
+import sklearn
+
+from geocif import geocif as gc
+from geocif import logger as log
+
+plt.style.use("default")
+sklearn.set_config(transform_output="pandas")
+
+import warnings
+
+warnings.simplefilter(action="ignore", category=FutureWarning)
+
+
+def run(inputs, logger, parser, section, item, type, values):
+    if type == "str":
+        original_value = parser.get(section, item)
+    elif type == "bool":
+        original_value = parser.getboolean(section, item)
+    elif type == "int":
+        original_value = parser.getint(section, item)
+    elif type == "float":
+        original_value = parser.getfloat(section, item)
+
+    for value in values:
+        if type == "str":
+            parser.set(section, item, value)
+        elif type == "bool":
+            parser.set(section, item, str(value))
+        elif type == "int":
+            parser.set(section, item, str(value))
+        elif type == "float":
+            parser.set(section, item, str(value))
+
+        gc.execute_models(inputs, logger, parser)
+
+    parser.set(section, item, original_value)
+
+    return parser
+
+
+def main(path_config_files=[Path("../config/geocif.txt")]):
+    logger, parser = log.setup_logger_parser(path_config_files)
+    inputs = gc.gather_inputs(parser)
+
+    logger.info("=============================")
+    logger.info("\tStarting GEOCIF Experiments")
+    logger.info("=============================")
+
+    # Experiment 1: include_lat_lon
+    logger.info("Experiment 1: include_lat_lon")
+    parser = run(
+        inputs, logger, parser, "experiments", "include_lat_lon", "bool", [True, False]
+    )
+
+    # Experiment 2: feature_selection
+    logger.info("Experiment 2: feature_selection")
+    parser = run(
+        inputs,
+        logger,
+        parser,
+        "experiments",
+        "feature_selection",
+        "str",
+        ["SelectKBest", "BorutaPy", "Leshy", "RFECV", "RFE"],
+    )
+
+    # Experiment 3: lag_years
+    logger.info("Experiment 3: lag_years")
+    parser = run(
+        inputs, logger, parser, "experiments", "lag_years", "int", [1, 2, 3, 4, 5]
+    )
+
+    # Experiment 4: lag_yield_as_feature
+    logger.info("Experiment 4: lag_yield_as_feature")
+    parser = run(
+        inputs,
+        logger,
+        parser,
+        "experiments",
+        "lag_yield_as_feature",
+        "bool",
+        [True, False],
+    )
+
+    # Experiment 5: median_years
+    logger.info("Experiment 5: median_years")
+    parser = run(
+        inputs, logger, parser, "experiments", "median_years", "int", [2, 3, 4, 5]
+    )
+
+    # Experiment 6: median_yield_as_feature
+    logger.info("Experiment 6: median_yield_as_feature")
+    parser = run(
+        inputs,
+        logger,
+        parser,
+        "experiments",
+        "median_yield_as_feature",
+        "bool",
+        [True, False],
+    )
+
+    # Experiment 7: analogous_year_yield_as_feature
+    logger.info("Experiment 7: analogous_year_yield_as_feature")
+    parser = run(
+        inputs,
+        logger,
+        parser,
+        "experiments",
+        "analogous_year_yield_as_feature",
+        "bool",
+        [True, False],
+    )
+
+
+if __name__ == "__main__":
+    main()

{geocif-0.1.34 → geocif-0.1.36}/geocif/geocif.py

@@ -114,6 +114,7 @@ class Geocif:
         self.analogous_year_yield_as_feature = self.parser.getboolean(
             "ML", "analogous_year_yield_as_feature"
         )
+        self.include_lat_lon = self.parser.getboolean("ML", "include_lat_lon")
         self.spatial_autocorrelation = self.parser.getboolean(
             "ML", "spatial_autocorrelation"
         )
@@ -187,7 +188,6 @@ class Geocif:
         os.makedirs(self.dir_analysis, exist_ok=True)
 
         self.db_path = self.dir_db / self.db_forecasts
-        output.config_to_db(self.db_path, self.parser, self.today_full)
 
         # self.pickle_file = self.base_dir / self.parser.get("outlook", "pickle_file")
         # obj_pickle = outlook.Outlook(self.pickle_file)
@@ -222,9 +222,9 @@ class Geocif:
             self.logger.info(f"Selected features: {self.selected_features}")
 
             """ Update model to include conformal estimates """
-            if "lat" not in self.selected_features:
+            if "lat" not in self.selected_features and self.include_lat_lon:
                 self.selected_features.append("lat")
-            if "lon" not in self.selected_features:
+            if "lon" not in self.selected_features and self.include_lat_lon:
                 self.selected_features.append("lon")
             X_train = df_region[self.selected_features + self.cat_features]
             dir_output = (
@@ -358,7 +358,9 @@ class Geocif:
                 # Remove any categorical features
                 X_test = X_test.drop(columns=self.cat_features)
                 X = torch.from_numpy(X_test.to_numpy()).float()
-                coord = torch.from_numpy(self.df_test[['lon', 'lat']].to_numpy()).float()
+                coord = torch.from_numpy(
+                    self.df_test[["lon", "lat"]].to_numpy()
+                ).float()
 
                 p = X.shape[1]
                 n = X.shape[0]
@@ -367,7 +369,9 @@ class Geocif:
                 data = geospaNN.make_graph(X, Y, coord, nn)
 
                 # remove categorical features from df_train
-                data_train = df_region[self.selected_features + self.cat_features + [self.target]]
+                data_train = df_region[
+                    self.selected_features + self.cat_features + [self.target]
+                ]
                 w_train = data_train.y - self.estimate(data_train.x)
 
             else:
@@ -401,6 +405,7 @@ class Geocif:
         df = pd.DataFrame(
             {
                 "Experiment_ID": np.full(shp, experiment_id),
+                "Experiment Name": np.full(shp, self.experiment_name),
                 "Date": np.full(shp, self.today),
                 "Time": np.full(shp, now),
                 "Country": np.full(shp, self.country),
@@ -479,6 +484,7 @@ class Geocif:
 
         # Create an index based on following columns
         index_columns = [
+            "Experiment Name",
             "Model",
             "Cluster Strategy",
             "Country",
@@ -487,10 +493,9 @@ class Geocif:
             "Harvest Year",
             "Stage Name",
         ]
-        try:
-            df.index = df.apply(lambda row: "_".join([str(row[col]) for col in index_columns]), axis=1)
-        except Exception as e:
-            breakpoint()
+        df.index = df.apply(
+            lambda row: "_".join([str(row[col]) for col in index_columns]), axis=1
+        )
 
         # name the index level
         df.index.set_names(["Index"], inplace=True)
@@ -558,8 +563,9 @@ class Geocif:
             self.feature_names.append("FCST")
 
         # Add lat and lon to feature names
-        self.feature_names.append("lat")
-        self.feature_names.append("lon")
+        if self.include_lat_lon:
+            self.feature_names.append("lat")
+            self.feature_names.append("lon")
 
         self.selected_features = []
 
@@ -821,7 +827,11 @@ class Geocif:
         self.dg_country["lon"] = self.dg_country.centroid.x
 
         # Add lat and lon columns to df by merging on Country Region column
-        df = df.merge(self.dg_country[["Country Region", "lat", "lon"]].drop_duplicates(), on="Country Region", how="left")
+        df = df.merge(
+            self.dg_country[["Country Region", "lat", "lon"]].drop_duplicates(),
+            on="Country Region",
+            how="left",
+        )
 
         dict_kwargs = {}
         dict_kwargs["all_stages"] = self.all_stages
@@ -918,6 +928,7 @@ class Geocif:
 
         self.forecast_season = forecast_season
         self.model_name = model
+        self.experiment_name = self.parser.get("ML", "experiment_name")
         self.ml_model = self.parser.getboolean(self.model_name, "ML_model")
         self.model_names = ast.literal_eval(self.parser.get(self.country, "models"))
         self.optimize = self.parser.getboolean(self.country, "optimize")

{geocif-0.1.34 → geocif-0.1.36}/geocif/indices_runner.py

@@ -165,13 +165,13 @@ class cei_runner(base.BaseGeo):
         combinations = [
             i
             for i in combinations
-            if "angola_maize" in i[3] or
-               "lesotho_maize" in i[3] or
+            if "angola_maize" in i[3] or "lesotho_maize" in i[3] or
             #   "namibia_" in i[2] or
-                "united_republic_of_tanzania_maize" in i[3] or
-            "zambia_maize" in i[3] or "zimbabwe_maize" in i[3] or
-            "south_africa_maize" in i[3] or
-            "mozambique_maize" in i[3]
+            "united_republic_of_tanzania_maize" in i[3]
+            or "zambia_maize" in i[3]
+            or "zimbabwe_maize" in i[3]
+            or "south_africa_maize" in i[3]
+            or "mozambique_maize" in i[3]
         ]
         #                 "malawi" in i[2]]
 

{geocif-0.1.34 → geocif-0.1.36}/geocif/indices_runner_v2.py

@@ -46,7 +46,9 @@ class cei_runner(base.BaseGeo):
         self.parse_config()
 
         self.dir_input = Path(self.parser.get("PATHS", "dir_input"))
-        self.base_dir = Path(self.parser.get("PATHS", "dir_crop_inputs"))
+        self.base_dir = Path(
+            r"D:\Users\ritvik\projects\GEOGLAM\Output\countries\malawi"
+        )  # Path(self.parser.get("PATHS", "dir_crop_inputs"))
         self.do_parallel = self.parser.getboolean("DEFAULT", "do_parallel")
 
     def collect_files(self):
@@ -155,22 +157,17 @@ class cei_runner(base.BaseGeo):
                 "ndvi",
                 False,  # redo
             )
-            for year in range(2001, ar.utcnow().year + 1)
+            for year in range(2024, ar.utcnow().year + 1)
             for status, path, filename, admin_zone, category in combinations
         ]
 
         # Only keep those entries in combinations where the third elemt is
         # mozambique, south_africa, angola or dem_people's_rep_of_korea
         # This is done to test the code for these countries
-        #combinations = [
-        #    i
-        #    for i in combinations
-        #    if "ethiopia_maize_s1" in i[3]
-        #]
-        #                 "malawi" in i[2]]
+        combinations = [i for i in combinations if "malawi_maize_s1" in i[3]]
 
         if True:
-            num_cpu = int(cpu_count() * 0.8)
+            num_cpu = int(cpu_count() * 0.5)
             with Pool(num_cpu) as p:
                 for i, _ in enumerate(p.imap_unordered(indices.process, combinations)):
                     pass
@@ -198,7 +195,7 @@ def run(path_config_files=[]):
     indices.validate_index_definitions()
 
     for method in [
-        "phenological_stages",  # "dekad_r"  # "dekad_r"
+        "biweekly_r",  # "dekad_r"  # "dekad_r"
     ]:  # , "full_season", "phenological_stages", "fraction_season"]:
         obj = cei_runner(path_config_files)
         obj.main(method)

{geocif-0.1.34 → geocif-0.1.36}/geocif/ml/correlations.py

@@ -246,6 +246,7 @@ def all_correlated_feature_by_time(df, **kwargs):
     Returns:
 
     """
+    THRESHOLD = 0.1
     national_correlation = kwargs.get("national_correlation")
     group_by = kwargs.get("groupby")
     combined_dict = kwargs.get("combined_dict")
@@ -264,8 +265,16 @@ def all_correlated_feature_by_time(df, **kwargs):
             df_corr = df_corr.dropna(thresh=len(df_corr) / 2, axis=1)
 
             if not df_corr.empty:
-                df_tmp = df_corr[df_corr.columns[(df_corr.mean() > 0.1)]]
-                dict_selected_features[region_id] = df_tmp.columns
+                df_tmp = df_corr[df_corr.columns[(abs(df_corr.mean()) > THRESHOLD)]]
+                # Add the columns to dict_selected_features along with the absolute mean value
+                absolute_medians = df_tmp.abs().median()
+
+                # Create a DataFrame to display the column names and their absolute median values
+                absolute_median_df = absolute_medians.reset_index()
+                absolute_median_df.columns = ['CEI', 'Median']
+
+                # Add the CEI and Median value to dict_selected_features
+                dict_selected_features[region_id] = absolute_median_df
 
                 df_tmp2 = (
                     df_tmp.median(axis=0)
@@ -293,24 +302,31 @@ def all_correlated_feature_by_time(df, **kwargs):
             else:
                 # HACK
                 df_corr = _all_correlated_feature_by_time(df, **kwargs)
-                dict_selected_features[region_id] = df_corr.columns
-                dict_best_cei[region_id] = {}
 
-                # dict_selected_features[region_id] = dict_selected_features[0]
-                # dict_best_cei[region_id] = dict_best_cei[0]
-                # Combine all unique values from the existing dictionary elements
-                # combined_metrics = set()
-                # for key in dict_selected_features:
-                #     breakpoint()
-                #     combined_metrics.update(dict_selected_features[key])
-                #
-                # # Add the combined set as a new element with key 3
-                # dict_selected_features[region_id] = sorted(list(combined_metrics))
+                df_tmp = df_corr[df_corr.columns[(abs(df_corr.mean()) > THRESHOLD)]]
+                # Add the columns to dict_selected_features along with the absolute mean value
+                absolute_medians = df_tmp.abs().median()
+
+                # Create a DataFrame to display the column names and their absolute median values
+                absolute_median_df = absolute_medians.reset_index()
+                absolute_median_df.columns = ['CEI', 'Median']
+
+                # Add the CEI and Median value to dict_selected_features
+                dict_selected_features[region_id] = absolute_median_df
+                dict_best_cei[region_id] = {}
     else:
         df_corr = _all_correlated_feature_by_time(df, **kwargs)
-        dict_selected_features[0] = df_corr.columns
+        df_tmp = df_corr[df_corr.columns[(abs(df_corr.mean()) > THRESHOLD)]]
+        # Add the columns to dict_selected_features along with the absolute mean value
+        absolute_medians = df_tmp.abs().median()
+
+        # Create a DataFrame to display the column names and their absolute median values
+        absolute_median_df = absolute_medians.reset_index()
+        absolute_median_df.columns = ['CEI', 'Median']
+
+        # Add the CEI and Median value to dict_selected_features
+        dict_selected_features[0] = absolute_median_df
 
-        df_corr = df_corr[df_corr.columns[(df_corr.mean() > 0.1)]]
         plot_feature_corr_by_time(df_corr, **kwargs)
 
     return dict_selected_features, dict_best_cei

{geocif-0.1.34 → geocif-0.1.36/geocif.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: geocif
-Version: 0.1.34
+Version: 0.1.36
 Summary: Models to visualize and forecast crop conditions and yields
 Home-page: https://ritviksahajpal.github.io/yield_forecasting/
 Author: Ritvik Sahajpal

{geocif-0.1.34 → geocif-0.1.36}/geocif.egg-info/SOURCES.txt

@@ -6,6 +6,7 @@ setup.cfg
 setup.py
 geocif/__init__.py
 geocif/analysis.py
+geocif/experiments.py
 geocif/geocif.py
 geocif/indices_runner.py
 geocif/indices_runner_v2.py

{geocif-0.1.34 → geocif-0.1.36}/setup.py

@@ -50,6 +50,6 @@ setup(
     test_suite="tests",
     tests_require=test_requirements,
     url="https://ritviksahajpal.github.io/yield_forecasting/",
-    version="0.1.34",
+    version="0.1.36",
     zip_safe=False,
 )