geocif 0.1.67__tar.gz → 0.1.69__tar.gz

{geocif-0.1.67/geocif.egg-info → geocif-0.1.69}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: geocif
-Version: 0.1.67
+Version: 0.1.69
 Summary: Models to visualize and forecast crop conditions and yields
 Home-page: https://ritviksahajpal.github.io/yield_forecasting/
 Author: Ritvik Sahajpal

{geocif-0.1.67 → geocif-0.1.69}/geocif/cei/definitions.py

@@ -1,11 +1,11 @@
 PHENOLOGICAL_STAGES = [1, 2, 3]
 dict_indices = {
     "GD4": ["Cold", "Growing degree days (sum of Tmean > 4 C)"],
-    #"CFD": ["Cold", "Maximum number of consecutive frost days (Tmin < 0 C)"],
-    #"FD": ["Cold", "Number of Frost Days (Tmin < 0C)"],
+    "CFD": ["Cold", "Maximum number of consecutive frost days (Tmin < 0 C)"],
+    "FD": ["Cold", "Number of Frost Days (Tmin < 0C)"],
     "HD17": ["Cold", "Heating degree days (sum of Tmean < 17 C)"],
-    #"ID": ["Cold", "Number of sharp Ice Days (Tmax < 0C)"],
-    #"CSDI": ["Cold", "Cold-spell duration index"],
+    "ID": ["Cold", "Number of sharp Ice Days (Tmax < 0C)"],
+    "CSDI": ["Cold", "Cold-spell duration index"],
     "TG10p": ["Cold", "Percentage of days when Tmean < 10th percentile"],
     "TN10p": ["Cold", "Percentage of days when Tmin < 10th percentile"],
     "TXn": ["Cold", "Minimum daily maximum temperature"],
@@ -70,10 +70,10 @@ dict_indices = {
         "Compound",
         "Days with TG > 75th percentile of daily mean temperature and RR >75th percentile of daily precipitation sum",
     ],
-    # "SD": ["Snow", "Mean of daily snow depth"],
-    # "SD1": ["Snow", "Number of days with snow depth >= 1 cm"],
-    # "SD5cm": ["Snow", "Number of days with snow depth >= 5 cm"],
-    # "SD50cm": ["Snow", "Number of days with snow depth >= 50 cm"],
+    "SD": ["Snow", "Mean of daily snow depth"],
+    "SD1": ["Snow", "Number of days with snow depth >= 1 cm"],
+    "SD5cm": ["Snow", "Number of days with snow depth >= 5 cm"],
+    "SD50cm": ["Snow", "Number of days with snow depth >= 50 cm"],
 }
 
 dict_ndvi = {

{geocif-0.1.67 → geocif-0.1.69}/geocif/geocif.py

@@ -243,11 +243,11 @@ class Geocif:
                     if any(cei in column for cei in self.use_ceis)
                 ]
             else:
-                # self.logger.info(f"Selecting features for {self.country} {self.crop}")
+                self.logger.info(f"Selecting features for {self.country} {self.crop}")
                 selector, _, self.selected_features = fs.select_features(
                     X_train, y_train, method=self.feature_selection
                 )
-                # self.logger.info(f"Selected features: {self.selected_features}")
+                self.logger.info(f"Selected features: {self.selected_features}")
 
             """ Update model to include conformal estimates """
             if "lat" not in self.selected_features and self.include_lat_lon_as_feature:
@@ -306,7 +306,7 @@ class Geocif:
                         X_train,
                         y_train,
                         cat_features=self.cat_features,
-                        verbose=False,
+                        verbose=True,
                     )
                 elif self.model_name in ["ngboost", "oblique", "tabpfn"]:
                     X_train = X_train.drop(
@@ -598,9 +598,9 @@ class Geocif:
             df_region[f"Median {self.target}"].values, 3
         )
 
-        if f"Median {self.target} (2014-2018)" in df_region.columns:
-            df.loc[:, f"Median {self.target} (2014-2018)"] = np.around(
-                df_region[f"Median {self.target} (2014-2018)"].values, 3
+        if f"Median {self.target} (2018-2022)" in df_region.columns:
+            df.loc[:, f"Median {self.target} (2018-2022)"] = np.around(
+                df_region[f"Median {self.target} (2018-2022)"].values, 3
             )
 
         if f"Median {self.target} (2013-2017)" in df_region.columns:
@@ -820,7 +820,7 @@ class Geocif:
                 + self.statistics_columns
                 + self.feature_names
                 + [f"Median {self.target}"]
-                + [f"Median {self.target} (2014-2018)"]
+                + [f"Median {self.target} (2018-2022)"]
                 + [f"Median {self.target} (2013-2017)"]
                 + ["Region_ID"]
             )
@@ -1012,11 +1012,11 @@ class Geocif:
         )
 
         df = fe.compute_user_median_statistics(
-            df, [2014, 2015, 2016, 2017, 2018]
+            df, range(2018, 2023)
         )
 
         df = fe.compute_user_median_statistics(
-            df, [2013, 2014, 2015, 2016, 2017]
+            df, range(2013, 2018)
         )
 
         if self.median_area_as_feature:
@@ -1393,6 +1393,9 @@ class Geocif:
             self.dg["ADM0_NAME"].str.lower().str.replace(" ", "_") == self.country
         ]
 
+        # Drop any duplicates based on Country Region column
+        self.dg_country = self.dg_country.drop_duplicates(subset=["Country Region"])
+
     def read_data(self, country, crop, season):
         """
 

{geocif-0.1.67 → geocif-0.1.69}/geocif/geocif_runner.py

@@ -36,7 +36,6 @@ def loop_execute(inputs):
     # config.trace_filter = GlobbingFilter(
     #     exclude=[
     #         "pycallgraph.*",
-    #         "torch*",
     #     ]
     # )
     #

geocif-0.1.69/geocif/indices_runner_angola.py

@@ -0,0 +1,212 @@
+import itertools
+import warnings
+from multiprocessing import Pool, cpu_count
+from pathlib import Path
+
+import arrow as ar
+import pandas as pd
+from tqdm import tqdm
+
+warnings.filterwarnings("ignore")
+
+from .cei import indices
+from geoprepare import base
+
+country = "angola"
+
+def remove_duplicates(lst):
+    """
+
+    :param lst:
+    :return:
+    """
+    return list(set([i for i in lst]))
+
+
+def get_admin_zone(country, dg_shp):
+    admin_zone = "admin_1"
+    country = country.title().replace(" ", "_")
+
+    # Read in shapefile
+    dg_country = dg_shp[dg_shp["ADMIN0"] == country]
+
+    # Is the ADMIN2 column all None? If so, return admin_1 else return admin_2
+    if dg_country.empty:
+        admin_zone = "admin_1"
+    elif not dg_country["ADMIN2"].isna().all():
+        admin_zone = "admin_2"
+
+    return admin_zone
+
+
+class cei_runner(base.BaseGeo):
+    def __init__(self, path_config_file):
+        super().__init__(path_config_file)
+
+        # Parse configuration files
+        self.parse_config()
+
+        self.dir_input = Path(self.parser.get("PATHS", "dir_input"))
+        import platform
+        if platform.system() == "Linux":
+            self.base_dir = Path(
+                rf"/gpfs/data1/cmongp1/GEOGLAM/Output/countries/{country}"
+            )
+        else:
+            self.base_dir = Path(
+                rf"D:\Users\ritvik\projects\GEOGLAM\Output\countries\{country}"
+            )  # Path(self.parser.get("PATHS", "dir_crop_inputs"))
+        self.do_parallel = self.parser.getboolean("DEFAULT", "do_parallel")
+
+    def collect_files(self):
+        """
+        1. Collect all the files which contain EO information
+        2. Exclude files from the `processed` directory if it is already in
+        processed_include_fall directory
+        3. Create a dataframe that contains the following columns:
+            - directory: name of directory where file is located
+            - path: full path to file
+            - filename: name of file
+        :return: Return the dataframe created above
+        """
+        import geopandas as gp
+
+        dg_shp = gp.read_file(
+            self.dir_input
+            / "Global_Datasets"
+            / "Regions"
+            / "Shps"
+            / "adm_shapefile.shp",
+            engine="pyogrio",
+        )
+
+        # Collect all the files which contain EO information
+        df_files = pd.DataFrame(columns=["directory", "path", "filename", "admin_zone"])
+        for filepath in self.base_dir.rglob("*.csv"):
+            country = filepath.parents[0].name
+
+            admin_zone = get_admin_zone(country, dg_shp)
+
+            # If country is not in cc.COUNTRIES then skip
+            # HACK: Skip korea for now, as it is giving errors
+            if country == "republic_of_korea":
+                continue
+
+            # Get name of directory one level up
+            process_type = filepath.parents[1].name
+
+            # Get name of file
+            filename = filepath.name
+
+            # Add to dataframe
+            df_files.loc[len(df_files)] = [process_type, filepath, filename, admin_zone]
+
+        # Exclude those rows where directory is processed and file is already in
+        # processed_include_fall directory
+        no_fall = df_files["directory"] == "processed"
+        include_fall = df_files[df_files["directory"] == "processed_include_fall"][
+            "filename"
+        ]
+
+        df_files = df_files[~(no_fall & (df_files["filename"].isin(include_fall)))]
+
+        return df_files
+
+    def process_combinations(self, df, method):
+        """
+        Create a list of tuples of the following:
+            - directory: name of directory where file is located
+            - path: full path to file
+            - filename: name of file
+            - method: whether to compute indices for phenological stages or not
+        This tuple will be used as input to the `process` function
+        :param df:
+        :param method:
+        :return:
+        """
+        combinations = []
+
+        for index, row in tqdm(df.iterrows()):
+            combinations.extend(
+                list(
+                    itertools.product([row[0]], [row[1]], [row[2]], [row[3]], [method])
+                )
+            )
+
+        combinations = remove_duplicates(combinations)
+
+        return combinations
+
+    def main(self, method):
+        """
+
+        :param method:
+        :return:
+        """
+        # Create a dataframe of the files to be analyzed
+        df_files = self.collect_files()
+
+        combinations = self.process_combinations(df_files, method)
+
+        # Add an element to the tuple to indicate the season
+        # Last element is redo flag which is True if the analysis is to be redone
+        # and False otherwise. Analysis is always redone for the current year
+        # and last year whether file exists or not
+        combinations = [
+            (
+                self.parser,
+                status,
+                path,
+                filename,
+                admin_zone,
+                category,
+                year,
+                "ndvi",
+                False,  # redo
+            )
+            for year in range(2001, ar.utcnow().year + 1)
+            for status, path, filename, admin_zone, category in combinations
+        ]
+
+        # Only keep those entries in combinations where the third elemt is
+        # mozambique, south_africa, angola or dem_people's_rep_of_korea
+        # This is done to test the code for these countries
+        combinations = [i for i in combinations if f"{country}_maize_s1" in i[3]]
+
+        if True:
+            num_cpu = int(cpu_count() * 0.9)
+            with Pool(num_cpu) as p:
+                for i, _ in enumerate(p.imap_unordered(indices.process, combinations)):
+                    pass
+        else:
+            # Use the code below if you want to test without parallelization or
+            # if you want to debug by using pdb
+            pbar = tqdm(combinations)
+            for i, val in enumerate(pbar):
+                pbar.set_description(
+                    f"Main loop {combinations[i][2]} {combinations[i][5]}"
+                )
+                indices.process(val)
+
+
+def run(path_config_files=[]):
+    """
+
+    Args:
+        path_config_files:
+
+    Returns:
+
+    """
+    """ Check dictionary keys to have no spaces"""
+    indices.validate_index_definitions()
+
+    for method in [
+        "monthly_r",  # "dekad_r"  # "dekad_r"
+    ]:  # , "full_season", "phenological_stages", "fraction_season"]:
+        obj = cei_runner(path_config_files)
+        obj.main(method)
+
+
+if __name__ == "__main__":
+    run()

{geocif-0.1.67 → geocif-0.1.69}/geocif/ml/correlations.py

@@ -295,13 +295,16 @@ def all_correlated_feature_by_time(df, **kwargs):
                     df_tmp2.loc[idx, "Type"] = combined_dict[row[0]][0]
 
                 # Compute median of each CEI and sort the dataframe based on the absolute value of the median
-                dict_best_cei[region_id] = (
-                    df_tmp2.groupby("Type")
-                    .max()
-                    .reset_index()
-                    .sort_values("Value", ascending=False)["Metric"]
-                    .values
-                )
+                try:
+                    dict_best_cei[region_id] = (
+                        df_tmp2.groupby("Type")
+                        .max()
+                        .reset_index()
+                        .sort_values("Value", ascending=False)["Metric"]
+                        .values
+                    )
+                except:
+                    breakpoint()
 
                 kwargs["region_id"] = region_id
                 _region_names = ", ".join([str(x) for x in group['Region'].unique()])

{geocif-0.1.67 → geocif-0.1.69}/geocif/ml/embedding.py

@@ -25,29 +25,32 @@ def _compute_correlations(X, y):
     feature_correlations = {}
 
     for feature in X.columns:
-        # Ignore columns that are object or categorical type
+        # Ignore object or categorical type columns
         if X[feature].dtypes.name in ["object", "category"]:
             continue
 
         f_series = X[feature]
 
-        # Ignore NaN values in either y and f_series
-        mask = ~ (np.isnan(y) | np.isnan(f_series))
-        y = y[mask]
-        f_series = f_series[mask]
+        # Ignore NaN values in either y or f_series
+        mask = ~(np.isnan(y) | np.isnan(f_series))
+        y_filtered = y[mask]
+        f_series_filtered = f_series[mask]
 
-        if np.std(f_series) == 0 or np.std(y) == 0:
+        # Handle cases where std is zero
+        if np.std(f_series_filtered) == 0 or np.std(y_filtered) == 0:
             feature_correlations[feature] = np.nan
         else:
             try:
-                r = pearsonr(y, f_series)[0]
+                r = pearsonr(y_filtered, f_series_filtered)[0]
                 feature_correlations[feature] = round(r, 3)
-            except:
+            except Exception as e:
+                print(f"Error computing correlation for {feature}: {e}")
                 feature_correlations[feature] = np.nan
 
     return feature_correlations
 
 
+
 def find_most_common_top_feature(top_feature_by_region):
     """
     Find the most common top feature and number of occurences

{geocif-0.1.67 → geocif-0.1.69}/geocif/ml/feature_engineering.py

@@ -39,10 +39,10 @@ def compute_last_year_yield(df, target_col="Yield (tn per ha)"):
 
     return df
 
-
 def compute_closest_years(all_years, harvest_year, number_lag_years):
     """
-    Finds the years closest to a given harvest year, excluding the harvest year itself.
+    Finds the historical years closest to a given harvest year,
+    excluding any future year (harvest_year itself and beyond).
 
     Args:
         all_years (array-like): List or array of all years to consider.
@@ -50,7 +50,8 @@ def compute_closest_years(all_years, harvest_year, number_lag_years):
         number_lag_years (int): Number of closest years to return.
 
     Returns:
-        list: Years closest to the given harvest year.
+        list: The historical years closest to the given harvest year.
+              Returns an empty list if no historical years exist.
     """
     # Exclude the harvest year before computation to simplify logic
     filtered_years = [year for year in all_years if year != harvest_year]
@@ -96,7 +97,7 @@ def compute_median_statistics(
             mask = (group["Harvest Year"].isin(closest_years)) & (
                 group["Region"] == region
             )
-            median_yield = group.loc[mask, target_col].median()
+            median_yield = group.loc[mask, target_col].mean()
             df.loc[
                 (df["Region"] == region) & (df["Harvest Year"] == harvest_year),
                 f"Median {target_col}",
@@ -186,7 +187,7 @@ def compute_lag_yield(
                 else:
                     # Add median yield
                     mask_group_median = group["Harvest Year"].isin(closest_years)
-                    median_yield = group.loc[mask_group_median, target_col].median()
+                    median_yield = group.loc[mask_group_median, target_col].mean()
 
                     df.loc[mask_region, col] = median_yield
 

{geocif-0.1.67 → geocif-0.1.69}/geocif/ml/feature_selection.py

@@ -188,10 +188,24 @@ def select_features(X, y, method="RFE", min_features_to_select=3, threshold_nan=
 
         # Get the selected feature names
         selected_features = X.columns[selected_features].tolist()
+    elif method == "lasso":
+        from sklearn.linear_model import LassoLarsCV
+        from sklearn.feature_selection import SelectFromModel
+
+        # Fit Lasso model (L1 regularization) to perform feature selection
+        lasso = LassoLarsCV(cv=5)
+        lasso.fit(X, y)
+
+        # Use SelectFromModel to remove features with zero coefficients
+        selector = SelectFromModel(lasso, prefit=True)
+
+        # Get the selected features
+        selected_features = X.columns[selector.get_support()].tolist()
+        print(selected_features)
     elif method == "BorutaPy":
         from boruta import BorutaPy
 
-        selector = BorutaPy(forest, n_estimators="auto", random_state=42)
+        selector = BorutaPy(forest, n_estimators="auto", random_state=42, verbose=0)
         selector.fit(X.values, y.values)
         selected_features_mask = selector.support_
         selected_features = X.columns[selected_features_mask].tolist()

{geocif-0.1.67 → geocif-0.1.69}/geocif/ml/stats.py

@@ -203,7 +203,7 @@ def add_statistics(
         fn = "illinois.csv"
     elif country == "Ethiopia":
         # HACK
-        fn = "ethiopia_wheat_summary_formatted.csv"
+        fn = "adm_crop_production.csv"
     else:
         fn = "adm_crop_production.csv"
     df_fewsnet = pd.read_csv(dir_stats / fn, low_memory=False)

{geocif-0.1.67 → geocif-0.1.69}/geocif/ml/trainers.py

@@ -278,7 +278,7 @@ def auto_train(
                 "loss_function": loss_function,
                 "early_stopping_rounds": 20,
                 "random_seed": seed,
-                "verbose": False,
+                "verbose": True,
             }
 
             if model_name == "catboost":

geocif-0.1.69/geocif/playground/wolayita_maize_mask.py

@@ -0,0 +1,156 @@
+import rasterio
+from rasterio.warp import calculate_default_transform, reproject, Resampling
+import numpy as np
+import matplotlib.pyplot as plt
+import math
+
+# Input / Output paths
+input_path = r"D:\Users\ritvik\projects\GEOGLAM\Input\Global_Datasets\Masks\wolayita_maize.tif"
+output_path = r"D:\Users\ritvik\projects\GEOGLAM\Input\Global_Datasets\Masks\wolayita_maize_5km_percentage.tif"
+
+import rasterio
+from rasterio.warp import calculate_default_transform, reproject, Resampling
+from math import ceil
+import numpy as np
+
+input_path = r"D:\Users\ritvik\projects\GEOGLAM\Input\Global_Datasets\Masks\wolayita_maize.tif"
+output_path = r"D:\Users\ritvik\projects\GEOGLAM\Input\Global_Datasets\Masks\wolayita_maize_5km_percentage.tif"
+
+with rasterio.open(input_path) as src:
+    # 1) If needed, assign correct CRS
+    # Example: if you know it's actually EPSG:32637 but isn't set
+    # src_crs = rasterio.crs.CRS.from_epsg(32637)
+    # else if it's already correct, do:
+    src_crs = src.crs
+
+    # 2) Decide your pixel size.
+    # If src_crs is lat/lon (EPSG:4326), use ~0.045 deg for ~5 km.
+    # If src_crs is UTM in meters, use 5000 for 5 km.
+    pixel_size = 0.045  # or 5000 if in meters
+
+    transform, width, height = calculate_default_transform(
+        src_crs,  # source crs
+        src_crs,  # target crs (same if you just want coarser in place)
+        src.width,
+        src.height,
+        *src.bounds,
+        resolution=pixel_size
+    )
+
+    # Prepare output fraction array
+    fraction_array = np.full((height, width), -9999, dtype=np.float32)
+
+    # Reproject with average -> fraction
+    reproject(
+        source=rasterio.band(src, 1),
+        destination=fraction_array,
+        src_transform=src.transform,
+        src_crs=src_crs,
+        dst_transform=transform,
+        dst_crs=src_crs,
+        resampling=Resampling.average,
+        dst_nodata=-9999
+    )
+
+# Now fraction_array should have values in [0..1], with -9999 for nodata.
+valid_mask = (fraction_array != -9999)
+
+if not np.any(valid_mask):
+    print("No valid cells at all (everything is nodata). This indicates a bounding box or CRS mismatch.")
+else:
+    frac_min = fraction_array[valid_mask].min()
+    frac_max = fraction_array[valid_mask].max()
+    print("Fraction min:", frac_min)
+    print("Fraction max:", frac_max)
+
+    # If both min and max are 0.0, it means there's truly no coverage or it's extremely small.
+    # Otherwise you might see something like 0.0, 0.01, 0.5, etc.
+
+    # Then let's see if maybe they're all below 0.005:
+    below_005 = (fraction_array[valid_mask] < 0.005).all()
+    print("All fractions < 0.5%?", below_005)
+
+breakpoint()
+with rasterio.open(input_path) as src:
+    # If src.crs is None but you KNOW it's EPSG:4326, assign it:
+    # src_crs = rasterio.crs.CRS.from_epsg(4326)
+    # Otherwise, just use what's in the file:
+    src_crs = src.crs
+
+    # Let's assume the file is already lat/lon (EPSG:4326).
+    # We'll define ~0.045° as "5 km" at the equator.
+    new_res = 0.045
+
+    # Calculate a new transform and new shape
+    # for coarser resolution in the SAME EPSG:4326.
+    transform, width, height = calculate_default_transform(
+        src_crs,  # src CRS
+        src_crs,  # dst CRS (same if you want to stay in lat/lon)
+        src.width,
+        src.height,
+        *src.bounds,
+        resolution=new_res  # sets pixel size to 0.045 degrees
+    )
+
+    # Read full data for histogram plotting
+    data_in = src.read(1, masked=True)
+    in_profile = src.profile.copy()
+
+# Plot input histogram (0 or 1)
+arr_in = data_in.compressed()
+plt.figure()
+plt.hist(arr_in, bins=[-0.5, 0.5, 1.5], edgecolor='black')
+plt.title("Input (0/1)")
+plt.xlabel("Value")
+plt.ylabel("Frequency")
+plt.show()
+
+# Prepare output array, float32 with sentinel -9999
+out_array = np.full((height, width), -9999, dtype=np.float32)
+
+with rasterio.open(input_path) as src:
+    reproject(
+        source=rasterio.band(src, 1),
+        destination=out_array,
+        src_transform=src.transform,
+        src_crs=src.crs,
+        dst_transform=transform,
+        dst_crs=src_crs,  # same
+        resampling=Resampling.average,
+        dst_nodata=-9999
+    )
+
+# Now out_array has fraction in [0..1]. Convert to % (0..100).
+breakpoint()
+mask_valid = (out_array != -9999)
+out_array[mask_valid] *= 100.0
+out_array[mask_valid] = np.rint(out_array[mask_valid])  # round
+out_array = out_array.astype(np.int32)
+
+# Update profile
+out_profile = in_profile.copy()
+out_profile.update({
+    'driver': 'GTiff',
+    'width': width,
+    'height': height,
+    'transform': transform,
+    'crs': src_crs,
+    'dtype': 'int32',
+    'nodata': -9999
+})
+
+# Write out
+with rasterio.open(output_path, 'w', **out_profile) as dst:
+    dst.write(out_array, 1)
+
+print("Wrote:", output_path)
+
+# Plot histogram of output (ignore -9999)
+out_data = np.where(out_array == -9999, np.nan, out_array)
+valid_data = out_data[~np.isnan(out_data)]
+plt.figure()
+plt.hist(valid_data, bins=50, edgecolor="black")
+plt.title("5km Percentage (0-100)")
+plt.xlabel("Percent cropped")
+plt.ylabel("Frequency")
+plt.show()

geocif-0.1.69/geocif/viz/gt.py

@@ -0,0 +1,69 @@
+from great_tables import GT, md, system_fonts
+import pandas as pd
+
+# Your data as a pandas DataFrame
+data = [
+    [2, "10<sup>th</sup>",          "<=14; >14",                     89.2, "2010 - 2021"],
+    [2, "25<sup>th</sup>",          "<=18.7; >18.7",                 82.2, "2010 - 2021"],
+    [2, "50<sup>th</sup>",          "<=24.6; >24.6",                 83.7, "2010 - 2021"],
+    [2, "75<sup>th</sup>",          "<=31; >31",                     88.3, "2010 - 2021"],
+    [2, "90<sup>th</sup>",          "<=38.9; >38.9",                 96.9, "2010 - 2021"],
+    [3, "33<sup>rd</sup>, 67<sup>th</sup>",  "<=20.3; 20.3 - 29.6; >29.6",     60.5, "2010 - 2021"],
+    [4, "25<sup>th</sup>, 50<sup>th</sup>, 75<sup>th</sup>",
+         "<=18.7; 18.7-24.6; 24.6-31; >31",                           64.4, "2010 - 2021"]
+]
+cols = ["Number of classes", "Percentile(s)", "Yield categories", "Accuracy (%)", "Years"]
+
+df = pd.DataFrame(data, columns=cols)
+
+# Create a Great Tables object
+gt_tbl = GT(data=df)
+
+# Example formatting, coloring, and styling
+gt_tbl = (gt_tbl
+    # Format the "Accuracy (%)" column to show one decimal place
+    .fmt_number(
+        columns=["Accuracy (%)"],
+        decimals=1
+    )
+    # Color-scale the "Accuracy (%)" column (optional)
+    #.data_color(
+    #    columns=["Accuracy (%)"],
+    #    palette=["tomato", "gold", "palegreen"],
+    #    domain=[50, 100]  # Range from the lowest to highest accuracy
+    #)
+    # Set column widths
+    .cols_width({
+        "Number classes":    "60px",
+        "Percentile(s)":     "140px",
+        "Yield categories":  "220px",
+        "Accuracy (%)":      "100px",
+        "Years":             "90px"
+    })
+    # Add a table header/title
+    .tab_header(
+        title=md("**Accuracy of Model for Different Yield Categories**")
+    )
+    # Add a source note (optional)
+    # .tab_source_note(
+    #     md(
+    #         "**Source**: Internal records<br>"
+    #         "**Note**: Data from 2010-2021"
+    #     )
+    # )
+    # Customize general table options
+    .tab_options(
+        heading_background_color='antiquewhite',
+        column_labels_background_color='antiquewhite',
+        source_notes_background_color='antiquewhite',
+        table_background_color='snow',
+        table_font_names=system_fonts("humanist"),
+        data_row_padding='2px'
+    )
+    # Align all columns center except "Yield categories", which might be longer text
+    .cols_align(align="center")
+    .cols_align(align="left", columns=["Yield categories"])
+)
+
+# Display the table
+GT.save(gt_tbl, file="aa.png")

{geocif-0.1.67 → geocif-0.1.69}/geocif/viz/tmp.py

@@ -1,6 +1,4 @@
 import geopandas as gpd
-import pandas as pd
-import matplotlib.pyplot as plt
 import palettable as pal
 import matplotlib.colors as mcolors
 
@@ -9,7 +7,7 @@ import glob
 import os
 
 # 1. Specify the directory containing your .dta files:
-data_dir = r"C:\Users\ritvik\Downloads\maize_yield\maize_yield"
+data_dir =  r"C:\Users\ritvik\Downloads\maize_yield (2)\maize_yield"
 
 # 2. Use glob to find all .dta files in that directory:
 dta_files = glob.glob(os.path.join(data_dir, "*.dta"))
@@ -20,6 +18,13 @@ dataframes = [pd.read_stata(f) for f in dta_files]
 # 4. Concatenate them all into one DataFrame (row-wise):
 merged_df = pd.concat(dataframes, ignore_index=True)
 
+# Replace null values in PROD98CQ with those in PROD columns
+merged_df['PROD98CQ'] = merged_df['PROD98CQ'].fillna(merged_df['PROD'])
+merged_df['YEAR'] = merged_df['YEAR'].fillna(merged_df['year'])
+
+# Drop rows where AREAH is 0
+merged_df = merged_df[merged_df['AREAH'] != 0]
+
 merged_df['ZONE'] = merged_df['ZONE'].astype(int)
 merged_df['DIST'] = merged_df['DIST'].astype(int)
 
@@ -36,7 +41,7 @@ merged_df['W_CODE'] = '7' + merged_df['W_CODE']
 merged_df['W_CODE'] = merged_df['W_CODE'].str.replace('.0', '')
 merged_df['W_CODE'] = merged_df['W_CODE'].astype(int)
 
-dg = gpd.read_file(r"D:\Users\ritvik\projects\GEOGLAM\Input\countries\wolayita\wolayita_dissolved.shp")
+dg = gpd.read_file(r"wolayita_dissolved.shp")
 dg = dg[['W_CODE', 'W_NAME']]
 
 # Merge the two dataframes on W_CODE
@@ -48,8 +53,8 @@ merged_df = merged_df.dropna(subset=['PROD98CQ', 'AREAH'])
 # Compte yield column
 merged_df['yield'] = merged_df['PROD98CQ'] / merged_df['AREAH']
 
-# create a new dataframe which computes average yield by W_NAME for each year
-df_avg_yield = merged_df.groupby(['W_NAME', 'YEAR'])['yield'].mean().reset_index()
+# create a new dataframe which computes average yield by W_NAME for each year, do a weighted average using FWEIGHT column
+df_avg_yield = merged_df.groupby(['W_NAME', 'YEAR']).apply(lambda x: np.average(x['yield'], weights=x['FWEIGHT'])).reset_index(name='yield')
 
 # Change W_NAME column to title case
 df_avg_yield['W_NAME'] = df_avg_yield['W_NAME'].str.title()
@@ -64,7 +69,15 @@ df_avg_yield = df_avg_yield.pivot(index='W_NAME', columns='YEAR', values='yield'
 df_avg_yield.index.name = None
 df_avg_yield.columns.name = None
 
-df_avg_yield.to_csv('wolayita_yields.csv')
+df_avg_yield.to_csv('wolayita_yields_v8.csv')
+breakpoint()
+# Compare wolayita_yields_v2.csv with wolayita_yields.csv
+# 1. Load the two CSV files
+df_v1 = pd.read_csv('wolayita_yields.csv')
+df_v2 = pd.read_csv('wolayita_yields_v2.csv')
+
+# 2. Check if the two DataFrames are equal
+print(df_v1.equals(df_v2))
 
 breakpoint()
 # 5. (Optional) Inspect the merged DataFrame

{geocif-0.1.67 → geocif-0.1.69/geocif.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: geocif
-Version: 0.1.67
+Version: 0.1.69
 Summary: Models to visualize and forecast crop conditions and yields
 Home-page: https://ritviksahajpal.github.io/yield_forecasting/
 Author: Ritvik Sahajpal

{geocif-0.1.67 → geocif-0.1.69}/geocif.egg-info/SOURCES.txt

@@ -72,9 +72,11 @@ geocif/playground/tmp2.py
 geocif/playground/tmp3.py
 geocif/playground/tmp4.py
 geocif/playground/tmp5.py
+geocif/playground/wolayita_maize_mask.py
 geocif/risk/__init__.py
 geocif/risk/impact_assessment.py
 geocif/viz/__init__.py
+geocif/viz/gt.py
 geocif/viz/plot.py
 geocif/viz/tmp.py
 tests/test_geocif.py

{geocif-0.1.67 → geocif-0.1.69}/setup.py

@@ -50,6 +50,6 @@ setup(
     test_suite="tests",
     tests_require=test_requirements,
     url="https://ritviksahajpal.github.io/yield_forecasting/",
-    version="0.1.67",
+    version="0.1.69",
     zip_safe=False,
 )

geocif-0.1.67/geocif/indices_runner_angola.py

@@ -1,212 +0,0 @@
-import itertools
-import warnings
-from multiprocessing import Pool, cpu_count
-from pathlib import Path
-
-import arrow as ar
-import pandas as pd
-from tqdm import tqdm
-
-warnings.filterwarnings("ignore")
-
-from .cei import indices
-from geoprepare import base
-
-country = "ethiopia"
-
-def remove_duplicates(lst):
-    """
-
-    :param lst:
-    :return:
-    """
-    return list(set([i for i in lst]))
-
-
-def get_admin_zone(country, dg_shp):
-    admin_zone = "admin_1"
-    country = country.title().replace(" ", "_")
-
-    # Read in shapefile
-    dg_country = dg_shp[dg_shp["ADMIN0"] == country]
-
-    # Is the ADMIN2 column all None? If so, return admin_1 else return admin_2
-    if dg_country.empty:
-        admin_zone = "admin_1"
-    elif not dg_country["ADMIN2"].isna().all():
-        admin_zone = "admin_2"
-
-    return admin_zone
-
-
-class cei_runner(base.BaseGeo):
-    def __init__(self, path_config_file):
-        super().__init__(path_config_file)
-
-        # Parse configuration files
-        self.parse_config()
-
-        self.dir_input = Path(self.parser.get("PATHS", "dir_input"))
-        import platform
-        if platform.system() == "Linux":
-            self.base_dir = Path(
-                rf"/gpfs/data1/cmongp1/GEOGLAM/Output/countries/{country}"
-            )
-        else:
-            self.base_dir = Path(
-                rf"D:\Users\ritvik\projects\GEOGLAM\Output\countries\{country}"
-            )  # Path(self.parser.get("PATHS", "dir_crop_inputs"))
-        self.do_parallel = self.parser.getboolean("DEFAULT", "do_parallel")
-
-    def collect_files(self):
-        """
-        1. Collect all the files which contain EO information
-        2. Exclude files from the `processed` directory if it is already in
-        processed_include_fall directory
-        3. Create a dataframe that contains the following columns:
-            - directory: name of directory where file is located
-            - path: full path to file
-            - filename: name of file
-        :return: Return the dataframe created above
-        """
-        import geopandas as gp
-
-        dg_shp = gp.read_file(
-            self.dir_input
-            / "Global_Datasets"
-            / "Regions"
-            / "Shps"
-            / "adm_shapefile.shp",
-            engine="pyogrio",
-        )
-
-        # Collect all the files which contain EO information
-        df_files = pd.DataFrame(columns=["directory", "path", "filename", "admin_zone"])
-        for filepath in self.base_dir.rglob("*.csv"):
-            country = filepath.parents[0].name
-
-            admin_zone = get_admin_zone(country, dg_shp)
-
-            # If country is not in cc.COUNTRIES then skip
-            # HACK: Skip korea for now, as it is giving errors
-            if country == "republic_of_korea":
-                continue
-
-            # Get name of directory one level up
-            process_type = filepath.parents[1].name
-
-            # Get name of file
-            filename = filepath.name
-
-            # Add to dataframe
-            df_files.loc[len(df_files)] = [process_type, filepath, filename, admin_zone]
-
-        # Exclude those rows where directory is processed and file is already in
-        # processed_include_fall directory
-        no_fall = df_files["directory"] == "processed"
-        include_fall = df_files[df_files["directory"] == "processed_include_fall"][
-            "filename"
-        ]
-
-        df_files = df_files[~(no_fall & (df_files["filename"].isin(include_fall)))]
-
-        return df_files
-
-    def process_combinations(self, df, method):
-        """
-        Create a list of tuples of the following:
-            - directory: name of directory where file is located
-            - path: full path to file
-            - filename: name of file
-            - method: whether to compute indices for phenological stages or not
-        This tuple will be used as input to the `process` function
-        :param df:
-        :param method:
-        :return:
-        """
-        combinations = []
-
-        for index, row in tqdm(df.iterrows()):
-            combinations.extend(
-                list(
-                    itertools.product([row[0]], [row[1]], [row[2]], [row[3]], [method])
-                )
-            )
-
-        combinations = remove_duplicates(combinations)
-
-        return combinations
-
-    def main(self, method):
-        """
-
-        :param method:
-        :return:
-        """
-        # Create a dataframe of the files to be analyzed
-        df_files = self.collect_files()
-
-        combinations = self.process_combinations(df_files, method)
-
-        # Add an element to the tuple to indicate the season
-        # Last element is redo flag which is True if the analysis is to be redone
-        # and False otherwise. Analysis is always redone for the current year
-        # and last year whether file exists or not
-        combinations = [
-            (
-                self.parser,
-                status,
-                path,
-                filename,
-                admin_zone,
-                category,
-                year,
-                "ndvi",
-                False,  # redo
-            )
-            for year in range(2001, ar.utcnow().year + 1)
-            for status, path, filename, admin_zone, category in combinations
-        ]
-
-        # Only keep those entries in combinations where the third elemt is
-        # mozambique, south_africa, angola or dem_people's_rep_of_korea
-        # This is done to test the code for these countries
-        combinations = [i for i in combinations if f"{country}_winter_wheat_s1" in i[3]]
-
-        if True:
-            num_cpu = int(cpu_count() * 0.9)
-            with Pool(num_cpu) as p:
-                for i, _ in enumerate(p.imap_unordered(indices.process, combinations)):
-                    pass
-        else:
-            # Use the code below if you want to test without parallelization or
-            # if you want to debug by using pdb
-            pbar = tqdm(combinations)
-            for i, val in enumerate(pbar):
-                pbar.set_description(
-                    f"Main loop {combinations[i][2]} {combinations[i][5]}"
-                )
-                indices.process(val)
-
-
-def run(path_config_files=[]):
-    """
-
-    Args:
-        path_config_files:
-
-    Returns:
-
-    """
-    """ Check dictionary keys to have no spaces"""
-    indices.validate_index_definitions()
-
-    for method in [
-        "monthly_r",  # "dekad_r"  # "dekad_r"
-    ]:  # , "full_season", "phenological_stages", "fraction_season"]:
-        obj = cei_runner(path_config_files)
-        obj.main(method)
-
-
-if __name__ == "__main__":
-    run()