teehr 0.5.1.dev10__tar.gz → 0.5.3__tar.gz

{teehr-0.5.1.dev10 → teehr-0.5.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: teehr
-Version: 0.5.1.dev10
+Version: 0.5.3
 Summary: Tools for Exploratory Evaluation in Hydrologic Research
 License: GNU v3
 License-File: LICENSE.md
@@ -34,7 +34,7 @@ Requires-Dist: psutil (>=7.0.0,<8)
 Requires-Dist: pyarrow (>=15.0.0,<21)
 Requires-Dist: pydantic (>=2.4.2,<3)
 Requires-Dist: pyiceberg (>=0.9.1,<1)
-Requires-Dist: pyspark[pandas-on-spark] (>=4,<5)
+Requires-Dist: pyspark[pandas-on-spark] (==4.0.1)
 Requires-Dist: rasterio (>=1.3.0,<2)
 Requires-Dist: rioxarray (>=0.15.0,<1)
 Requires-Dist: s3fs (>=2023.10.0,<2026)

{teehr-0.5.1.dev10 → teehr-0.5.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "teehr"
-version = "0.5.1dev10"
+version = "0.5.3"
 description = "Tools for Exploratory Evaluation in Hydrologic Research"
 authors = [
     "RTI International",
@@ -29,7 +29,7 @@ h5py = ">=3.12.1,<4"
 pyarrow = ">=15.0.0,<21"
 httpx = ">=0.25.1,<1"
 pandas = ">=2.2.0,<3"
-pyspark = {extras = ["pandas-on-spark"], version = ">=4,<5"}
+pyspark = {extras = ["pandas-on-spark"], version = "4.0.1"}
 dataretrieval = ">=1.0.9,<2"
 numba = ">=0.60.0,<1"
 arch = ">=7.0.0,<8"

{teehr-0.5.1.dev10 → teehr-0.5.3}/src/teehr/__init__.py

@@ -1,7 +1,7 @@
 """Initialize the TEEHR package."""
 import warnings
 
-__version__ = "0.5.1dev10"
+__version__ = "0.5.3"
 
 with warnings.catch_warnings():
     warnings.simplefilter("ignore", UserWarning)

{teehr-0.5.1.dev10 → teehr-0.5.3}/src/teehr/evaluation/fetch.py

@@ -911,8 +911,7 @@ class Fetch:
             ),
             timeseries_type=timeseries_type,
             write_mode=write_mode,
-            drop_duplicates=drop_duplicates,
-            drop_overlapping_assimilation_values=drop_overlapping_assimilation_values  # noqa
+            drop_duplicates=drop_duplicates
         )
 
     def nwm_operational_grids(
@@ -1215,6 +1214,5 @@ class Fetch:
             in_path=Path(self.nwm_cache_dir),
             timeseries_type=timeseries_type,
             write_mode=write_mode,
-            drop_duplicates=drop_duplicates,
-            drop_overlapping_assimilation_values=drop_overlapping_assimilation_values  # noqa
+            drop_duplicates=drop_duplicates
         )

{teehr-0.5.1.dev10 → teehr-0.5.3}/src/teehr/evaluation/metrics.py

@@ -239,11 +239,34 @@ class Metrics:
         """
         for model in include_metrics:
             if model.reference_configuration is not None:
+                """
                 self.df = self._calculate_metric_skill_score(
                     model.output_field_name,
                     model.reference_configuration,
                     group_by
                 )
+                """
+                # 1) get the original cols ahead of skill score join
+                original_cols = self.df.columns
+                # 2) calculate skill score sdf
+                sdf = self._calculate_metric_skill_score(
+                    model.output_field_name,
+                    model.reference_configuration,
+                    group_by
+                )
+                # 3) remove original metric column from skill score sdf
+                sdf = sdf.drop(model.output_field_name)
+                # 3) get join columns
+                join_cols = parse_fields_to_list(group_by)
+                # 4) join returned table back to self.df, trim
+                self.df = self.df.join(
+                    sdf,
+                    on=join_cols,
+                    how="left"
+                ).select(
+                    *original_cols,
+                    F.col(f"{model.output_field_name}_skill_score")
+                )
 
             if model.unpack_results:
                 self.df = model.unpack_function(
@@ -292,11 +315,20 @@ class Metrics:
             temp_col = f"{config}_{metric_field}_skill"
             pivot_sdf = pivot_sdf.withColumn(
                 temp_col,
-                1 - F.col(config) / F.col(reference_configuration)
+                1 - F.try_divide(F.col(config), F.col(reference_configuration))
             ).withColumn(
                 "configuration_name",
                 F.lit(config)
             )
+            # warn user if try_divide results in nulls (division by zero)
+            null_count = pivot_sdf.filter(F.col(temp_col).isNull()).count()
+            if null_count > 0:
+                logger.warning(
+                    f"Division by zero encountered when calculating skill "
+                    f"score for configuration '{config}' relative to "
+                    f"reference configuration '{reference_configuration}'. "
+                    f"{null_count} null values were produced."
+                )
             # Join skill score values from the pivot table.
             join_cols = group_by_strings + ["configuration_name"]
             sdf = sdf.join(

{teehr-0.5.1.dev10 → teehr-0.5.3}/src/teehr/fetching/nwm/grid_utils.py

@@ -183,7 +183,8 @@ def fetch_and_format_nwm_grids(
     overwrite_output: bool,
     location_id_prefix: Union[str, None],
     variable_mapper: Dict[str, Dict[str, str]],
-    timeseries_type: TimeseriesTypeEnum
+    timeseries_type: TimeseriesTypeEnum,
+    drop_overlapping_assimilation_values: bool
 ):
     """Compute weighted average, grouping by reference time.
 
@@ -243,6 +244,11 @@ def fetch_and_format_nwm_grids(
             Path(output_parquet_dir), f"{ref_time_str}.parquet"
         )
         z_hour_df.sort_values([LOCATION_ID, VALUE_TIME], inplace=True)
+
+        if drop_overlapping_assimilation_values and "assim" in nwm_configuration_name:
+            # Set reference_time to NaT for assimilation values
+            z_hour_df.loc[:, REFERENCE_TIME] = pd.NaT
+
         write_timeseries_parquet_file(
             filepath=parquet_filepath,
             overwrite_output=overwrite_output,

{teehr-0.5.1.dev10 → teehr-0.5.3}/src/teehr/fetching/nwm/nwm_grids.py

@@ -382,5 +382,6 @@ def nwm_grids_to_parquet(
             overwrite_output=overwrite_output,
             location_id_prefix=location_id_prefix,
             variable_mapper=variable_mapper,
-            timeseries_type=timeseries_type
+            timeseries_type=timeseries_type,
+            drop_overlapping_assimilation_values=drop_overlapping_assimilation_values
         )

{teehr-0.5.1.dev10 → teehr-0.5.3}/src/teehr/fetching/nwm/nwm_points.py

@@ -348,5 +348,6 @@ def nwm_to_parquet(
             overwrite_output,
             nwm_version,
             variable_mapper,
-            timeseries_type
+            timeseries_type,
+            drop_overlapping_assimilation_values
         )

{teehr-0.5.1.dev10 → teehr-0.5.3}/src/teehr/fetching/nwm/point_utils.py

@@ -102,7 +102,8 @@ def process_chunk_of_files(
     overwrite_output: bool,
     nwm_version: str,
     variable_mapper: Dict[str, Dict[str, str]],
-    timeseries_type: TimeseriesTypeEnum
+    timeseries_type: TimeseriesTypeEnum,
+    drop_overlapping_assimilation_values: bool
 ):
     """Assemble a table for a chunk of NWM files."""
     location_ids = np.array(location_ids).astype(int)
@@ -156,6 +157,12 @@ def process_chunk_of_files(
         end = f"{end_json[1]}T{end_json[3][1:3]}F{end_json[6][1:]}"
         filename = f"{start}_{end}.parquet"
 
+    if drop_overlapping_assimilation_values and "assim" in configuration:
+        # Set reference_time to NaT for assimilation values
+        df_output = output_table.to_pandas()
+        df_output.loc[:, REFERENCE_TIME] = pd.NaT
+        output_table = pa.Table.from_pandas(df_output, schema=schema)
+
     write_timeseries_parquet_file(
         Path(output_parquet_dir, filename),
         overwrite_output,
@@ -176,7 +183,8 @@ def fetch_and_format_nwm_points(
     overwrite_output: bool,
     nwm_version: str,
     variable_mapper: Dict[str, Dict[str, str]],
-    timeseries_type: TimeseriesTypeEnum
+    timeseries_type: TimeseriesTypeEnum,
+    drop_overlapping_assimilation_values: bool
 ):
     """Fetch NWM point data and save as parquet files.
 
@@ -211,6 +219,12 @@ def fetch_and_format_nwm_points(
         they already exist.  True = overwrite; False = fail.
     nwm_version : str
         Specified NWM version.
+    variable_mapper : Dict[str, Dict[str, str]]
+        A mapping dictionary for variable names and units.
+    timeseries_type : TimeseriesTypeEnum
+        The type of timeseries being processed.
+    drop_overlapping_assimilation_values : bool
+        Whether to drop assimilation values that overlap in value_time.
     """
     output_parquet_dir = Path(output_parquet_dir)
     if not output_parquet_dir.exists():
@@ -241,5 +255,6 @@ def fetch_and_format_nwm_points(
             overwrite_output,
             nwm_version,
             variable_mapper,
-            timeseries_type
+            timeseries_type,
+            drop_overlapping_assimilation_values
         )

{teehr-0.5.1.dev10 → teehr-0.5.3}/src/teehr/loading/s3/clone_from_s3.py

@@ -49,9 +49,22 @@ def subset_the_table(
     """Subset the dataset based on location and start/end time."""
     if table.name == "locations" and primary_location_ids is not None:
         sdf_in = sdf_in.filter(sdf_in.id.isin(primary_location_ids))
-    elif table.name == "location_attributes" and primary_location_ids is not None:
+        # warn user if any primary_location_ids were excluded
+        available_ids = set(
+            sdf_in.select("id").rdd.flatMap(
+                lambda x: x).collect()
+                )
+        missing_ids = set(primary_location_ids) - available_ids
+        if len(missing_ids) > 0:
+            logger.warning(
+                "The following primary_location_ids were not found in the "
+                f"locations table and will be excluded: {missing_ids}"
+            )
+    elif table.name == "location_attributes" and \
+            primary_location_ids is not None:
         sdf_in = sdf_in.filter(sdf_in.location_id.isin(primary_location_ids))
-    elif table.name == "location_crosswalks" and primary_location_ids is not None:
+    elif table.name == "location_crosswalks" and \
+            primary_location_ids is not None:
         sdf_in = sdf_in.filter(
             sdf_in.primary_location_id.isin(primary_location_ids)
         )
@@ -64,7 +77,8 @@ def subset_the_table(
         if primary_location_ids is not None:
             secondary_ids = (
                 ev.location_crosswalks.to_sdf()
-                .select("secondary_location_id").rdd.flatMap(lambda x: x).collect()
+                .select("secondary_location_id").rdd.flatMap(
+                    lambda x: x).collect()
             )
             sdf_in = sdf_in.filter(sdf_in.location_id.isin(secondary_ids))
     elif table.name == "joined_timeseries":
@@ -119,7 +133,6 @@ def clone_from_s3(
 
     Note: future version will allow subsetting the tables to clone.
     """
-
     # Make the Evaluation directories
     logger.info(f"Creating directories for evaluation: {evaluation_name}")
     Path(ev.cache_dir).mkdir()
@@ -174,7 +187,10 @@ def clone_from_s3(
         logger.debug(f"Making directory {table.dir}")
         Path(table.dir).mkdir()
 
-        logger.debug(f"Cloning {table.name} from {s3_dataset_path}/{table.name}/ to {table.dir}")
+        logger.debug(
+            f"Cloning {table.name} from {s3_dataset_path}/{table.name}/ to "
+            f"{table.dir}"
+            )
 
         sdf_in = table._read_files(
             path=f"{s3_dataset_path}/{table.name}/",
@@ -200,7 +216,8 @@ def clone_from_s3(
     dest = f"{ev.scripts_dir}/user_defined_fields.py"
     logger.debug(f"Copying from {source}/ to {dest}")
 
-    # ToDo: there is a permission issue that prevents copying the entire directory.
+    # ToDo: there is a permission issue that prevents copying the entire
+    # directory.
     # This works for now.
     with fsspec.open(source, 'r', anon=True) as file:
         with open(dest, 'w') as f:

{teehr-0.5.1.dev10 → teehr-0.5.3}/src/teehr/loading/timeseries.py

@@ -247,8 +247,7 @@ def validate_and_insert_timeseries(
     timeseries_type: str,
     pattern: str = "**/*.parquet",
     write_mode: TableWriteEnum = "append",
-    drop_duplicates: bool = True,
-    drop_overlapping_assimilation_values: bool = False
+    drop_duplicates: bool = True
 ):
     """Validate and insert primary timeseries data.
 
@@ -272,12 +271,6 @@ def validate_and_insert_timeseries(
     drop_duplicates : bool, optional (default: True)
         Whether to drop duplicates in the dataframe before writing
         to the table.
-    drop_overlapping_assimilation_values: Optional[bool] = True
-        Whether to drop overlapping assimilation values. Default is True.
-        If True, values that overlap in value_time are dropped, keeping those with
-        the most recent reference_time. In this case, all reference_time values
-        are set to None. If False, overlapping values are kept and reference_time
-        is retained.
     """ # noqa
     in_path = Path(in_path)
     logger.info(f"Validating and inserting timeseries data from {in_path}")
@@ -292,9 +285,6 @@ def validate_and_insert_timeseries(
     # Read the converted files to Spark DataFrame
     df = table._read_files(in_path, pattern)
 
-    if drop_overlapping_assimilation_values:
-        df = df.withColumn("reference_time", lit(None))
-
     # Validate using the _validate() method
     validated_df = table._validate(
         df=df,

{teehr-0.5.1.dev10 → teehr-0.5.3}/src/teehr/metrics/deterministic_funcs.py

@@ -18,7 +18,8 @@ def _transform(
         p: pd.Series,
         s: pd.Series,
         model: MetricsBasemodel,
-        t: Optional[pd.Series] = None
+        t: Optional[pd.Series] = None,
+        threshold_series: Optional[pd.Series] = None
 ) -> tuple:
     """Apply timeseries transform for metrics calculations."""
     # Apply transform
@@ -31,25 +32,37 @@ def _transform(
                     )
                     p = p + EPSILON
                     s = s + EPSILON
+                    if threshold_series is not None:
+                        threshold_series = threshold_series + EPSILON
                 logger.debug("Applying log transform")
                 p = np.log(p)
                 s = np.log(s)
+                if threshold_series is not None:
+                    threshold_series = np.log(threshold_series)
             case TransformEnum.sqrt:
                 logger.debug("Applying square root transform")
                 p = np.sqrt(p)
                 s = np.sqrt(s)
+                if threshold_series is not None:
+                    threshold_series = np.sqrt(threshold_series)
             case TransformEnum.square:
                 logger.debug("Applying square transform")
                 p = np.square(p)
                 s = np.square(s)
+                if threshold_series is not None:
+                    threshold_series = np.square(threshold_series)
             case TransformEnum.cube:
                 logger.debug("Applying cube transform")
                 p = np.power(p, 3)
                 s = np.power(s, 3)
+                if threshold_series is not None:
+                    threshold_series = np.power(threshold_series, 3)
             case TransformEnum.exp:
                 logger.debug("Applying exponential transform")
                 p = np.exp(p)
                 s = np.exp(s)
+                if threshold_series is not None:
+                    threshold_series = np.exp(threshold_series)
             case TransformEnum.inv:
                 if model.add_epsilon:
                     logger.debug(
@@ -57,13 +70,19 @@ def _transform(
                     )
                     p = p + EPSILON
                     s = s + EPSILON
+                    if threshold_series is not None:
+                        threshold_series = threshold_series + EPSILON
                 logger.debug("Applying inverse transform")
                 p = 1.0 / p
                 s = 1.0 / s
+                if threshold_series is not None:
+                    threshold_series = 1.0 / threshold_series
             case TransformEnum.abs:
                 logger.debug("Applying absolute value transform")
                 p = np.abs(p)
                 s = np.abs(s)
+                if threshold_series is not None:
+                    threshold_series = np.abs(threshold_series)
             case _:
                 raise ValueError(
                     f"Unsupported transform: {model.transform}"
@@ -71,24 +90,41 @@ def _transform(
     else:
         logger.debug("No transform specified, using original values")
 
-    # Remove invalid values and align series
-    if t is not None:
-        if isinstance(t, pd.Series):
+    # Remove invalid values and align series if transform applied
+    if model.transform is not None:
+        logger.debug("Removing invalid values and aligning series")
+        if (t is not None) and (threshold_series is not None):
+            valid_mask = np.isfinite(p) & np.isfinite(s)
+            p = p[valid_mask]
+            s = s[valid_mask]
+            t = t[valid_mask]
+            threshold_series = threshold_series[valid_mask]
+        elif t is not None:
             valid_mask = np.isfinite(p) & np.isfinite(s)
             p = p[valid_mask]
             s = s[valid_mask]
             t = t[valid_mask]
+        elif threshold_series is not None:
+            valid_mask = np.isfinite(p) & np.isfinite(s)
+            p = p[valid_mask]
+            s = s[valid_mask]
+            threshold_series = threshold_series[valid_mask]
         else:
-            raise TypeError(
-                "t must be a pandas Series, not {type(t)}"
+            valid_mask = np.isfinite(p) & np.isfinite(s)
+            p = p[valid_mask]
+            s = s[valid_mask]
+        logger.debug(
+            f"Removed {len(valid_mask) - np.sum(valid_mask)} invalid entries"
+            " from the transformed input series"
             )
-    else:
-        valid_mask = np.isfinite(p) & np.isfinite(s)
-        p = p[valid_mask]
-        s = s[valid_mask]
 
-    if t is not None:
+    # return results
+    if (t is not None) and (threshold_series is not None):
+        return p, s, t, threshold_series
+    elif t is not None:
         return p, s, t
+    elif threshold_series is not None:
+        return p, s, threshold_series
     else:
         return p, s
 
@@ -659,3 +695,177 @@ def max_value_timedelta(model: MetricsBasemodel) -> Callable:
         return td.total_seconds()
 
     return max_value_timedelta_inner
+
+
+# Categorical Metrics
+def _validate_threshold_field(threshold_series: pd.Series) -> float:
+    """Validate threshold input field."""
+    unique_thresholds = threshold_series.unique()
+    if len(unique_thresholds) != 1:
+        raise ValueError(
+            "Threshold field must contain a single unique value for each"
+            " population grouping."
+        )
+    threshold = unique_thresholds[0]
+    return threshold
+
+
+def confusion_matrix(model: MetricsBasemodel) -> Callable:
+    """Create the confusion_matrix metric function.
+
+    Returns counts of TP, TN, FP, FN as a dictionary.
+
+    :math:`TP=\\sum((prim>=threshold_{prim})\\ and\\ (sec>=threshold_{sec}))`
+    :math:`TN=\\sum((prim<threshold_{prim})\\ and\\ (sec<threshold_{sec}))`
+    :math:`FP=\\sum((prim<threshold_{prim})\\ and\\ (sec>=threshold_{sec}))`
+    :math:`FN=\\sum((prim>=threshold_{prim})\\ and\\ (sec<threshold_{sec}))`
+    """ # noqa
+    logger.debug("Building the confusion_matrix metric function")
+
+    def confusion_matrix_inner(p: pd.Series,
+                               s: pd.Series,
+                               threshold_series: pd.Series) -> dict:
+        """Confusion matrix counts."""
+        p, s, threshold_series = _transform(p,
+                                            s,
+                                            model,
+                                            None,
+                                            threshold_series)
+
+        threshold = _validate_threshold_field(threshold_series)
+
+        tp = np.sum((p >= threshold) & (s >= threshold))
+        tn = np.sum((p < threshold) & (s < threshold))
+        fp = np.sum((p < threshold) & (s >= threshold))
+        fn = np.sum((p >= threshold) & (s < threshold))
+        result = {"TP": tp, "TN": tn, "FP": fp, "FN": fn}
+
+        return result
+
+    return confusion_matrix_inner
+
+
+def false_alarm_ratio(model: MetricsBasemodel) -> Callable:
+    """Create the false_alarm_ratio metric function.
+
+    :math:`FAR=\\frac{FP}{(TP+FP)}`
+    """ # noqa
+    logger.debug("Building the false_alarm_ratio metric function")
+
+    def false_alarm_ratio_inner(p: pd.Series,
+                                s: pd.Series,
+                                threshold_series: pd.Series) -> float:
+        """Calculate False alarm ratio."""
+        p, s, threshold_series = _transform(p,
+                                            s,
+                                            model,
+                                            None,
+                                            threshold_series)
+
+        threshold = _validate_threshold_field(threshold_series)
+
+        tp = np.sum((p >= threshold) & (s >= threshold))
+        fp = np.sum((p < threshold) & (s >= threshold))
+        if (tp + fp) == 0:
+            result = np.nan
+        else:
+            result = fp / (tp + fp)
+
+        return result
+
+    return false_alarm_ratio_inner
+
+
+def probability_of_detection(model: MetricsBasemodel) -> Callable:
+    """Create the probability_of_detection metric function.
+
+    :math:`POD=\\frac{TP}{(TP+FN)}`
+    """ # noqa
+    logger.debug("Building the probability_of_detection metric function")
+
+    def probability_of_detection_inner(p: pd.Series,
+                                       s: pd.Series,
+                                       threshold_series: pd.Series) -> float:
+        """Probability of detection."""
+        p, s, threshold_series = _transform(p,
+                                            s,
+                                            model,
+                                            None,
+                                            threshold_series)
+
+        threshold = _validate_threshold_field(threshold_series)
+
+        tp = np.sum((p >= threshold) & (s >= threshold))
+        fn = np.sum((p >= threshold) & (s < threshold))
+        if (tp + fn) == 0:
+            result = np.nan
+        else:
+            result = tp / (tp + fn)
+
+        return result
+
+    return probability_of_detection_inner
+
+
+def probability_of_false_detection(model: MetricsBasemodel) -> Callable:
+    """Create the probability_of_false_detection metric function.
+
+    :math:`POFD=\\frac{FP}{(FP+TN)}`
+    """ # noqa
+    logger.debug("Building the probability_of_false_detection metric function")
+
+    def probability_of_false_detection_inner(p: pd.Series,
+                                             s: pd.Series,
+                                             threshold_series: pd.Series
+                                             ) -> float:
+        """Probability of false detection."""
+        p, s, threshold_series = _transform(p,
+                                            s,
+                                            model,
+                                            None,
+                                            threshold_series)
+
+        threshold = _validate_threshold_field(threshold_series)
+
+        fp = np.sum((p < threshold) & (s >= threshold))
+        tn = np.sum((p < threshold) & (s < threshold))
+        if (fp + tn) == 0:
+            result = np.nan
+        else:
+            result = fp / (fp + tn)
+
+        return result
+
+    return probability_of_false_detection_inner
+
+
+def critical_success_index(model: MetricsBasemodel) -> Callable:
+    """Create the critical_success_index metric function.
+
+    :math:`CSI=\\frac{TP}{(TP+FP+FN)}`
+    """ # noqa
+    logger.debug("Building the critical_success_index metric function")
+
+    def critical_success_index_inner(p: pd.Series,
+                                     s: pd.Series,
+                                     threshold_series: pd.Series) -> float:
+        """Critical success index."""
+        p, s, threshold_series = _transform(p,
+                                            s,
+                                            model,
+                                            None,
+                                            threshold_series)
+
+        threshold = _validate_threshold_field(threshold_series)
+
+        tp = np.sum((p >= threshold) & (s >= threshold))
+        fp = np.sum((p < threshold) & (s >= threshold))
+        fn = np.sum((p >= threshold) & (s < threshold))
+        if (tp + fp + fn) == 0:
+            result = np.nan
+        else:
+            result = tp / (tp + fp + fn)
+
+        return result
+
+    return critical_success_index_inner