core-lens 0.1.dev189__tar.gz → 0.1.dev191__tar.gz

{core_lens-0.1.dev189 → core_lens-0.1.dev191}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: core-lens
-Version: 0.1.dev189
+Version: 0.1.dev191
 Summary: Query, analyse, and visualise CoreStack's microwatershed and Earth science data through a clean, composable Python API.
 Project-URL: Homepage, https://github.com/ApoorvaKashyap/core-lens
 Project-URL: Issues, https://github.com/ApoorvaKashyap/core-lens/issues

{core_lens-0.1.dev189 → core_lens-0.1.dev191}/src/core_lens/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.1.dev189'
-__version_tuple__ = version_tuple = (0, 1, 'dev189')
+__version__ = version = '0.1.dev191'
+__version_tuple__ = version_tuple = (0, 1, 'dev191')
 
 __commit_id__ = commit_id = None

{core_lens-0.1.dev189 → core_lens-0.1.dev191}/src/core_lens/base/namespaces/stats.py

@@ -741,6 +741,8 @@ class StatsNamespace:
         }
         return self._r._replace(data=data, has_geometry=False, metadata=metadata)
 
+    import warnings
+
     def anomaly(
         self,
         column: str,
@@ -760,20 +762,32 @@ class StatsNamespace:
                 used as the reference/"normal" period. Required for timeseries; optional
                 for cross-sectional.
             target_years (tuple[int, int] | None, optional): ``(from_year, to_year)`` inclusive,
-                the years actually scored for anomalies. Timeseries: defaults to "everything
-                after ``baseline[1]``" if omitted (old behaviour preserved). Cross-sectional:
-                defaults to all rows if omitted. Must not overlap ``baseline``.
+                years actually scored. Timeseries: defaults to "everything after
+                ``baseline[1]``" if omitted. Cross-sectional: defaults to all rows.
+                Must not overlap ``baseline``.
             threshold (float, optional): Sigma / score threshold for anomaly flag (default 2.0).
 
         Returns:
             Result: data has ``key_col | anomaly_score | is_anomaly`` (cross-sectional) or
-            ``key_col | year | anomaly_score | is_anomaly`` (timeseries, baseline excluded,
-            restricted to ``target_years`` if given). ``metadata`` includes
-            ``n_entities_dropped`` (entities skipped for insufficient baseline obs).
+            ``key_col | <time_col> | anomaly_score | is_anomaly`` (timeseries, baseline
+            excluded, restricted to ``target_years`` if given). ``<time_col>`` is the
+            finer-grained date/period column when the data is sub-annual, falling back
+            to plain ``year`` for annual data. ``metadata`` includes ``n_entities_dropped``.
 
         Raises:
             ValueError: If ``mode``/``method`` invalid, observation count too low, or
                 ``baseline`` and ``target_years`` overlap.
+
+        Notes:
+            **STL fitting scope.** For ``AnomalyTsMethod.STL`` the decomposition
+            (trend + seasonal + remainder) is fit on the *entire* series of each entity,
+            baseline and target years together. ``baseline`` is used only to calibrate the scale (standard deviation) of the remainder; ``target_years`` selects which points are scored. Scores are ``remainder / baseline_remainder_std``.
+
+            Because target observations take part in the fit, the trend and seasonal components can partly absorb target-period anomalies, shrinking the remainder and lowering the score. The effect grows with the size and duration of the anomaly; a sustained shift can be absorbed almost entirely into the trend. Scores therefore measure deviation from a locally fitted trend and seasonal pattern, not from the baseline's "normal" behaviour, and the method is conservative: it may miss anomalies but is unlikely to over-flag them.
+
+            A stricter alternative is to fit on the baseline only and forecast the target period (e.g. ``statsmodels.tsa.forecasting.stl.STLForecast``), scoring forecast errors against the baseline error spread. This is not implemented.
+
+            For sub-annual data, residuals are matched to rows by the finer time column (``date``/``period``), not by year, so each observation gets its own score.
         """
         if baseline is not None and target_years is not None:
             b_lo, b_hi = baseline
@@ -784,8 +798,6 @@ class StatsNamespace:
                     f"target_years={target_years} overlap. They must be disjoint."
                 )
 
-        # Single explicit collect boundary — everything below is eager by necessity
-        # (numpy/scipy/statsmodels have no lazy path).
         df = self._r.df()
         if isinstance(df, pl.LazyFrame):
             df = df.collect()
@@ -819,7 +831,6 @@ class StatsNamespace:
                     f"but only got {len(ref_vals)}."
                 )
 
-            # target_years scopes which rows get scored/returned; baseline stays reference-only.
             eval_df = df
             if target_years is not None and has_year:
                 eval_df = df.filter(
@@ -889,7 +900,6 @@ class StatsNamespace:
                     "threshold": threshold,
                 }
 
-            # Build straight from numpy — no .tolist() round-trip.
             data = eval_df.select(pl.col(key)).with_columns(
                 pl.Series("anomaly_score", scores, dtype=pl.Float64),
                 pl.Series("is_anomaly", flags, dtype=pl.Boolean),
@@ -913,6 +923,13 @@ class StatsNamespace:
                     "Ensure data is at annual or sub_annual resolution."
                 )
 
+            # Finer-grained time column for sub-annual data, if present.
+            # `year_col` still drives baseline/target-year windowing regardless.
+            date_col = next(
+                (c for c in ("date", "period", "period_start") if c in df.columns), None
+            )
+            output_time_col = date_col or year_col
+
             min_obs = _MIN_OBS[method.value]
             n_dropped = 0
 
@@ -963,14 +980,15 @@ class StatsNamespace:
                     .with_columns(
                         (pl.col("anomaly_score").abs() > threshold).alias("is_anomaly")
                     )
-                    .select(key, year_col, "anomaly_score", "is_anomaly")
+                    .select(key, output_time_col, "anomaly_score", "is_anomaly")
                     .drop_nulls("anomaly_score")
                 )
 
-            # ---- CUSUM / STL: inherently sequential/per-entity, keep loop but partition_by ----
+            # ---- CUSUM / STL: inherently sequential/per-entity, partition_by instead of filter-loop ----
             else:
                 rows: list[dict[str, Any]] = []
-                for sub in df.sort(year_col).partition_by(key, maintain_order=True):
+                sort_cols = list(dict.fromkeys([year_col, output_time_col]))
+                for sub in df.sort(sort_cols).partition_by(key, maintain_order=True):
                     eid = sub[key][0]
                     base_vals = (
                         sub.filter(
@@ -992,10 +1010,15 @@ class StatsNamespace:
                         )
                     else:
                         eval_sub = sub.filter(pl.col(year_col) > baseline[1])
-                    eval_vals = eval_sub[column].drop_nulls().to_numpy().astype(float)
-                    eval_years = eval_sub.filter(pl.col(column).is_not_null())[
+                    eval_sub = eval_sub.filter(pl.col(column).is_not_null())
+
+                    eval_vals = eval_sub[column].to_numpy().astype(float)
+                    eval_years_lookup = eval_sub[
                         year_col
-                    ].to_list()
+                    ].to_list()  # for baseline/dict matching
+                    eval_times = eval_sub[
+                        output_time_col
+                    ].to_list()  # for display in output
 
                     ts_scores: list[float] = []
                     ts_flags: list[bool] = []
@@ -1013,43 +1036,56 @@ class StatsNamespace:
                             ts_flags.append(s > h)
 
                     else:  # STL
+                        full = sub.filter(pl.col(column).is_not_null())
+                        full_vals = full[column].to_numpy().astype(float)
+                        full_years = full[year_col].to_list()
+                        full_times = full[output_time_col].to_list()
+
+                        period = (
+                            26 if len(full_vals) >= 52 else max(2, len(full_vals) // 2)
+                        )
+                        min_required = 2 * period
+                        min_reliable = (
+                            3 * period
+                        )  # want ~3 cycles minimum for a trustworthy seasonal fit, not just 2
+
+                        # if len(full_vals) < min_reliable:
+                        #     warnings.warn(
+                        #         f"StatsNamespace.anomaly: entity {eid!r} has {len(full_vals)} points, "
+                        #         f"period={period} needs >= {min_reliable} for a reliable STL fit. Skipping."
+                        #     )
+                        #     n_dropped += 1
+                        #     continue
+
+                        if len(full_vals) < min_required:
+                            warnings.warn(
+                                f"StatsNamespace.anomaly: entity {eid!r} has {len(full_vals)} points, "
+                                f"needs >= {min_required} for period={period}. Skipping."
+                            )
+                            n_dropped += 1
+                            continue
+
                         try:
                             from statsmodels.tsa.seasonal import STL  # type: ignore[import-untyped]
 
-                            full = sub.filter(pl.col(column).is_not_null())
-                            full_vals = full[column].to_numpy().astype(float)
-                            full_years = full[year_col].to_list()
-                            if len(full_vals) < min_obs:
-                                n_dropped += 1
-                                continue
-
-                            # Seasonal period: sub-annual cadence assumed 24 steps/year
-                            # (e.g. fortnightly). Falls back to half the series length
-                            # for shorter records. Adjust if your data's cadence differs.
-                            period = (
-                                24
-                                if len(full_vals) >= 24
-                                else max(2, len(full_vals) // 2)
-                            )
                             res = STL(full_vals, period=period).fit()
                             resid = res.resid
 
-                            # Match residuals to years EXPLICITLY (not positional slicing) —
-                            # positional slicing silently misaligns if there are gaps between
-                            # baseline and eval rows.
-                            resid_by_year = dict(zip(full_years, resid))
+                            # Key residuals by the fine-grained time column (date/period).
+                            # Keying by year collapses ~26 fortnights into one value.
+                            resid_by_time = dict(zip(full_times, resid))
                             base_resid = [
-                                resid_by_year[y]
-                                for y in full_years
+                                r
+                                for y, r in zip(full_years, resid)
                                 if baseline[0] <= y <= baseline[1]
                             ]
                             std = _sf(pl.Series(base_resid).std(ddof=1)) or 1.0
 
                             ts_scores = [
-                                float(resid_by_year[y] / std)
-                                if y in resid_by_year
+                                float(resid_by_time[t] / std)
+                                if t in resid_by_time
                                 else float("nan")
-                                for y in eval_years
+                                for t in eval_times
                             ]
                             ts_flags = [
                                 abs(s) > threshold if s == s else False
@@ -1060,31 +1096,32 @@ class StatsNamespace:
                             warnings.warn(
                                 f"StatsNamespace.anomaly: STL fit failed for entity {eid!r}: {e}"
                             )
-                            ts_scores = [float("nan")] * len(eval_years)
-                            ts_flags = [False] * len(eval_years)
+                            ts_scores = [float("nan")] * len(eval_years_lookup)
+                            ts_flags = [False] * len(eval_years_lookup)
 
-                    for yr, sc, fl in zip(eval_years, ts_scores, ts_flags):
+                    for t, sc, fl in zip(eval_times, ts_scores, ts_flags):
                         rows.append(
                             {
                                 key: eid,
-                                year_col: yr,
+                                output_time_col: t,
                                 "anomaly_score": float(sc),
                                 "is_anomaly": bool(fl),
                             }
                         )
 
-                data = (
-                    pl.DataFrame(rows)
-                    if rows
-                    else pl.DataFrame(
+                if rows:
+                    data = pl.DataFrame(rows)
+                else:
+                    data = pl.DataFrame(
                         {
                             key: pl.Series([], dtype=pl.String),
-                            year_col: pl.Series([], dtype=pl.Int32),
+                            output_time_col: pl.Series(
+                                [], dtype=pl.Date if date_col else pl.Int32
+                            ),
                             "anomaly_score": pl.Series([], dtype=pl.Float64),
                             "is_anomaly": pl.Series([], dtype=pl.Boolean),
                         }
                     )
-                )
 
             global_base_vals = (
                 df.filter(pl.col(year_col).is_between(baseline[0], baseline[1]))[column]