oda-data 2.8.0__tar.gz → 2.8.2__tar.gz

{oda_data-2.8.0 → oda_data-2.8.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: oda_data
-Version: 2.8.0
+Version: 2.8.2
 Summary: A python package to work with Official Development Assistance data from the OECD DAC.
 Author: Jorge Rivera, Miguel Haro Ruiz
 Author-email: Jorge Rivera <jorge.rivera@one.org>, Miguel Haro Ruiz <miguel.haroruiz@one.org>

{oda_data-2.8.0 → oda_data-2.8.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "oda_data"
-version = "2.8.0"
+version = "2.8.2"
 description = "A python package to work with Official Development Assistance data from the OECD DAC."
 requires-python = ">=3.11"
 readme = "README.md"

{oda_data-2.8.0 → oda_data-2.8.2}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "oda_data"
-version = "2.8.0"
+version = "2.8.2"
 description = "A python package to work with Official Development Assistance data from the OECD DAC."
 requires-python = ">=3.11"
 

{oda_data-2.8.0 → oda_data-2.8.2}/src/oda_data/api/constants.py

@@ -13,7 +13,14 @@ MarkerScore = Literal[
 ]
 
 PolicyMarker = Literal[
-    "gender", "environment", "nutrition", "disability", "biodiversity"
+    "gender",
+    "environment",
+    "climate_adaptation",
+    "climate_mitigation",
+    "biodiversity",
+    "desertification",
+    "disability",
+    "nutrition",
 ]
 
 CURRENCIES: dict[str, str] = {

{oda_data-2.8.0 → oda_data-2.8.2}/src/oda_data/api/sources.py

@@ -18,7 +18,6 @@ from oda_data.clean_data.common import (
     convert_dot_stat_to_data_explorer_codes,
 )
 from oda_data.clean_data.schema import (
-    CORE_CRS_MAPPING,
     AidDataSchema,
     ODASchema,
 )
@@ -42,20 +41,6 @@ from oda_data.tools.cache import (
 )
 
 
-def translate_cols_and_filters_to_raw(
-    columns: list | None, filters: list[tuple] | None
-) -> tuple[list | None, list[tuple] | None]:
-    crs_mapping = {v: k for k, v in CORE_CRS_MAPPING.items()}
-    cols = [crs_mapping.get(col, col) for col in columns] if columns else columns
-    filters = (
-        [(crs_mapping.get(col, col), op, val) for col, op, val in filters]
-        if filters
-        else None
-    )
-
-    return cols, filters
-
-
 def _filters_to_query(filters: list[tuple[str, str, list]]) -> str:
     """Transform a list of filters into a query string.
 

{oda_data-2.8.0 → oda_data-2.8.2}/src/oda_data/indicators/crs/crs_functions.py

@@ -36,15 +36,20 @@ def _group_by_mapped_channel(df: pd.DataFrame) -> pd.DataFrame:
 def _rolling_period_total(df: pd.DataFrame, period_length: int = 3) -> pd.DataFrame:
     """Calculate a rolling total of `period_length` years.
 
-    Thin wrapper around the shared, vectorised `rolling_window_total`
-    (`oda_data.indicators.research.imputation_shares`), which replaced this
-    function's own year-by-year deep-copy/concat loop. A (grouper, year)
-    combination is only emitted once its group has a full `period_length`-
-    year window within the years present in `df`.
+    Wraps the shared, vectorised `rolling_window_total`
+    (`oda_data.indicators.research.imputation_shares`). A (grouper, year)
+    combination is emitted once its group has a full `period_length`-year
+    window within the years present in `df`, and only when some value in that
+    window is nonzero. `rolling_window_total` fills a window in which the group
+    has no rows with zeros, and those rows would otherwise become zero-share
+    rows for years before or after the group existed.
     """
     values = list(crs_value_cols().values())
 
-    return rolling_window_total(df, period_length=period_length, value_cols=values)
+    totals = rolling_window_total(df, period_length=period_length, value_cols=values)
+    has_value = totals[values].fillna(0).ne(0).any(axis=1)
+
+    return totals.loc[has_value].reset_index(drop=True)
 
 
 def _purpose_share(value_row: pd.DataFrame, value_col: str) -> pd.Series:

{oda_data-2.8.0 → oda_data-2.8.2}/src/oda_data/indicators/research/policy_markers.py

@@ -1,5 +1,6 @@
+from typing import get_args
+
 import pandas as pd
-from pyarrow import ArrowInvalid
 
 from oda_data.api.constants import (
     EXTENDED_PROVIDER_PURPOSE_GROUPER,
@@ -8,18 +9,35 @@ from oda_data.api.constants import (
     Measure,
     PolicyMarker,
 )
-from oda_data.api.sources import translate_cols_and_filters_to_raw
 from oda_data.clean_data.common import convert_units
 from oda_data.clean_data.schema import ODASchema
 
-
-def _marker_score_map(marker: PolicyMarker | str) -> list[int] | None:
-    return {
-        "significant": [1],
-        "principal": [2],
-        "not_targeted": [0],
-        "total_targeted": [1, 2],
-    }.get(marker)
+# CRS marker values kept by each score. Desertification also reports 3, a principal
+# objective that supports an action programme to combat desertification. The
+# not_screened and total_allocable scores have no values: not_screened keeps blank
+# markers and total_allocable keeps every row.
+_MARKER_SCORE_VALUES: dict[str, list[int] | None] = {
+    "significant": [1],
+    "principal": [2, 3],
+    "not_targeted": [0],
+    "total_targeted": [1, 2, 3],
+    "not_screened": None,
+    "total_allocable": None,
+}
+
+
+def _marker_score_map(marker_score: MarkerScore | str) -> list[int] | None:
+    """Return the CRS marker values a score keeps, or None if it filters no values.
+
+    Raises:
+        ValueError: If the score is not a MarkerScore.
+    """
+    if marker_score not in _MARKER_SCORE_VALUES:
+        raise ValueError(
+            f"Unknown marker_score {marker_score!r}; "
+            f"expected one of {list(_MARKER_SCORE_VALUES)}"
+        )
+    return _MARKER_SCORE_VALUES[marker_score]
 
 
 def _marker_modality_filter() -> list[tuple[str, str, list]]:
@@ -63,19 +81,30 @@ def bilateral_policy_marker(
         providers (list | int | None, optional): Providers to filter the data. Defaults to None.
         recipients (list | int | None, optional): Recipients to filter the data. Defaults to None.
         measure (Measure | str, optional): Measure type. Defaults to "gross_disbursement".
-        marker (PolicyMarker): Policy marker. Options are: "gender", "environment", "nutrition",
-           "disability", "biodiversity"
-        marker_score (MarkerScore): Policy marker score.
-          Options are: "significant", "principal", "not_targeted", "not_screened", "total_targeted", "total_allocable".
-        oda_only (bool, optional): Whether to include only ODA-related data. Defaults to False.
+        marker (PolicyMarker): Policy marker. Options are: "gender", "environment",
+           "climate_adaptation", "climate_mitigation", "biodiversity", "desertification",
+           "disability", "nutrition".
+        marker_score (MarkerScore): Policy marker score. "principal", "significant",
+          "not_targeted" and "not_screened" (a blank marker) split allocable aid with no
+          overlap. "total_targeted" is principal plus significant, and "total_allocable"
+          is all allocable aid, screened or not.
+        oda_only (bool, optional): Whether to include only ODA flows. Defaults to True.
         currency (str, optional): Target currency. Defaults to "USD".
         base_year (int | None, optional): Base year for conversion. Defaults to None.
 
     Returns:
         pd.DataFrame: Dataframe with spending by purpose.
+
+    Raises:
+        ValueError: If the marker or the score is not one of the options above.
     """
     from oda_data.api.sources import CRSData
 
+    if marker not in get_args(PolicyMarker):
+        raise ValueError(
+            f"Unknown marker {marker!r}; expected one of {list(get_args(PolicyMarker))}"
+        )
+
     # Get the relevant measure
     measure = MEASURES["CRS"][measure]["column"]
 
@@ -108,25 +137,14 @@ def bilateral_policy_marker(
     crs = CRSData(providers=providers, years=years, recipients=recipients)
 
     # Read the data and group by provider and purpose
+    data = crs.read(
+        columns=[*grouper, measure, marker],
+        additional_filters=filters,
+        using_bulk_download=True,
+    )
 
-    cols = [*grouper, measure]
-
-    try:
-        data = crs.read(
-            columns=cols,
-            additional_filters=filters,
-            using_bulk_download=True,
-        )
-    except ArrowInvalid:
-        c, f = translate_cols_and_filters_to_raw(cols, filters)
-        data = crs.read(
-            columns=c,
-            additional_filters=f,
-            using_bulk_download=True,
-        )
-
-    # if marker is not_screened, we need to filter out the screened data
-    if marker == "not_screened":
+    # Keep only unscreened activities (blank marker)
+    if marker_score == "not_screened":
         data = data.loc[lambda d: d[marker].isna()]
 
     data = (

{oda_data-2.8.0 → oda_data-2.8.2}/src/oda_data/indicators/research/sector_imputations.py

@@ -208,15 +208,18 @@ def _read_core_contributions(
 
 
 def _get_channel_share_proxies() -> pd.DataFrame:
-    """Read the reviewed proxy table.
+    """Read the proxy table, keeping only rows marked `reviewed=true`.
 
     Returns:
         pd.DataFrame: columns channel_code, channel_name, proxy_channel_code,
         proxy_type ("parent_fund" or "fixed_purpose"), fixed_purpose_code,
         rationale, reviewed. `proxy_channel_code` is null for `fixed_purpose`
         rows; `fixed_purpose_code` is null for `parent_fund` rows.
+
+    Raises:
+        ValueError: If the table has no `reviewed` column.
     """
-    return pd.read_csv(
+    proxies = pd.read_csv(
         ODAPaths.cleaning / PROXY_TABLE_FILE,
         dtype={
             ODASchema.CHANNEL_CODE: "Int64",
@@ -224,6 +227,12 @@ def _get_channel_share_proxies() -> pd.DataFrame:
             "fixed_purpose_code": "Int64",
         },
     )
+    if "reviewed" not in proxies.columns:
+        raise ValueError(
+            f"{PROXY_TABLE_FILE} has no 'reviewed' column, so unreviewed proxies "
+            "cannot be told apart from reviewed ones."
+        )
+    return proxies.loc[proxies["reviewed"].eq(True)].reset_index(drop=True)
 
 
 def _apply_proxies(