oda-data 2.7.0__tar.gz → 2.7.1__tar.gz

{oda_data-2.7.0 → oda_data-2.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: oda_data
-Version: 2.7.0
+Version: 2.7.1
 Summary: A python package to work with Official Development Assistance data from the OECD DAC.
 Author: Jorge Rivera, Miguel Haro Ruiz
 Author-email: Jorge Rivera <jorge.rivera@one.org>, Miguel Haro Ruiz <miguel.haroruiz@one.org>

oda_data-2.7.1/pyproject.toml

@@ -0,0 +1,217 @@
+[project]
+name = "oda_data"
+version = "2.7.1"
+description = "A python package to work with Official Development Assistance data from the OECD DAC."
+requires-python = ">=3.11"
+readme = "README.md"
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3 :: Only",
+    "Topic :: Scientific/Engineering",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+dependencies = [
+    "cachetools>=6.2.0",
+    "filelock>=3.20.0",
+    "oda-reader>=1.6.0",
+    "platformdirs>=4.0.0",
+    "pandas[excel,parquet,performance]>=2.2.0",
+    "pydeflate>=2.3.3",
+    "reader>=3.18",
+    "thefuzz>=0.22.1",
+]
+
+[project.license]
+text = "MIT"
+
+[[project.authors]]
+name = "Jorge Rivera"
+email = "jorge.rivera@one.org"
+
+[[project.authors]]
+name = "Miguel Haro Ruiz"
+email = "miguel.haroruiz@one.org"
+
+[project.urls]
+Homepage = "https://github.com/ONEcampaign/oda_data_package"
+Repository = "https://github.com/ONEcampaign/oda_data_package"
+Documentation = "https://github.com/ONEcampaign/oda_data_package"
+
+[build-system]
+requires = ["uv_build>=0.11.8,<0.12"]
+build-backend = "uv_build"
+
+[dependency-groups]
+dev = [
+    "pre-commit>=4.5.0",
+    "ruff>=0.14.9",
+    "ty>=0.0.33",
+    "mkdocs-material>=9.7.0",
+]
+test = [
+    "pytest>=8.0.0",
+    "pytest-mock>=3.12.0",
+    "pytest-cov>=4.1.0",
+    "pytest-xdist>=3.5.0",
+    "freezegun>=1.4.0",
+]
+
+[tool.ruff]
+line-length = 88
+target-version = "py311"
+extend-include = ["*.ipynb"]
+exclude = [
+    ".git",
+    ".venv",
+    "venv",
+    "__pycache__",
+    "build",
+    "dist",
+    "*.egg-info",
+    ".pytest_cache",
+    "htmlcov",
+]
+
+[tool.ruff.lint]
+select = [
+    "E",
+    "W",
+    "F",
+    "I",
+    "N",
+    "UP",
+    "B",
+    "C4",
+    "SIM",
+    "PL",
+    "RUF",
+    "ERA",
+    "PERF",
+    "ANN",
+]
+ignore = [
+    "E501",
+    "PLR0913",
+    "PLR0912",
+    "PLR0914",
+    "PLR0915",
+    "PLR2004",
+    "PLC0415",
+    "ANN401",
+]
+fixable = ["ALL"]
+unfixable = []
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**/*.py" = [
+    "ANN",
+    "E501",
+    "PLR2004",
+    "RUF002",
+    "RUF003",
+    "PLW0603",
+    "ERA001",
+]
+"src/oda_data/__init__.py" = [
+    "PLW0603",
+    "PLW0602",
+    "F811",
+]
+"src/oda_data/cache/config.py" = ["PLW0603"]
+"src/oda_data/cache/api.py" = ["PLW0603"]
+"src/oda_data/tools/cache.py" = ["PLW2901"]
+"src/oda_data/api/oecd.py" = ["PLR0912"]
+"src/oda_data/indicators/crs/crs_functions.py" = ["B023"]
+"src/oda_data/indicators/research/sector_imputations.py" = ["B023"]
+"src/oda_data/tools/sector_lists.py" = ["N816"]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"
+skip-magic-trailing-comma = false
+line-ending = "auto"
+
+[tool.ty.environment]
+python-version = "3.11"
+
+[tool.ty.src]
+include = ["src/oda_data"]
+
+[tool.ty.analysis]
+replace-imports-with-any = [
+    "pandas",
+    "pandas.**",
+    "numpy",
+    "numpy.**",
+]
+
+[tool.ty.rules]
+possibly-missing-attribute = "ignore"
+possibly-unresolved-reference = "ignore"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+addopts = """
+    -v
+    --strict-markers
+    --tb=short
+    --cov=src/oda_data
+    --cov-report=term-missing
+    --cov-report=html
+    --cov-branch
+    -m "not integration and not network"
+"""
+markers = [
+    """slow: marks tests as slow (deselect with '-m "not slow"')""",
+    "integration: marks tests that call real OECD API (deselected by default)",
+    "network: marks tests requiring live network access (opt-in via RUN_NETWORK_TESTS=1; deselected by default)",
+    "unit: marks tests as unit tests",
+    "cache: marks tests related to caching functionality",
+    "validation: marks tests related to input validation",
+    "cleaning: marks tests related to data cleaning",
+]
+norecursedirs = [
+    ".git",
+    ".venv",
+    "venv",
+    "__pycache__",
+    "*.egg-info",
+    ".pytest_cache",
+    "htmlcov",
+]
+
+[tool.coverage.run]
+source = ["src/oda_data"]
+branch = true
+omit = [
+    "*/tests/*",
+    "*/__init__.py",
+    "*/config.py",
+    "*/logger.py",
+]
+
+[tool.coverage.report]
+precision = 2
+show_missing = true
+exclude_lines = [
+    "pragma: no cover",
+    "def __repr__",
+    "raise AssertionError",
+    "raise NotImplementedError",
+    "if __name__ == .__main__.:",
+    "if TYPE_CHECKING:",
+    "@abstractmethod",
+]

oda_data-2.7.0/pyproject.toml → oda_data-2.7.1/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "oda_data"
-version = "2.7.0"
+version = "2.7.1"
 description = "A python package to work with Official Development Assistance data from the OECD DAC."
 requires-python = ">=3.11"
 

{oda_data-2.7.0 → oda_data-2.7.1}/src/oda_data/api/sources.py

@@ -308,6 +308,17 @@ class DACSource(Source):
             )
             self.de_sectors = check_strings(sectors)
 
+    def _extra_hash_components(self) -> list[tuple]:
+        """Extra (name, predicate, value) tuples folded into the cache-key hash.
+
+        They bypass the read-filter path, never reaching a parquet predicate
+        or a query string. Folding them into the hash lets a constructor
+        flag that changes row-level behaviour, without being expressed as a
+        filter tuple, still produce a different cache key. Subclasses
+        override this; the default is a no-op.
+        """
+        return []
+
     def _get_filtered_download_filters(self) -> dict:
         """Generates a dictionary of filters for downloading the dataset.
 
@@ -438,7 +449,9 @@ class DACSource(Source):
         """
         # Create filters and generate cache key
         filters = self._get_read_filters(additional_filters=additional_filters)
-        param_hash = generate_param_hash(filters if filters else [])
+        param_hash = generate_param_hash(
+            (filters if filters else []) + self._extra_hash_components()
+        )
 
         # When refresh=True, skip the upper cache tiers entirely so the bulk
         # fetcher's refresh=True actually re-downloads and the upper tiers see
@@ -640,6 +653,7 @@ class CRSData(DACSource):
         years: list[int] | range | int | None = None,
         providers: list[int] | int | None = None,
         recipients: list[int] | int | None = None,
+        exclude_multilateral_core: bool = True,
     ) -> None:
         """
         Initialize CRSData.
@@ -648,9 +662,17 @@ class CRSData(DACSource):
             years (list[int] | range): List or range of years to filter.
             providers (Optional[list[int]]): List of provider codes.
             recipients (Optional[list[int]]): List of recipient codes.
+            exclude_multilateral_core (bool): Whether to exclude CRS rows that report
+                a donor's core (unearmarked) contribution to a multilateral
+                organisation (``bi_multi == 2``). The OECD's CRS changelog states
+                these rows must be excluded to avoid double-counting bilateral
+                flows. Defaults to True. Rows with a missing ``bi_multi`` value
+                are always kept. Pass False to get the raw CRS totals including
+                those core-contribution rows.
         """
         super().__init__()
         self._init_filters(years=years, providers=providers, recipients=recipients)
+        self.exclude_multilateral_core = exclude_multilateral_core
         self._param_hash = None
 
     def _create_bulk_fetcher(self) -> Callable[[Path], None]:
@@ -660,6 +682,55 @@ class CRSData(DACSource):
         """
         return create_crs_bulk_fetcher()
 
+    def _extra_hash_components(self) -> list[tuple]:
+        return [("exclude_multilateral_core", "==", self.exclude_multilateral_core)]
+
+    def _exclude_core_contributions(self, df: pd.DataFrame) -> pd.DataFrame:
+        """Drop rows reporting a donor's core contribution to a multilateral organisation.
+
+        No-op when ``exclude_multilateral_core`` is False, or when the ``bi_multi``
+        column isn't present (e.g. it was excluded from a column projection).
+        Rows with a missing ``bi_multi`` value are kept. A plain ``!= 2`` filter
+        would drop them, since a null comparison is null rather than true.
+        """
+        if not self.exclude_multilateral_core:
+            return df
+        column = self.schema.BI_MULTI
+        if column not in df.columns:
+            return df
+        return df.loc[(df[column] != 2) | df[column].isna()].reset_index(drop=True)
+
+    def _fetch_from_bulk_cache(
+        self,
+        filters: list[tuple] | None,
+        columns: list | None,
+        *,
+        refresh: bool = False,
+    ) -> pd.DataFrame:
+        """Read from the bulk cache, then apply the core-contribution exclusion.
+
+        ``bi_multi`` is read even when not in the requested ``columns``, so the
+        exclusion can be applied before column projection, then dropped again if
+        the caller didn't ask for it.
+        """
+        read_columns = columns
+        added_bi_multi = False
+        if (
+            self.exclude_multilateral_core
+            and columns is not None
+            and self.schema.BI_MULTI not in columns
+        ):
+            read_columns = [*columns, self.schema.BI_MULTI]
+            added_bi_multi = True
+
+        df = super()._fetch_from_bulk_cache(filters, read_columns, refresh=refresh)
+        df = self._exclude_core_contributions(df)
+
+        if added_bi_multi:
+            df = df.drop(columns=[self.schema.BI_MULTI])
+
+        return df
+
     def download(self) -> pd.DataFrame:
         """Downloads CRS data via API (filtered query).
 
@@ -672,7 +743,7 @@ class CRSData(DACSource):
             filters=self._get_filtered_download_filters(),
         )
         logger.info("CRS data downloaded successfully.")
-        return clean_raw_df(df)
+        return self._exclude_core_contributions(clean_raw_df(df))
 
 
 class MultiSystemData(DACSource):

{oda_data-2.7.0 → oda_data-2.7.1}/src/oda_data/indicators/research/sector_imputations.py

@@ -145,9 +145,15 @@ def spending_by_purpose(
     oda_only: bool = False,
     currency: str = "USD",
     base_year: int | None = None,
+    exclude_multilateral_core: bool = True,
 ) -> pd.DataFrame:
     """Retrieves and processes spending data by purpose.
 
+    By default, CRS rows that report a donor's core (unearmarked) contribution to a
+    multilateral organisation (`bi_multi == 2`) are excluded. Those contributions are
+    redistributed across purposes separately, by `imputed_multilateral_by_purpose`.
+    A caller that leaves them in this bilateral total would double count them.
+
     Args:
         years (list | int | range, optional): Years to filter the data. Defaults to None.
         providers (list | int | None, optional): Providers to filter the data. Defaults to None.
@@ -155,6 +161,10 @@ def spending_by_purpose(
         oda_only (bool, optional): Whether to include only ODA-related data. Defaults to False.
         currency (str, optional): Target currency. Defaults to "USD".
         base_year (int | None, optional): Base year for conversion. Defaults to None.
+        exclude_multilateral_core (bool, optional): Whether to exclude CRS rows
+            reporting a donor's core contribution to a multilateral organisation
+            (`bi_multi == 2`). Defaults to True. Pass False for a total that
+            includes those core-contribution rows.
 
     Returns:
         pd.DataFrame: Dataframe with spending by purpose.
@@ -175,7 +185,11 @@ def spending_by_purpose(
     filters = [("category", "in", [10, 60])] if oda_only else []
 
     # Set up the CRS data object
-    crs = CRSData(providers=providers, years=years)
+    crs = CRSData(
+        providers=providers,
+        years=years,
+        exclude_multilateral_core=exclude_multilateral_core,
+    )
 
     # Read the data and group by provider and purpose
     data = (
@@ -340,6 +354,14 @@ def imputed_multilateral_by_purpose(
 ) -> pd.DataFrame:
     """Computes imputed multilateral spending by purpose.
 
+    The multilateral spending shares this imputes onto (via
+    `multilateral_spending_shares_by_channel_and_purpose_smoothed`, which calls
+    `spending_by_purpose`) exclude CRS rows reporting a donor's core contribution to a
+    multilateral organisation (`bi_multi == 2`) by default. Those core contributions
+    are already captured here, on the `core_multilateral_contributions_by_provider`
+    leg, via MultiSystem data. Including them on the CRS leg too would double count
+    them.
+
     Args:
         years (list | int | range, optional): Years to filter the data. Defaults to None.
         providers (list | int | None, optional): Providers to filter the data. Defaults to None.