oda-data 2.7.0__tar.gz → 2.8.0__tar.gz

{oda_data-2.7.0 → oda_data-2.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: oda_data
-Version: 2.7.0
+Version: 2.8.0
 Summary: A python package to work with Official Development Assistance data from the OECD DAC.
 Author: Jorge Rivera, Miguel Haro Ruiz
 Author-email: Jorge Rivera <jorge.rivera@one.org>, Miguel Haro Ruiz <miguel.haroruiz@one.org>
@@ -18,12 +18,11 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Dist: cachetools>=6.2.0
 Requires-Dist: filelock>=3.20.0
-Requires-Dist: oda-reader>=1.6.0
+Requires-Dist: oda-reader>=1.9.0
 Requires-Dist: platformdirs>=4.0.0
 Requires-Dist: pandas[excel,parquet,performance]>=2.2.0
 Requires-Dist: pydeflate>=2.3.3
 Requires-Dist: reader>=3.18
-Requires-Dist: thefuzz>=0.22.1
 Requires-Python: >=3.11
 Project-URL: Homepage, https://github.com/ONEcampaign/oda_data_package
 Project-URL: Repository, https://github.com/ONEcampaign/oda_data_package

oda_data-2.8.0/pyproject.toml

@@ -0,0 +1,226 @@
+[project]
+name = "oda_data"
+version = "2.8.0"
+description = "A python package to work with Official Development Assistance data from the OECD DAC."
+requires-python = ">=3.11"
+readme = "README.md"
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3 :: Only",
+    "Topic :: Scientific/Engineering",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+dependencies = [
+    "cachetools>=6.2.0",
+    "filelock>=3.20.0",
+    "oda-reader>=1.9.0",
+    "platformdirs>=4.0.0",
+    "pandas[excel,parquet,performance]>=2.2.0",
+    "pydeflate>=2.3.3",
+    "reader>=3.18",
+]
+
+[project.license]
+text = "MIT"
+
+[[project.authors]]
+name = "Jorge Rivera"
+email = "jorge.rivera@one.org"
+
+[[project.authors]]
+name = "Miguel Haro Ruiz"
+email = "miguel.haroruiz@one.org"
+
+[project.urls]
+Homepage = "https://github.com/ONEcampaign/oda_data_package"
+Repository = "https://github.com/ONEcampaign/oda_data_package"
+Documentation = "https://github.com/ONEcampaign/oda_data_package"
+
+[build-system]
+requires = ["uv_build>=0.11.8,<0.12"]
+build-backend = "uv_build"
+
+[dependency-groups]
+dev = [
+    "pre-commit>=4.5.0",
+    "ruff>=0.14.9",
+    "ty>=0.0.33",
+    "mkdocs-material>=9.7.0",
+]
+test = [
+    "pytest>=8.0.0",
+    "pytest-mock>=3.12.0",
+    "pytest-cov>=4.1.0",
+    "pytest-xdist>=3.5.0",
+    "freezegun>=1.4.0",
+]
+maintenance = [
+    "resolvekit==0.1.12 ; python_version >= '3.12'",
+    "pyyaml>=6.0 ; python_version >= '3.12'",
+]
+
+[tool.uv]
+default-groups = [
+    "dev",
+    "test",
+]
+
+[tool.ruff]
+line-length = 88
+target-version = "py311"
+extend-include = ["*.ipynb"]
+exclude = [
+    ".git",
+    ".venv",
+    "venv",
+    "__pycache__",
+    "build",
+    "dist",
+    "*.egg-info",
+    ".pytest_cache",
+    "htmlcov",
+]
+
+[tool.ruff.lint]
+select = [
+    "E",
+    "W",
+    "F",
+    "I",
+    "N",
+    "UP",
+    "B",
+    "C4",
+    "SIM",
+    "PL",
+    "RUF",
+    "ERA",
+    "PERF",
+    "ANN",
+]
+ignore = [
+    "E501",
+    "PLR0913",
+    "PLR0912",
+    "PLR0914",
+    "PLR0915",
+    "PLR2004",
+    "PLC0415",
+    "ANN401",
+]
+fixable = ["ALL"]
+unfixable = []
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**/*.py" = [
+    "ANN",
+    "E501",
+    "PLR2004",
+    "RUF002",
+    "RUF003",
+    "PLW0603",
+    "ERA001",
+]
+"src/oda_data/__init__.py" = [
+    "PLW0603",
+    "PLW0602",
+    "F811",
+]
+"src/oda_data/cache/config.py" = ["PLW0603"]
+"src/oda_data/cache/api.py" = ["PLW0603"]
+"src/oda_data/tools/cache.py" = ["PLW2901"]
+"src/oda_data/api/oecd.py" = ["PLR0912"]
+"src/oda_data/indicators/crs/crs_functions.py" = ["B023"]
+"src/oda_data/indicators/research/sector_imputations.py" = ["B023"]
+"src/oda_data/tools/sector_lists.py" = ["N816"]
+
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"
+skip-magic-trailing-comma = false
+line-ending = "auto"
+
+[tool.ty.environment]
+python-version = "3.11"
+
+[tool.ty.src]
+include = ["src/oda_data"]
+
+[tool.ty.analysis]
+replace-imports-with-any = [
+    "pandas",
+    "pandas.**",
+    "numpy",
+    "numpy.**",
+]
+
+[tool.ty.rules]
+possibly-missing-attribute = "ignore"
+possibly-unresolved-reference = "ignore"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+addopts = """
+    -v
+    --strict-markers
+    --tb=short
+    --cov=src/oda_data
+    --cov-report=term-missing
+    --cov-report=html
+    --cov-branch
+    -m "not integration and not network"
+"""
+markers = [
+    """slow: marks tests as slow (deselect with '-m "not slow"')""",
+    "integration: marks tests that call real OECD API (deselected by default)",
+    "network: marks tests requiring live network access (opt-in via RUN_NETWORK_TESTS=1; deselected by default)",
+    "unit: marks tests as unit tests",
+    "cache: marks tests related to caching functionality",
+    "validation: marks tests related to input validation",
+    "cleaning: marks tests related to data cleaning",
+]
+norecursedirs = [
+    ".git",
+    ".venv",
+    "venv",
+    "__pycache__",
+    "*.egg-info",
+    ".pytest_cache",
+    "htmlcov",
+]
+
+[tool.coverage.run]
+source = ["src/oda_data"]
+branch = true
+omit = [
+    "*/tests/*",
+    "*/__init__.py",
+    "*/config.py",
+    "*/logger.py",
+]
+
+[tool.coverage.report]
+precision = 2
+show_missing = true
+exclude_lines = [
+    "pragma: no cover",
+    "def __repr__",
+    "raise AssertionError",
+    "raise NotImplementedError",
+    "if __name__ == .__main__.:",
+    "if TYPE_CHECKING:",
+    "@abstractmethod",
+]

oda_data-2.7.0/pyproject.toml → oda_data-2.8.0/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "oda_data"
-version = "2.7.0"
+version = "2.8.0"
 description = "A python package to work with Official Development Assistance data from the OECD DAC."
 requires-python = ">=3.11"
 
@@ -28,12 +28,11 @@ classifiers = [
 dependencies = [
     "cachetools>=6.2.0",
     "filelock>=3.20.0",
-    "oda-reader>=1.6.0",
+    "oda-reader>=1.9.0",
     "platformdirs>=4.0.0",
     "pandas[excel,parquet,performance]>=2.2.0",
     "pydeflate>=2.3.3",
     "reader>=3.18",
-    "thefuzz>=0.22.1",
 ]
 
 [project.urls]
@@ -59,6 +58,21 @@ test = [
     "pytest-xdist>=3.5.0",
     "freezegun>=1.4.0",
 ]
+# Maintenance-only tooling for scripts/refresh_channel_crosswalk.py. Never a
+# runtime dependency of the oda_data package; excluded from the default
+# groups below so a plain `uv sync` never requires it (resolvekit needs
+# Python >=3.12, oda_data supports >=3.11). Install explicitly with
+# `uv sync --group maintenance`.
+maintenance = [
+    "resolvekit==0.1.12 ; python_version >= '3.12'",
+    "pyyaml>=6.0 ; python_version >= '3.12'",
+]
+
+[tool.uv]
+# Preserve today's implicit default (dev + test both installed by a plain
+# `uv sync` / `uv run`); without this, adding the "maintenance" group above
+# would otherwise be pulled into that same default set.
+default-groups = ["dev", "test"]
 
 [tool.ruff]
 # Set the maximum line length

{oda_data-2.7.0 → oda_data-2.8.0}/src/oda_data/api/sources.py

@@ -308,6 +308,17 @@ class DACSource(Source):
             )
             self.de_sectors = check_strings(sectors)
 
+    def _extra_hash_components(self) -> list[tuple]:
+        """Extra (name, predicate, value) tuples folded into the cache-key hash.
+
+        They bypass the read-filter path, never reaching a parquet predicate
+        or a query string. Folding them into the hash lets a constructor
+        flag that changes row-level behaviour, without being expressed as a
+        filter tuple, still produce a different cache key. Subclasses
+        override this; the default is a no-op.
+        """
+        return []
+
     def _get_filtered_download_filters(self) -> dict:
         """Generates a dictionary of filters for downloading the dataset.
 
@@ -358,6 +369,15 @@ class DACSource(Source):
         except Exception:
             return "unknown"
 
+    def _get_release_id(self) -> str | None:
+        """Best-effort upstream release identity for the bulk cache manifest.
+
+        Overridden by subclasses that go through the OECD bulk file service
+        (CRSData, MultiSystemData) with a cheap, non-download probe. The
+        default is None, which falls back to version/TTL-only staleness.
+        """
+        return None
+
     def _fetch_from_bulk_cache(
         self,
         filters: list[tuple] | None,
@@ -382,6 +402,7 @@ class DACSource(Source):
             fetcher=self._create_bulk_fetcher(),
             ttl_days=30,
             version=self._get_package_version(),
+            release_id=self._get_release_id(),
         )
 
         # This blocks if another thread is downloading (prevents concurrent waste)
@@ -438,12 +459,24 @@ class DACSource(Source):
         """
         # Create filters and generate cache key
         filters = self._get_read_filters(additional_filters=additional_filters)
-        param_hash = generate_param_hash(filters if filters else [])
+        param_hash = generate_param_hash(
+            (filters if filters else []) + self._extra_hash_components()
+        )
 
         # When refresh=True, skip the upper cache tiers entirely so the bulk
         # fetcher's refresh=True actually re-downloads and the upper tiers see
         # the fresh data. Without this guard a warm memory/query cache would
-        # short-circuit the refresh, returning stale data.
+        # short-circuit the refresh, returning stale data. Also evict the
+        # memory-tier entries this call is about to overwrite up front,
+        # rather than relying solely on the post-fetch write — this closes
+        # the window in which a concurrent reader could still observe the
+        # stale value while the refresh is in flight (#162).
+        if refresh:
+            self.memory_cache.pop(param_hash, None)
+            if columns:
+                self.memory_cache.pop(
+                    generate_projection_hash(param_hash, columns), None
+                )
         if not refresh:
             # 1. Try memory cache (thread-safe, fastest)
             _cached = self.memory_cache.get(param_hash)
@@ -640,6 +673,7 @@ class CRSData(DACSource):
         years: list[int] | range | int | None = None,
         providers: list[int] | int | None = None,
         recipients: list[int] | int | None = None,
+        exclude_multilateral_core: bool = True,
     ) -> None:
         """
         Initialize CRSData.
@@ -648,9 +682,17 @@ class CRSData(DACSource):
             years (list[int] | range): List or range of years to filter.
             providers (Optional[list[int]]): List of provider codes.
             recipients (Optional[list[int]]): List of recipient codes.
+            exclude_multilateral_core (bool): Whether to exclude CRS rows that report
+                a donor's core (unearmarked) contribution to a multilateral
+                organisation (``bi_multi == 2``). The OECD's CRS changelog states
+                these rows must be excluded to avoid double-counting bilateral
+                flows. Defaults to True. Rows with a missing ``bi_multi`` value
+                are always kept. Pass False to get the raw CRS totals including
+                those core-contribution rows.
         """
         super().__init__()
         self._init_filters(years=years, providers=providers, recipients=recipients)
+        self.exclude_multilateral_core = exclude_multilateral_core
         self._param_hash = None
 
     def _create_bulk_fetcher(self) -> Callable[[Path], None]:
@@ -660,6 +702,61 @@ class CRSData(DACSource):
         """
         return create_crs_bulk_fetcher()
 
+    def _extra_hash_components(self) -> list[tuple]:
+        return [("exclude_multilateral_core", "==", self.exclude_multilateral_core)]
+
+    def _get_release_id(self) -> str | None:
+        """Cheap upstream release id for the CRS bulk file (#162)."""
+        from oda_data.tools.cache import get_crs_release_id
+
+        return get_crs_release_id()
+
+    def _exclude_core_contributions(self, df: pd.DataFrame) -> pd.DataFrame:
+        """Drop rows reporting a donor's core contribution to a multilateral organisation.
+
+        No-op when ``exclude_multilateral_core`` is False, or when the ``bi_multi``
+        column isn't present (e.g. it was excluded from a column projection).
+        Rows with a missing ``bi_multi`` value are kept. A plain ``!= 2`` filter
+        would drop them, since a null comparison is null rather than true.
+        """
+        if not self.exclude_multilateral_core:
+            return df
+        column = self.schema.BI_MULTI
+        if column not in df.columns:
+            return df
+        return df.loc[(df[column] != 2) | df[column].isna()].reset_index(drop=True)
+
+    def _fetch_from_bulk_cache(
+        self,
+        filters: list[tuple] | None,
+        columns: list | None,
+        *,
+        refresh: bool = False,
+    ) -> pd.DataFrame:
+        """Read from the bulk cache, then apply the core-contribution exclusion.
+
+        ``bi_multi`` is read even when not in the requested ``columns``, so the
+        exclusion can be applied before column projection, then dropped again if
+        the caller didn't ask for it.
+        """
+        read_columns = columns
+        added_bi_multi = False
+        if (
+            self.exclude_multilateral_core
+            and columns is not None
+            and self.schema.BI_MULTI not in columns
+        ):
+            read_columns = [*columns, self.schema.BI_MULTI]
+            added_bi_multi = True
+
+        df = super()._fetch_from_bulk_cache(filters, read_columns, refresh=refresh)
+        df = self._exclude_core_contributions(df)
+
+        if added_bi_multi:
+            df = df.drop(columns=[self.schema.BI_MULTI])
+
+        return df
+
     def download(self) -> pd.DataFrame:
         """Downloads CRS data via API (filtered query).
 
@@ -672,7 +769,7 @@ class CRSData(DACSource):
             filters=self._get_filtered_download_filters(),
         )
         logger.info("CRS data downloaded successfully.")
-        return clean_raw_df(df)
+        return self._exclude_core_contributions(clean_raw_df(df))
 
 
 class MultiSystemData(DACSource):
@@ -722,6 +819,12 @@ class MultiSystemData(DACSource):
         """
         return create_multisystem_bulk_fetcher()
 
+    def _get_release_id(self) -> str | None:
+        """Cheap upstream release id for the MultiSystem bulk file (#162)."""
+        from oda_data.tools.cache import get_multisystem_release_id
+
+        return get_multisystem_release_id()
+
     def download(self) -> pd.DataFrame:
         """Downloads MultiSystem data via API (filtered query).
 

{oda_data-2.7.0 → oda_data-2.8.0}/src/oda_data/cache/__init__.py

@@ -2,11 +2,12 @@
 
 Public symbols re-exported from sub-modules:
 
-- ``path``, ``entries``, ``clear``, ``size``, ``invalidate``,
+- ``path``, ``entries``, ``clear``, ``size``, ``invalidate``, ``release_info``,
   ``enable_cache``, ``disable_cache`` — from :mod:`oda_data.cache.api`
 - ``migrate`` — from :mod:`oda_data.cache._migrate`
 - ``set_cache_root`` — from :mod:`oda_data.cache.config`
-- ``CacheRecord``, ``MigrationResult``, ``Scope`` — from :mod:`oda_data.cache.types`
+- ``CacheRecord``, ``MigrationResult``, ``ReleaseInfo``, ``Scope`` — from
+  :mod:`oda_data.cache.types`
 """
 
 from oda_data.cache._migrate import migrate
@@ -17,14 +18,16 @@ from oda_data.cache.api import (
     entries,
     invalidate,
     path,
+    release_info,
     size,
 )
 from oda_data.cache.config import set_cache_root
-from oda_data.cache.types import CacheRecord, MigrationResult, Scope
+from oda_data.cache.types import CacheRecord, MigrationResult, ReleaseInfo, Scope
 
 __all__ = [
     "CacheRecord",
     "MigrationResult",
+    "ReleaseInfo",
     "Scope",
     "clear",
     "disable_cache",
@@ -33,6 +36,7 @@ __all__ = [
     "invalidate",
     "migrate",
     "path",
+    "release_info",
     "set_cache_root",
     "size",
 ]