oda-data 2.4.2__tar.gz → 2.6.0__tar.gz

{oda_data-2.4.2 → oda_data-2.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: oda-data
-Version: 2.4.2
+Version: 2.6.0
 Summary: A python package to work with Official Development Assistance data from the OECD DAC.
 Author: Jorge Rivera, Miguel Haro Ruiz
 Author-email: Jorge Rivera <jorge.rivera@one.org>, Miguel Haro Ruiz <miguel.haroruiz@one.org>
@@ -18,7 +18,8 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Dist: cachetools>=6.2.0
 Requires-Dist: filelock>=3.20.0
-Requires-Dist: oda-reader>=1.4.1
+Requires-Dist: oda-reader>=1.6.0
+Requires-Dist: platformdirs>=4.0.0
 Requires-Dist: pandas[excel,parquet,performance]>=2.2.0
 Requires-Dist: pydeflate>=2.3.3
 Requires-Dist: reader>=3.18
@@ -448,21 +449,29 @@ dac_members = providers["dac_members"]
 
 ### Cache Management
 
-Control the package's caching behavior:
+`oda_data` stores downloaded data in a per-OS, version-segmented directory
+(`~/Library/Caches/oda-data/<version>/` on macOS, `~/.cache/oda-data/<version>/`
+on Linux, and the equivalent `AppData\Local` path on Windows). The location can be
+overridden with `oda_data.set_cache_root(path)` or the `ODA_DATA_CACHE_DIR`
+environment variable. See [CACHING.md](CACHING.md) for full details on disk
+locations, scopes, multi-process behavior, and migration from older versions.
 
-```python
-from oda_data import clear_cache, disable_cache, enable_cache
+The fastest recovery path for a corrupt or stale cache is:
 
-# Clear all cached data
-clear_cache()
+```python
+from oda_data import cache
 
-# Temporarily disable caching (for development/testing)
-disable_cache()
+# Clear only the raw zip archives and let them re-download on next read:
+cache.clear("raw")
 
-# Re-enable caching
-enable_cache()
+# Or wipe everything:
+cache.clear("all")
 ```
 
+The legacy helpers `clear_cache()`, `disable_cache()`, and `enable_cache()` continue
+to work and are thin wrappers over `cache.clear("all")`, `cache.disable_cache("all")`,
+and `cache.enable_cache("all")` respectively.
+
 ### Compatibility with Version 1.x
 
 For users migrating from v1.x, a compatibility layer is available:

{oda_data-2.4.2 → oda_data-2.6.0}/README.md

@@ -417,21 +417,29 @@ dac_members = providers["dac_members"]
 
 ### Cache Management
 
-Control the package's caching behavior:
+`oda_data` stores downloaded data in a per-OS, version-segmented directory
+(`~/Library/Caches/oda-data/<version>/` on macOS, `~/.cache/oda-data/<version>/`
+on Linux, and the equivalent `AppData\Local` path on Windows). The location can be
+overridden with `oda_data.set_cache_root(path)` or the `ODA_DATA_CACHE_DIR`
+environment variable. See [CACHING.md](CACHING.md) for full details on disk
+locations, scopes, multi-process behavior, and migration from older versions.
 
-```python
-from oda_data import clear_cache, disable_cache, enable_cache
+The fastest recovery path for a corrupt or stale cache is:
 
-# Clear all cached data
-clear_cache()
+```python
+from oda_data import cache
 
-# Temporarily disable caching (for development/testing)
-disable_cache()
+# Clear only the raw zip archives and let them re-download on next read:
+cache.clear("raw")
 
-# Re-enable caching
-enable_cache()
+# Or wipe everything:
+cache.clear("all")
 ```
 
+The legacy helpers `clear_cache()`, `disable_cache()`, and `enable_cache()` continue
+to work and are thin wrappers over `cache.clear("all")`, `cache.disable_cache("all")`,
+and `cache.enable_cache("all")` respectively.
+
 ### Compatibility with Version 1.x
 
 For users migrating from v1.x, a compatibility layer is available:

{oda_data-2.4.2 → oda_data-2.6.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "oda_data"
-version = "2.4.2"
+version = "2.6.0"
 description = "A python package to work with Official Development Assistance data from the OECD DAC."
 requires-python = ">=3.11"
 
@@ -28,7 +28,8 @@ classifiers = [
 dependencies = [
     "cachetools>=6.2.0",
     "filelock>=3.20.0",
-    "oda-reader>=1.4.1",
+    "oda-reader>=1.6.0",
+    "platformdirs>=4.0.0",
     "pandas[excel,parquet,performance]>=2.2.0",
     "pydeflate>=2.3.3",
     "reader>=3.18",
@@ -109,11 +110,16 @@ unfixable = []
 
 [tool.ruff.lint.per-file-ignores]
 # Allow test files to have longer lines and more assertions
-"tests/**/*.py" = ["E501", "PLR2004", "RUF002", "RUF003"]  # Allow × in comments/docstrings
+"tests/**/*.py" = ["E501", "PLR2004", "RUF002", "RUF003", "PLW0603"]  # Allow × in comments/docstrings; PLW0603 for spawn-pool worker globals
 # Allow global variable usage in __init__.py (architectural decision)
 "src/oda_data/__init__.py" = ["PLW0603", "PLW0602", "F811"]
+# Lazy module-level state (override flag, chained-root cache, auto-migrate gate, 5GB warning latch).
+"src/oda_data/cache/config.py" = ["PLW0603"]
+"src/oda_data/cache/api.py" = ["PLW0603"]
 # Allow tmpdir reassignment pattern (Path conversion)
 "src/oda_data/tools/cache.py" = ["PLW2901"]
+# Existing complexity in OECDClient._apply_filters — out of scope for refactor here.
+"src/oda_data/api/oecd.py" = ["PLR0912"]
 # Allow loop variable binding patterns (intentional for lambda closures)
 "src/oda_data/indicators/crs/crs_functions.py" = ["B023"]
 "src/oda_data/indicators/research/sector_imputations.py" = ["B023"]
@@ -132,3 +138,6 @@ skip-magic-trailing-comma = false
 
 # Like Black, automatically detect line endings
 line-ending = "auto"
+
+# Note: pytest configuration lives in `pytest.ini` (which takes precedence over
+# this file). Markers and addopts must be added there, not here.

oda_data-2.6.0/src/oda_data/__init__.py

@@ -0,0 +1,113 @@
+import warnings
+
+from oda_data import cache
+from oda_data.api.oecd import OECDClient
+from oda_data.api.sources import (
+    AidDataData,
+    CRSData,
+    DAC1Data,
+    DAC2AData,
+    MultiSystemData,
+)
+from oda_data.cache.config import set_cache_root
+from oda_data.indicators.research import sector_imputations
+from oda_data.indicators.research.policy_markers import bilateral_policy_marker
+from oda_data.tools.compatibility import ODAData
+from oda_data.tools.gni import add_gni_share_column
+from oda_data.tools.groupings import provider_groupings, recipient_groupings
+from oda_data.tools.names.add import add_names_columns
+from oda_data.tools.sector_lists import add_broad_sectors, add_sectors
+
+_DEPRECATION_WARNED_SET_DATA_PATH = False
+
+
+def clear_cache() -> None:
+    """Back-compat alias for oda_data.cache.clear('all').
+
+    Clears all scopes (bulk, query, http, raw). Prefer ``oda_data.cache.clear()``.
+    """
+    from oda_data.cache import clear
+
+    clear("all")
+
+
+def disable_cache() -> None:
+    """Back-compat alias for oda_data.cache.disable_cache('all').
+
+    Disables all cache scopes. Prefer ``oda_data.cache.disable_cache()``.
+    """
+    from oda_data.cache import disable_cache as _disable
+
+    _disable("all")
+
+
+def enable_cache() -> None:
+    """Back-compat alias for oda_data.cache.enable_cache('all').
+
+    Enables all cache scopes. Prefer ``oda_data.cache.enable_cache()``.
+    """
+    from oda_data.cache import enable_cache as _enable
+
+    _enable("all")
+
+
+def set_data_path(path) -> None:
+    """Set the path for data outputs (parquet exports).
+
+    Deprecated: cache location is now controlled via set_cache_root() or
+    the ODA_DATA_CACHE_DIR env var. A one-time DeprecationWarning is emitted
+    on first call. Shim preserved through 2.x, removed in 3.0.
+    """
+    global _DEPRECATION_WARNED_SET_DATA_PATH
+
+    from pathlib import Path
+
+    from pydeflate import set_pydeflate_path
+
+    from oda_data.config import ODAPaths
+
+    if not _DEPRECATION_WARNED_SET_DATA_PATH:
+        warnings.warn(
+            "oda_data.set_data_path() now governs only data outputs (parquet exports),"
+            " not the cache. If you previously called set_data_path() to control where"
+            " downloads are cached, also call"
+            " oda_data.set_cache_root('<your_path>') or set the ODA_DATA_CACHE_DIR env"
+            " var. The set_data_path() shim itself is preserved through oda_data 2.x"
+            " and removed in 3.0.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        _DEPRECATION_WARNED_SET_DATA_PATH = True
+
+    ODAPaths.raw_data = Path(path).resolve()
+    # Create on explicit user request; avoids import-time side effects.
+    ODAPaths.raw_data.mkdir(parents=True, exist_ok=True)
+    ODAPaths.pydeflate = ODAPaths.raw_data / ".pydeflate"
+    set_pydeflate_path(ODAPaths.raw_data)
+    # Do NOT call set_cache_dir here — cache chaining is handled lazily by
+    # oda_data_cache_root() inside cache/config.py.
+
+
+__all__ = [
+    "AidDataData",
+    "CRSData",
+    "DAC1Data",
+    "DAC2AData",
+    "MultiSystemData",
+    "ODAData",
+    "OECDClient",
+    "add_broad_sectors",
+    "add_gni_share_column",
+    "add_names_columns",
+    "add_sectors",
+    "bilateral_policy_marker",
+    "cache",
+    "clear_cache",
+    "disable_cache",
+    "enable_cache",
+    "provider_groupings",
+    "recipient_groupings",
+    "sector_imputations",
+    "set_cache_root",
+    "set_data_path",
+]

{oda_data-2.4.2 → oda_data-2.6.0}/src/oda_data/api/sources.py

@@ -123,41 +123,37 @@ class Source:
 
     @property
     def bulk_cache(self) -> BulkCacheManager:
-        """Get bulk cache manager singleton.
-
-        Creates one bulk cache per process, recreating if data directory changes.
-        """
-        current_raw = Path(ODAPaths.raw_data).resolve()
-        # Fast-path without lock
-        bc = Source._shared_bulk_cache
-        if bc is not None and bc.base_dir.parent.resolve() == current_raw:
-            return bc
-
-        # Slow-path with lock
-        with Source._cache_lock:
-            bc = Source._shared_bulk_cache
-            if bc is None or bc.base_dir.parent.resolve() != current_raw:
-                Source._shared_bulk_cache = BulkCacheManager(current_raw)
-            return Source._shared_bulk_cache
+        """Get bulk cache manager singleton, recreating if the cache root changed."""
+        return self._shared_manager(BulkCacheManager, "_shared_bulk_cache")
 
     @property
     def query_cache(self) -> QueryCacheManager:
-        """Get query cache manager singleton.
+        """Get query cache manager singleton, recreating if the cache root changed."""
+        return self._shared_manager(QueryCacheManager, "_shared_query_cache")
+
+    @classmethod
+    def _shared_manager(cls, manager_cls, slot: str):
+        """Return a process-wide manager rooted at ``cache_root / "oda-data"``.
 
-        Creates one query cache per process, recreating if data directory changes.
+        The singleton is intentionally anchored on ``Source`` (not on the
+        runtime subclass) so that every subclass shares the same instance.
+
+        The unresolved equality check on the fast path avoids a ``realpath``
+        syscall on every property access; we only ``.resolve()`` when the
+        configured root visibly changed.
         """
-        current_raw = Path(ODAPaths.raw_data).resolve()
-        # Fast-path without lock
-        qc = Source._shared_query_cache
-        if qc is not None and qc.base_dir.parent.resolve() == current_raw:
-            return qc
-
-        # Slow-path with lock
-        with Source._cache_lock:
-            qc = Source._shared_query_cache
-            if qc is None or qc.base_dir.parent.resolve() != current_raw:
-                Source._shared_query_cache = QueryCacheManager(current_raw)
-            return Source._shared_query_cache
+        current = Path(ODAPaths.cache_root) / "oda-data"
+        existing = getattr(Source, slot)
+        if existing is not None and existing.base_dir.parent == current:
+            return existing
+        with cls._cache_lock:
+            existing = getattr(Source, slot)
+            if (
+                existing is None
+                or existing.base_dir.parent.resolve() != current.resolve()
+            ):
+                setattr(Source, slot, manager_cls(current))
+            return getattr(Source, slot)
 
     def _add_filter(self, column: str, predicate: str, value: str | int | list) -> None:
         """Adds a filter to the dataset, ensuring no duplicate columns.
@@ -348,14 +344,23 @@ class DACSource(Source):
             return "unknown"
 
     def _fetch_from_bulk_cache(
-        self, filters: list[tuple] | None, columns: list | None
+        self,
+        filters: list[tuple] | None,
+        columns: list | None,
+        *,
+        refresh: bool = False,
     ) -> pd.DataFrame:
         """Fetch data from bulk cache with download coordination.
 
-        This method uses FileLock via BulkCacheManager - if another thread is
+        This method uses FileLock via BulkCacheManager — if another thread is
         downloading, this call blocks until download completes.
 
         Uses PyArrow predicate pushdown and column projection for maximum efficiency.
+
+        Args:
+            filters: Optional parquet predicate filters.
+            columns: Optional column subset.
+            refresh: If True, bypass the bulk cache and re-download.
         """
         entry = BulkCacheEntry(
             key=f"{self.__class__.__name__}_bulk",
@@ -365,7 +370,7 @@ class DACSource(Source):
         )
 
         # This blocks if another thread is downloading (prevents concurrent waste)
-        bulk_path = self.bulk_cache.ensure(entry, refresh=False)
+        bulk_path = self.bulk_cache.ensure(entry, refresh=refresh)
 
         pyarrow_filters = filters if filters else None
 
@@ -396,6 +401,8 @@ class DACSource(Source):
         using_bulk_download: bool = False,
         additional_filters: list[tuple] | None = None,
         columns: list | None = None,
+        *,
+        refresh: bool = False,
     ):
         """Reads a dataset with 3-tier cache coordination.
 
@@ -408,6 +415,8 @@ class DACSource(Source):
             using_bulk_download: Whether to read from bulk cache
             additional_filters: Additional filters to apply
             columns: Subset of columns to read
+            refresh: If True, bypass the bulk cache and re-download. Only has
+                effect when ``using_bulk_download=True``.
 
         Returns:
             Filtered and cleaned DataFrame
@@ -416,45 +425,49 @@ class DACSource(Source):
         filters = self._get_read_filters(additional_filters=additional_filters)
         param_hash = generate_param_hash(filters if filters else [])
 
-        # 1. Try memory cache (thread-safe, fastest)
-        df = self.memory_cache.get(param_hash)
-        if df is not None:
+        # When refresh=True, skip the upper cache tiers entirely so the bulk
+        # fetcher's refresh=True actually re-downloads and the upper tiers see
+        # the fresh data. Without this guard a warm memory/query cache would
+        # short-circuit the refresh, returning stale data.
+        if not refresh:
+            # 1. Try memory cache (thread-safe, fastest)
+            df = self.memory_cache.get(param_hash)
             # Only use if all requested columns are available
-            if not columns or all(c in df.columns for c in columns):
+            if df is not None and (
+                not columns or all(c in df.columns for c in columns)
+            ):
                 logger.info("Cache hit: memory")
-                return self._apply_columns_and_clean(
-                    df, columns, already_cleaned=True
-                )
+                return self._apply_columns_and_clean(df, columns, already_cleaned=True)
 
-        # For projected reads, also check a projection-specific memory cache
-        if columns:
-            proj_hash = generate_projection_hash(param_hash, columns)
-            df = self.memory_cache.get(proj_hash)
+            # For projected reads, also check a projection-specific memory cache
+            if columns:
+                proj_hash = generate_projection_hash(param_hash, columns)
+                df = self.memory_cache.get(proj_hash)
+                if df is not None:
+                    logger.info("Cache hit: memory (projection)")
+                    return df
+
+            # 2. Try query cache (on-disk parquet, fast)
+            df = self.query_cache.load(
+                self.__class__.__name__, param_hash, None, columns
+            )
             if df is not None:
-                logger.info("Cache hit: memory (projection)")
+                logger.info("Cache hit: query cache")
+                # Only cache in memory if we loaded full data (no column selection)
+                # This way memory cache is reusable for queries with different column subsets
+                if not columns:
+                    self._cache_in_memory(param_hash, df)
+                # Data is already cleaned and column-selected at parquet read level
                 return df
 
-        # 2. Try query cache (on-disk parquet, fast)
-        df = self.query_cache.load(self.__class__.__name__, param_hash, None, columns)
-        if df is not None:
-            logger.info("Cache hit: query cache")
-            # Only cache in memory if we loaded full data (no column selection)
-            # This way memory cache is reusable for queries with different column subsets
-            if not columns:
-                self._cache_in_memory(param_hash, df)
-            # Data is already cleaned and column-selected at parquet read level
-            return df
-
         # 3. Cache miss (query/memory) - need to fetch data
         if using_bulk_download:
             logger.info("Query cache miss - reading from bulk cache")
-            df = self._fetch_from_bulk_cache(filters, columns)
+            df = self._fetch_from_bulk_cache(filters, columns, refresh=refresh)
             if columns:
                 # Cache projected result under a projection-specific key
                 # so it doesn't pollute the shared filter-only cache
-                self._cache_in_memory(
-                    generate_projection_hash(param_hash, columns), df
-                )
+                self._cache_in_memory(generate_projection_hash(param_hash, columns), df)
                 return df
         else:
             # Download via API (uses init-time filters only)
@@ -802,12 +815,15 @@ class AidDataData(AidDataSource):
         self,
         additional_filters: list[tuple] | None = None,
         columns: list | None = None,
+        *,
+        refresh: bool = False,
     ):
         """Reads AidData from bulk cache with filtering.
 
         Args:
             additional_filters: Additional filters to apply
             columns: Subset of columns to read
+            refresh: If True, bypass the bulk cache and re-download.
 
         Returns:
             Filtered DataFrame
@@ -816,35 +832,39 @@ class AidDataData(AidDataSource):
         filters = self._get_read_filters(additional_filters=additional_filters)
         param_hash = generate_param_hash(filters if filters else [])
 
-        # 1. Try memory cache
-        df = self.memory_cache.get(param_hash)
-        if df is not None:
+        # When refresh=True, skip the upper cache tiers so the bulk fetcher's
+        # refresh=True actually re-downloads. See DACSource.read for context.
+        if not refresh:
+            # 1. Try memory cache
+            df = self.memory_cache.get(param_hash)
             # Only use if all requested columns are available
-            if not columns or all(c in df.columns for c in columns):
+            if df is not None and (
+                not columns or all(c in df.columns for c in columns)
+            ):
                 logger.info("Cache hit: memory")
-                return self._apply_columns_and_clean(
-                    df, columns, already_cleaned=True
-                )
+                return self._apply_columns_and_clean(df, columns, already_cleaned=True)
 
-        # For projected reads, also check a projection-specific memory cache
-        if columns:
-            proj_hash = generate_projection_hash(param_hash, columns)
-            df = self.memory_cache.get(proj_hash)
+            # For projected reads, also check a projection-specific memory cache
+            if columns:
+                proj_hash = generate_projection_hash(param_hash, columns)
+                df = self.memory_cache.get(proj_hash)
+                if df is not None:
+                    logger.info("Cache hit: memory (projection)")
+                    return df
+
+            # 2. Try query cache
+            # Use column selection at parquet read level for efficiency
+            df = self.query_cache.load(
+                self.__class__.__name__, param_hash, None, columns
+            )
             if df is not None:
-                logger.info("Cache hit: memory (projection)")
+                logger.info("Cache hit: query cache")
+                # Only cache in memory if we loaded full data (no column selection)
+                if not columns:
+                    self._cache_in_memory(param_hash, df)
+                # Data is already cleaned and column-selected at parquet read level
                 return df
 
-        # 2. Try query cache
-        # Use column selection at parquet read level for efficiency
-        df = self.query_cache.load(self.__class__.__name__, param_hash, None, columns)
-        if df is not None:
-            logger.info("Cache hit: query cache")
-            # Only cache in memory if we loaded full data (no column selection)
-            if not columns:
-                self._cache_in_memory(param_hash, df)
-            # Data is already cleaned and column-selected at parquet read level
-            return df
-
         # 3. Fetch from bulk cache (always bulk for AidData)
         entry = BulkCacheEntry(
             key=f"{self.__class__.__name__}_bulk",
@@ -853,7 +873,7 @@ class AidDataData(AidDataSource):
             version=self._get_package_version(),
         )
 
-        bulk_path = self.bulk_cache.ensure(entry, refresh=False)
+        bulk_path = self.bulk_cache.ensure(entry, refresh=refresh)
 
         # Use PyArrow predicate pushdown and column selection for maximum efficiency
         pyarrow_filters = filters if filters else None
@@ -868,9 +888,7 @@ class AidDataData(AidDataSource):
             )
             # Cache under projection-specific key to avoid polluting
             # the shared filter-only cache
-            self._cache_in_memory(
-                generate_projection_hash(param_hash, columns), df
-            )
+            self._cache_in_memory(generate_projection_hash(param_hash, columns), df)
             return df
         else:
             logger.info("Query cache miss - reading from bulk cache")

oda_data-2.6.0/src/oda_data/cache/__init__.py

@@ -0,0 +1,38 @@
+"""oda_data.cache — typed cache management surface.
+
+Public symbols re-exported from sub-modules:
+
+- ``path``, ``entries``, ``clear``, ``size``, ``invalidate``,
+  ``enable_cache``, ``disable_cache`` — from :mod:`oda_data.cache.api`
+- ``migrate`` — from :mod:`oda_data.cache._migrate`
+- ``set_cache_root`` — from :mod:`oda_data.cache.config`
+- ``CacheRecord``, ``MigrationResult``, ``Scope`` — from :mod:`oda_data.cache.types`
+"""
+
+from oda_data.cache._migrate import migrate
+from oda_data.cache.api import (
+    clear,
+    disable_cache,
+    enable_cache,
+    entries,
+    invalidate,
+    path,
+    size,
+)
+from oda_data.cache.config import set_cache_root
+from oda_data.cache.types import CacheRecord, MigrationResult, Scope
+
+__all__ = [
+    "CacheRecord",
+    "MigrationResult",
+    "Scope",
+    "clear",
+    "disable_cache",
+    "enable_cache",
+    "entries",
+    "invalidate",
+    "migrate",
+    "path",
+    "set_cache_root",
+    "size",
+]