pydeflate 2.1.2__py3-none-any.whl → 2.2.0__py3-none-any.whl

pydeflate/__init__.py

@@ -1,47 +1,91 @@
 __author__ = """Jorge Rivera"""
-__version__ = "2.1.2"
+__version__ = "2.2.0"
 
 from pydeflate.deflate.deflators import (
+    imf_cpi_deflate,
+    imf_cpi_e_deflate,
+    imf_gdp_deflate,
     oecd_dac_deflate,
     wb_cpi_deflate,
     wb_gdp_deflate,
     wb_gdp_linked_deflate,
-    imf_cpi_deflate,
-    imf_gdp_deflate,
-    imf_cpi_e_deflate,
 )
-
 from pydeflate.deflate.legacy_deflate import deflate
 from pydeflate.exchange.exchangers import (
+    imf_exchange,
     oecd_dac_exchange,
     wb_exchange,
     wb_exchange_ppp,
-    imf_exchange,
 )
-from pydeflate.pydeflate_config import setup_logger
+from pydeflate.pydeflate_config import set_data_dir, setup_logger
 
+from pydeflate.context import (
+    PydeflateContext,
+    get_default_context,
+    pydeflate_session,
+    set_default_context,
+    temporary_context,
+)
+from pydeflate.exceptions import (
+    CacheError,
+    ConfigurationError,
+    DataSourceError,
+    MissingDataError,
+    NetworkError,
+    PluginError,
+    PydeflateError,
+    SchemaValidationError,
+)
+from pydeflate.plugins import (
+    get_source,
+    is_source_registered,
+    list_sources,
+    register_source,
+)
 
-def set_pydeflate_path(path):
-    from pathlib import Path
-    from pydeflate.pydeflate_config import PYDEFLATE_PATHS
 
-    """Set the path to the data folder."""
-    global PYDEFLATE_PATHS
+def set_pydeflate_path(path):
+    """Set the path to the pydeflate data cache directory."""
 
-    PYDEFLATE_PATHS.data = Path(path).resolve()
+    return set_data_dir(path)
 
 
 __all__ = [
-    "set_pydeflate_path",
+    # Deflation functions
+    "deflate",
+    "imf_cpi_deflate",
+    "imf_cpi_e_deflate",
+    "imf_gdp_deflate",
     "oecd_dac_deflate",
-    "oecd_dac_exchange",
     "wb_cpi_deflate",
     "wb_gdp_deflate",
     "wb_gdp_linked_deflate",
-    "wb_exchange",
-    "imf_cpi_deflate",
-    "imf_gdp_deflate",
-    "imf_cpi_e_deflate",
+    # Exchange functions
     "imf_exchange",
-    "deflate",
+    "oecd_dac_exchange",
+    "wb_exchange",
+    "wb_exchange_ppp",
+    # Configuration
+    "set_pydeflate_path",
+    "setup_logger",
+    # Context management
+    "PydeflateContext",
+    "get_default_context",
+    "pydeflate_session",
+    "set_default_context",
+    "temporary_context",
+    # Exceptions
+    "CacheError",
+    "ConfigurationError",
+    "DataSourceError",
+    "MissingDataError",
+    "NetworkError",
+    "PluginError",
+    "PydeflateError",
+    "SchemaValidationError",
+    # Plugin system
+    "get_source",
+    "is_source_registered",
+    "list_sources",
+    "register_source",
 ]

pydeflate/cache.py

@@ -0,0 +1,139 @@
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Callable, Dict, Iterable, Optional
+
+from filelock import FileLock
+
+from pydeflate.pydeflate_config import get_data_dir
+
+ISO_FORMAT = "%Y-%m-%dT%H:%M:%S.%f%z"
+
+
+@dataclass(frozen=True)
+class CacheEntry:
+    """Describe a cacheable dataset."""
+
+    key: str
+    filename: str
+    fetcher: Callable[[Path], None]
+    ttl_days: int = 30
+    version: str | None = None
+
+
+@dataclass(frozen=True)
+class CacheRecord:
+    key: str
+    path: Path
+    downloaded_at: datetime
+    ttl_days: int
+    version: str | None
+
+
+class CacheError(RuntimeError):
+    pass
+
+
+class CacheManager:
+    """Handle cached datasets stored under the pydeflate data directory."""
+
+    def __init__(self, base_dir: Path | None = None) -> None:
+        self.base_dir = (base_dir or get_data_dir()).resolve()
+        self.base_dir.mkdir(parents=True, exist_ok=True)
+        self.manifest_path = self.base_dir / "manifest.json"
+        self._lock = FileLock(str(self.base_dir / ".cache.lock"))
+        self._manifest: Dict[str, dict] = self._load_manifest()
+
+    # ------------------------------------------------------------------
+    def ensure(self, entry: CacheEntry, *, refresh: bool = False) -> Path:
+        """Return a local path for the given entry, downloading when needed."""
+
+        with self._lock:
+            record = self._manifest.get(entry.key)
+            path = self.base_dir / entry.filename
+
+            if not refresh and record and path.exists():
+                if not self._is_stale(record, entry):
+                    return path
+
+            path.parent.mkdir(parents=True, exist_ok=True)
+            tmp_path = Path(f"{path}.tmp-{os.getpid()}")
+            try:
+                entry.fetcher(tmp_path)
+                tmp_path.replace(path)
+            finally:
+                if tmp_path.exists():
+                    tmp_path.unlink(missing_ok=True)
+
+            self._manifest[entry.key] = {
+                "filename": entry.filename,
+                "downloaded_at": datetime.now(timezone.utc).strftime(ISO_FORMAT),
+                "ttl_days": entry.ttl_days,
+                "version": entry.version,
+            }
+            self._save_manifest()
+            return path
+
+    # ------------------------------------------------------------------
+    def list_records(self) -> Iterable[CacheRecord]:
+        for key, payload in self._manifest.items():
+            path = self.base_dir / payload["filename"]
+            yield CacheRecord(
+                key=key,
+                path=path,
+                downloaded_at=datetime.strptime(payload["downloaded_at"], ISO_FORMAT),
+                ttl_days=payload["ttl_days"],
+                version=payload.get("version"),
+            )
+
+    # ------------------------------------------------------------------
+    def clear(self, key: str | None = None) -> None:
+        with self._lock:
+            if key is None:
+                for payload in self._manifest.values():
+                    (self.base_dir / payload["filename"]).unlink(missing_ok=True)
+                self._manifest = {}
+            else:
+                payload = self._manifest.pop(key, None)
+                if payload:
+                    (self.base_dir / payload["filename"]).unlink(missing_ok=True)
+            self._save_manifest()
+
+    # ------------------------------------------------------------------
+    def _is_stale(self, record: dict, entry: CacheEntry) -> bool:
+        version_changed = entry.version is not None and entry.version != record.get(
+            "version"
+        )
+        downloaded = datetime.strptime(record["downloaded_at"], ISO_FORMAT)
+        age = datetime.now(timezone.utc) - downloaded
+        ttl = timedelta(days=entry.ttl_days)
+        return version_changed or age > ttl
+
+    # ------------------------------------------------------------------
+    def _load_manifest(self) -> Dict[str, dict]:
+        if not self.manifest_path.exists():
+            return {}
+        try:
+            return json.loads(self.manifest_path.read_text())
+        except json.JSONDecodeError:
+            return {}
+
+    # ------------------------------------------------------------------
+    def _save_manifest(self) -> None:
+        payload = json.dumps(self._manifest, indent=2)
+        self.manifest_path.write_text(payload)
+
+
+_CACHE_MANAGER: Optional[CacheManager] = None
+
+
+def cache_manager() -> CacheManager:
+    global _CACHE_MANAGER
+    base_dir = get_data_dir().resolve()
+    if _CACHE_MANAGER is None or _CACHE_MANAGER.base_dir != base_dir:
+        _CACHE_MANAGER = CacheManager(base_dir)
+    return _CACHE_MANAGER

pydeflate/constants.py

@@ -0,0 +1,121 @@
+"""Constants used throughout pydeflate.
+
+Centralizing constants eliminates magic strings and makes refactoring safer.
+"""
+
+from __future__ import annotations
+
+
+class PydeflateColumns:
+    """Standard column names used in pydeflate DataFrames."""
+
+    # Index columns
+    YEAR = "pydeflate_year"
+    ENTITY_CODE = "pydeflate_entity_code"
+    ISO3 = "pydeflate_iso3"
+
+    # Data columns
+    EXCHANGE = "pydeflate_EXCHANGE"
+    EXCHANGE_D = "pydeflate_EXCHANGE_D"
+
+    # Deflator columns
+    NGDP_D = "pydeflate_NGDP_D"
+    NGDP_DL = "pydeflate_NGDP_DL"
+    CPI = "pydeflate_CPI"
+    PCPI = "pydeflate_PCPI"
+    PCPIE = "pydeflate_PCPIE"
+    DAC_DEFLATOR = "pydeflate_DAC_DEFLATOR"
+
+    # Standard index
+    STANDARD_INDEX = [YEAR, ENTITY_CODE, ISO3]
+
+    @classmethod
+    def deflator_column(cls, kind: str) -> str:
+        """Get deflator column name for a given kind.
+
+        Args:
+            kind: Deflator type (e.g., 'NGDP_D', 'CPI')
+
+        Returns:
+            Full column name with pydeflate_ prefix
+        """
+        if kind.startswith("pydeflate_"):
+            return kind
+        return f"pydeflate_{kind}"
+
+
+class CurrencyCodes:
+    """Common currency code mappings."""
+
+    # ISO3 to common codes
+    USD = "USA"
+    EUR = "EUR"  # For most sources
+    EUR_DAC = "EUI"  # For DAC source
+    GBP = "GBR"
+    JPY = "JPN"
+    CAD = "CAN"
+
+    # Special codes
+    LCU = "LCU"  # Local Currency Unit
+    PPP = "PPP"  # Purchasing Power Parity
+    DAC = "DAC"  # DAC members
+
+    # Mapping for user convenience
+    COMMON_ALIASES = {
+        "USD": USA,
+        "EUR": EUR,
+        "GBP": GBR,
+        "JPY": JPN,
+        "CAD": CAN,
+    }
+
+    @classmethod
+    def resolve(cls, code: str, source: str | None = None) -> str:
+        """Resolve a currency code to ISO3.
+
+        Args:
+            code: Currency code (USD, EUR, etc.) or ISO3
+            source: Data source name (affects EUR mapping for DAC)
+
+        Returns:
+            ISO3 country code or special code (LCU, PPP)
+        """
+        # Handle EUR special case for DAC
+        if code == "EUR" and source == "DAC":
+            return cls.EUR_DAC
+
+        # Try aliases
+        return cls.COMMON_ALIASES.get(code, code)
+
+
+class DataSources:
+    """Names of built-in data sources."""
+
+    IMF = "IMF"
+    WORLD_BANK = "World Bank"
+    WORLD_BANK_PPP = "World Bank PPP"
+    DAC = "DAC"
+
+    # Aliases
+    WB = "World Bank"
+    OECD = "DAC"
+
+    ALL_SOURCES = [IMF, WORLD_BANK, WORLD_BANK_PPP, DAC]
+
+
+class CacheDefaults:
+    """Default values for caching."""
+
+    TTL_DAYS_IMF = 60  # IMF data updates less frequently
+    TTL_DAYS_WB = 30  # World Bank monthly updates
+    TTL_DAYS_DAC = 30  # DAC data
+    DEFAULT_TTL = 30
+
+
+class ValidationConfig:
+    """Validation configuration."""
+
+    MIN_YEAR = 1960  # No data before 1960
+    MAX_YEAR = 2100  # No projections beyond 2100
+    MIN_EXCHANGE_RATE = 1e-6  # Extremely low but non-zero
+    MAX_EXCHANGE_RATE = 1e6  # Extremely high but finite

pydeflate/context.py

@@ -0,0 +1,211 @@
+"""Context management for dependency injection.
+
+This module provides a context-based approach to managing pydeflate's
+configuration, cache, and logging. This eliminates global state and
+enables better testability and parallel execution.
+"""
+
+from __future__ import annotations
+
+import logging
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Generator
+
+from pydeflate.cache import CacheManager
+from pydeflate.pydeflate_config import get_data_dir
+
+
+@dataclass
+class PydeflateContext:
+    """Encapsulates all runtime configuration for pydeflate operations.
+
+    This class holds the cache manager, data directory, logger, and other
+    runtime settings. Using a context object instead of global variables
+    enables:
+    - Multiple independent configurations in the same process
+    - Better testability (mock the context instead of globals)
+    - Thread-safe parallel operations
+    - Clear dependency tracking
+
+    Attributes:
+        data_dir: Directory where deflator/exchange data is cached
+        cache_manager: Manages cached datasets
+        logger: Logger instance for this context
+        log_level: Logging level (DEBUG, INFO, WARNING, ERROR)
+        enable_validation: Whether to validate data schemas (recommended: True)
+        config: Additional configuration options
+    """
+
+    data_dir: Path
+    cache_manager: CacheManager | None = None
+    logger: logging.Logger | None = None
+    log_level: int = logging.INFO
+    enable_validation: bool = True
+    config: dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self):
+        """Initialize cache manager and logger if not provided."""
+        if self.cache_manager is None:
+            self.cache_manager = CacheManager(self.data_dir)
+
+        if self.logger is None:
+            self.logger = self._create_logger()
+
+    def _create_logger(self) -> logging.Logger:
+        """Create a logger for this context."""
+        from pydeflate.pydeflate_config import setup_logger
+
+        logger = setup_logger(f"pydeflate.{id(self)}")
+        logger.setLevel(self.log_level)
+        return logger
+
+    @classmethod
+    def create(
+        cls,
+        data_dir: str | Path | None = None,
+        log_level: int = logging.INFO,
+        enable_validation: bool = True,
+        **config,
+    ) -> PydeflateContext:
+        """Factory method to create a context with sensible defaults.
+
+        Args:
+            data_dir: Path to cache directory. If None, uses default from config.
+            log_level: Logging level for this context
+            enable_validation: Enable schema validation
+            **config: Additional configuration options
+
+        Returns:
+            New PydeflateContext instance
+        """
+        if data_dir is None:
+            data_dir = get_data_dir()
+        else:
+            data_dir = Path(data_dir).expanduser().resolve()
+
+        return cls(
+            data_dir=data_dir,
+            log_level=log_level,
+            enable_validation=enable_validation,
+            config=config,
+        )
+
+
+# Thread-local storage for default context
+import threading
+
+_thread_local = threading.local()
+
+
+def get_default_context() -> PydeflateContext:
+    """Get the default context for the current thread.
+
+    If no context has been set, creates one with default settings.
+
+    Returns:
+        PydeflateContext for current thread
+    """
+    if not hasattr(_thread_local, "context"):
+        _thread_local.context = PydeflateContext.create()
+    return _thread_local.context
+
+
+def set_default_context(context: PydeflateContext) -> None:
+    """Set the default context for the current thread.
+
+    Args:
+        context: Context to use as default
+    """
+    _thread_local.context = context
+
+
+@contextmanager
+def pydeflate_session(
+    data_dir: str | Path | None = None,
+    log_level: int = logging.INFO,
+    enable_validation: bool = True,
+    **config,
+) -> Generator[PydeflateContext, None, None]:
+    """Context manager for pydeflate operations with custom configuration.
+
+    This is the recommended way to use pydeflate when you need custom
+    configuration. It ensures clean setup and teardown.
+
+    Example:
+        >>> from pydeflate.context import pydeflate_session
+        >>> with pydeflate_session(data_dir="/tmp/my_cache") as ctx:
+        ...     # Use ctx for deflation operations
+        ...     result = imf_gdp_deflate(df, context=ctx, ...)
+
+    Args:
+        data_dir: Path to cache directory
+        log_level: Logging level
+        enable_validation: Enable schema validation
+        **config: Additional configuration
+
+    Yields:
+        PydeflateContext configured with the given parameters
+    """
+    context = PydeflateContext.create(
+        data_dir=data_dir,
+        log_level=log_level,
+        enable_validation=enable_validation,
+        **config,
+    )
+
+    # Save previous default context
+    previous_context = getattr(_thread_local, "context", None)
+
+    try:
+        # Set as default for this thread
+        set_default_context(context)
+        yield context
+    finally:
+        # Restore previous context
+        if previous_context is not None:
+            set_default_context(previous_context)
+        elif hasattr(_thread_local, "context"):
+            delattr(_thread_local, "context")
+
+
+@contextmanager
+def temporary_context(
+    **overrides,
+) -> Generator[PydeflateContext, None, None]:
+    """Create a temporary context with specific overrides.
+
+    This is useful for testing or temporarily changing configuration.
+
+    Example:
+        >>> from pydeflate.context import temporary_context
+        >>> with temporary_context(enable_validation=False) as ctx:
+        ...     # Validation disabled for this block
+        ...     result = process_data(ctx=ctx)
+
+    Args:
+        **overrides: Configuration overrides (log_level, enable_validation, etc.)
+
+    Yields:
+        Temporary PydeflateContext with overrides applied
+    """
+    default = get_default_context()
+
+    # Create new context with overrides
+    config = default.config.copy()
+    config.update(overrides.get("config", {}))
+
+    temp_ctx = PydeflateContext.create(
+        data_dir=overrides.get("data_dir", default.data_dir),
+        log_level=overrides.get("log_level", default.log_level),
+        enable_validation=overrides.get("enable_validation", default.enable_validation),
+        **config,
+    )
+
+    previous = get_default_context()
+    try:
+        set_default_context(temp_ctx)
+        yield temp_ctx
+    finally:
+        set_default_context(previous)

pydeflate/core/api.py

@@ -6,17 +6,17 @@ from pydeflate.core.source import Source
 from pydeflate.sources.common import AvailableDeflators
 from pydeflate.utils import (
     create_pydeflate_year,
-    merge_user_and_pydeflate_data,
-    get_unmatched_pydeflate_data,
-    get_matched_pydeflate_data,
     flag_missing_pydeflate_data,
+    get_matched_pydeflate_data,
+    get_unmatched_pydeflate_data,
+    merge_user_and_pydeflate_data,
 )
 
 
 def resolve_common_currencies(currency: str, source: str) -> str:
     mapping = {
         "USD": "USA",
-        "EUR": "EMU",
+        "EUR": "EUR",
         "GBP": "GBR",
         "JPY": "JPN",
         "CAD": "CAN",
@@ -85,7 +85,6 @@ def _base_operation(
         "pydeflate_EXCHANGE" if exchange else "pydeflate_deflator"
     ]
 
-    # Apply the correct operation based on `exchange` and `reversed`
     if (exchange and not reversed_) or (not exchange and reversed_):
         base_obj._merged_data[target_value_column] = (x * y).round(6)
     else:
@@ -299,33 +298,56 @@ class BaseDeflate:
         # drop where necessary data is missing
         data = data.set_index(self._idx).dropna(how="any").reset_index()
 
+        # For to_current=True, we need the base year exchange rate
+        # Extract base year exchange rates and merge them
+        if self.to_current:
+            entity_col = self._idx[1]  # pydeflate_iso3 or pydeflate_entity_code
+            base_year_rates = (
+                data[data["pydeflate_year"] == self.price_deflator.base_year]
+                .filter([entity_col, "pydeflate_EXCHANGE"])
+                .rename(columns={"pydeflate_EXCHANGE": "pydeflate_EXCHANGE_BASE"})
+            )
+            data = data.merge(base_year_rates, on=entity_col, how="left")
+
         # Calculate price-exchange deflator
         data["pydeflate_deflator"] = self._calculate_deflator_value(
             data[f"pydeflate_{self.price_deflator.price_kind}"],
             data["pydeflate_EXCHANGE_D"],
             data[f"pydeflate_EXCHANGE"],
+            data.get("pydeflate_EXCHANGE_BASE", data[f"pydeflate_EXCHANGE"]),
         )
 
         self.pydeflate_data = data
 
     def _calculate_deflator_value(
-        self, price_def: pd.Series, exchange_def: pd.Series, exchange_rate: pd.Series
+        self,
+        price_def: pd.Series,
+        exchange_def: pd.Series,
+        exchange_rate: pd.Series,
+        exchange_rate_base: pd.Series,
     ):
         """Compute the combined deflator value using price deflator, exchange deflator, and rates.
 
         Args:
             price_def (pd.Series): Series of price deflator values.
             exchange_def (pd.Series): Series of exchange deflator values.
-            exchange_rate (pd.Series): Series of exchange rates.
+            exchange_rate (pd.Series): Series of exchange rates for each year.
+            exchange_rate_base (pd.Series): Series of exchange rates at base year.
 
         Returns:
             pd.Series: Series with combined deflator values.
         """
-        return (
-            (exchange_def * exchange_rate) / price_def
-            if self.to_current
-            else price_def / (exchange_def * exchange_rate)
-        )
+        # Calculate deflator based on direction of conversion
+        if self.to_current:
+            # For constant -> current: deflator = 100 / (exchange_rate_base * price_def)
+            # Use BASE YEAR exchange rate (not year's rate) because:
+            # - We start with constant values at base year prices
+            # - The base year exchange rate converts between currencies at base year
+            # - The price_def adjusts for price level changes from base year to target year
+            return 100 / (exchange_rate_base * price_def)
+        else:
+            # For current -> constant: standard formula
+            return price_def / (exchange_def * exchange_rate)
 
     def _merge_components(self, df: pd.DataFrame, other: pd.DataFrame):
         """Combine data components, merging deflator and exchange rate information.