puremacro 0.92.0__py3-none-any.whl

puremacro/__init__.py

@@ -0,0 +1,8 @@
+"""puremacro — Pyodide-compatible empirical macro toolbox.
+
+See README.md for the iPad / juno.sh quickstart and module-by-module
+overview. Public API is curated in submodules; nothing is exported at
+the top level beyond the version string.
+"""
+__version__ = "0.92.0"
+__all__ = ["__version__"]

puremacro/_backend.py

@@ -0,0 +1,92 @@
+"""Shared pluggable compute backends for puremacro's compute-heavy modules.
+
+Four tiers:
+  - "numpy" : reference implementation and correctness ORACLE. Always
+    available; keeps puremacro's Pyodide-compatible core intact.
+  - "numba" : CPU JIT (+ prange). Compiled kernels, NOT an array namespace.
+  - "mlx"   : Apple-Silicon GPU. numpy-like namespace (mlx.core); xp-generic
+    kernels run on it unchanged. float32-only (Metal has no float64).
+  - "cupy"  : NVIDIA GPU. numpy-like namespace (cupy); xp-generic kernels run
+    unchanged. Optional [cuda] extra; cannot be installed/run without CUDA.
+
+numba/mlx/cupy are optional extras detected via importlib.util.find_spec and
+skipped, never hard-failed. (Promoted from models/nested_dmp/backend.py so the
+nested DMP model and the general vfi engine share one source of truth.)
+"""
+from __future__ import annotations
+
+import importlib.util
+
+import numpy as np
+
+SUPPORTED = ("numpy", "numba", "mlx", "cupy")
+
+
+def backend_available(name: str) -> bool:
+    """True if backend ``name`` can be used in this environment."""
+    if name == "numpy":
+        return True
+    if name in ("numba", "mlx", "cupy"):
+        return importlib.util.find_spec(name) is not None
+    raise ValueError(f"Unknown backend {name!r}; supported: {SUPPORTED}")
+
+
+def available_backends() -> tuple[str, ...]:
+    """Tuple of installed backends, numpy first."""
+    return tuple(b for b in SUPPORTED if backend_available(b))
+
+
+def get_array_namespace(name: str):
+    """Return a numpy-like array module for namespace-style backends.
+
+    "numpy"->numpy; "mlx"->mlx.core; "cupy"->cupy. "numba" is not a namespace.
+    """
+    if name == "numpy":
+        return np
+    if name == "mlx":
+        if not backend_available("mlx"):
+            raise ImportError(
+                "mlx not installed; on Apple Silicon run "
+                "`pip install puremacro[backend]`."
+            )
+        import mlx.core as mx
+
+        return mx
+    if name == "cupy":
+        if not backend_available("cupy"):
+            raise ImportError(
+                "cupy not installed; on an NVIDIA host run "
+                "`pip install puremacro[cuda]` (choose the wheel matching your "
+                "CUDA toolkit, e.g. cupy-cuda12x)."
+            )
+        import cupy as cp
+
+        return cp
+    if name == "numba":
+        raise ValueError(
+            "numba backend uses compiled kernels, not an array namespace; "
+            "call the kernels_numba.* functions directly."
+        )
+    raise ValueError(f"Unknown backend {name!r}; supported: {SUPPORTED}")
+
+
+def to_numpy(x) -> np.ndarray:
+    """Bring an array from any backend back to numpy.
+
+    cupy blocks implicit conversion, so route it through cp.asnumpy; numpy and
+    mlx (which forces evaluation) go through np.asarray.
+    """
+    if type(x).__module__.split(".")[0] == "cupy":
+        import cupy as cp
+
+        return cp.asnumpy(x)
+    return np.asarray(x)
+
+
+__all__ = [
+    "SUPPORTED",
+    "backend_available",
+    "available_backends",
+    "get_array_namespace",
+    "to_numpy",
+]

puremacro/_cache_db.py

@@ -0,0 +1,216 @@
+"""SQLite connection manager + schema bootstrap for puremacro's data
+infrastructure.
+
+A single SQLite file at ``~/.cache/puremacro/cache.db`` (overridable
+via ``$PUREMACRO_HTTP_CACHE_DIR``) hosts three tables:
+
+- ``http_cache`` — replaces the flat-file HTTP cache; backs
+  ``puremacro._http_cache.cache_read``/``cache_write``.
+- ``alfred_vintages`` — persistent FRED-ALFRED vintage panel; backs
+  ``puremacro.vintages.AlfredVintageStore``.
+- ``schema_version`` — registry for future migrations.
+
+WAL journal mode is enabled so multiple notebooks against the same DB
+do not block each other on writes. One ``sqlite3.Connection`` per
+process is kept alive in a module-level singleton.
+"""
+from __future__ import annotations
+
+import os
+import sqlite3
+from pathlib import Path
+
+
+_DDL_HTTP_CACHE = """
+CREATE TABLE IF NOT EXISTS http_cache (
+    key            TEXT PRIMARY KEY,
+    url            TEXT NOT NULL,
+    fetched_at     INTEGER NOT NULL,
+    content_type   TEXT,
+    body           BLOB NOT NULL
+);
+"""
+
+_DDL_HTTP_CACHE_IDX = (
+    "CREATE INDEX IF NOT EXISTS http_cache_fetched_at_idx "
+    "ON http_cache(fetched_at);"
+)
+
+_DDL_ALFRED_VINTAGES = """
+CREATE TABLE IF NOT EXISTS alfred_vintages (
+    series_id        TEXT NOT NULL,
+    observation_date TEXT NOT NULL,
+    vintage_date     TEXT NOT NULL,
+    value            REAL,
+    PRIMARY KEY (series_id, observation_date, vintage_date)
+);
+"""
+
+_DDL_ALFRED_VINTAGES_IDX = (
+    "CREATE INDEX IF NOT EXISTS alfred_vintages_series_vintage_idx "
+    "ON alfred_vintages(series_id, vintage_date);"
+)
+
+_DDL_CONNECTOR_EVENTS = """
+CREATE TABLE IF NOT EXISTS connector_events (
+    ts             INTEGER NOT NULL,
+    source         TEXT NOT NULL,
+    outcome        TEXT NOT NULL,
+    fallback_used  TEXT NOT NULL
+);
+"""
+
+_DDL_CONNECTOR_EVENTS_IDX = (
+    "CREATE INDEX IF NOT EXISTS connector_events_ts_source_idx "
+    "ON connector_events(ts, source);"
+)
+
+_DDL_SCHEMA_VERSION = """
+CREATE TABLE IF NOT EXISTS schema_version (
+    component TEXT PRIMARY KEY,
+    version   INTEGER NOT NULL
+);
+"""
+
+_SCHEMA_SEED = [("http_cache", 1), ("alfred_vintages", 1),
+                ("connector_events", 1)]
+
+
+def default_db_path() -> Path:
+    """Resolve the canonical cache DB path.
+
+    ``$PUREMACRO_HTTP_CACHE_DIR`` overrides everything:
+      - if it ends with ``.db``, return that path verbatim;
+      - otherwise treat as a directory and return ``<dir>/cache.db``.
+    Default: ``~/.cache/puremacro/cache.db``.
+    """
+    env = os.environ.get("PUREMACRO_HTTP_CACHE_DIR")
+    if env:
+        p = Path(env)
+        if p.suffix == ".db":
+            return p
+        return p / "cache.db"
+    return Path.home() / ".cache" / "puremacro" / "cache.db"
+
+
+def bootstrap_schema(conn: sqlite3.Connection) -> None:
+    """Idempotent: create tables + seed schema_version rows if missing."""
+    cur = conn.cursor()
+    cur.execute(_DDL_HTTP_CACHE)
+    cur.execute(_DDL_HTTP_CACHE_IDX)
+    cur.execute(_DDL_ALFRED_VINTAGES)
+    cur.execute(_DDL_ALFRED_VINTAGES_IDX)
+    cur.execute(_DDL_CONNECTOR_EVENTS)
+    cur.execute(_DDL_CONNECTOR_EVENTS_IDX)
+    cur.execute(_DDL_SCHEMA_VERSION)
+    for component, version in _SCHEMA_SEED:
+        cur.execute(
+            "INSERT OR IGNORE INTO schema_version (component, version) "
+            "VALUES (?, ?)",
+            (component, version),
+        )
+    conn.commit()
+
+
+_CONN: sqlite3.Connection | None = None
+_CONN_PATH: Path | None = None
+
+
+def get_conn(db_path: Path | None = None) -> sqlite3.Connection:
+    """Return the module-level singleton SQLite connection.
+
+    Lazily opens on first call. Enables WAL mode and bootstraps the
+    schema. If called twice with different ``db_path`` arguments
+    (e.g., in tests), closes the previous connection and opens a new one.
+    """
+    global _CONN, _CONN_PATH
+    target = db_path or default_db_path()
+    if _CONN is not None and _CONN_PATH == target:
+        return _CONN
+    if _CONN is not None:
+        _CONN.close()
+    target.parent.mkdir(parents=True, exist_ok=True)
+    conn = sqlite3.connect(
+        target, timeout=30.0, isolation_level=None, check_same_thread=False,
+    )
+    conn.execute("PRAGMA journal_mode=WAL")
+    conn.execute("PRAGMA synchronous=NORMAL")
+    conn.execute("PRAGMA foreign_keys=ON")
+    bootstrap_schema(conn)
+    _CONN = conn
+    _CONN_PATH = target
+    return _CONN
+
+
+def close_conn() -> None:
+    """Close the singleton connection (used by tests)."""
+    global _CONN, _CONN_PATH
+    if _CONN is not None:
+        try:
+            _CONN.close()
+        finally:
+            _CONN = None
+            _CONN_PATH = None
+
+
+def migrate_from_flat_files(
+    conn: sqlite3.Connection,
+    flat_cache_dir: Path,
+    *,
+    remove: bool = False,
+) -> int:
+    """Walk ``flat_cache_dir`` for ``*.bin`` + ``*.json`` sidecar pairs.
+
+    Inserts each into ``http_cache`` with ``INSERT OR IGNORE`` semantics
+    (idempotent). If ``remove=True``, unlinks each migrated file pair
+    after successful insert. Returns the count actually migrated.
+
+    Failures on a per-file basis are warned-and-skipped, not raised —
+    migration is opportunistic.
+    """
+    import json
+    import warnings
+
+    if not flat_cache_dir.exists():
+        return 0
+    migrated = 0
+    for bin_path in flat_cache_dir.glob("*.bin"):
+        sidecar = bin_path.with_suffix(".json")
+        if not sidecar.exists():
+            continue
+        try:
+            meta = json.loads(sidecar.read_text())
+            url = meta["url"]
+            fetched_at = int(float(meta["fetched_at"]))
+            content_type = meta.get("content_type")
+            body = bin_path.read_bytes()
+            key = bin_path.stem
+            cur = conn.execute(
+                "INSERT OR IGNORE INTO http_cache "
+                "(key, url, fetched_at, content_type, body) "
+                "VALUES (?, ?, ?, ?, ?)",
+                (key, url, fetched_at, content_type, body),
+            )
+            if cur.rowcount > 0:
+                migrated += 1
+                if remove:
+                    bin_path.unlink(missing_ok=True)
+                    sidecar.unlink(missing_ok=True)
+        except (OSError, ValueError, KeyError) as e:
+            warnings.warn(
+                f"puremacro._cache_db: skipping {bin_path.name}: {e}",
+                UserWarning,
+                stacklevel=2,
+            )
+            continue
+    conn.commit()
+    return migrated
+
+
+__all__ = [
+    "default_db_path",
+    "bootstrap_schema",
+    "get_conn",
+    "close_conn",
+    "migrate_from_flat_files",
+]

puremacro/_codes.py

@@ -0,0 +1,31 @@
+"""Country / aggregate code canonicalization.
+
+Used everywhere the panel might receive aggregate codes from upstream
+fetchers (OECD `EA20`, `EU27`, `OECD`; WUI / GPR `WLD`; etc.) so the
+panel only ever contains pure-country observations.
+"""
+from __future__ import annotations
+
+import pandas as pd
+
+IS_AGGREGATE: frozenset[str] = frozenset({
+    "EA", "EA12", "EA17", "EA19", "EA20",
+    "EU", "EU27", "EU27_2020", "EU28",
+    "OECD", "OECDE", "OECDA",
+    "G7", "G20", "BRICS",
+    "WLD", "WORLD", "ADV", "EME",
+    "LATAM", "ASIA", "AFRICA", "MENA",
+})
+
+
+def is_country(code: object) -> bool:
+    """True iff *code* is a length-3 alpha string and not in IS_AGGREGATE."""
+    if not isinstance(code, str):
+        return False
+    return len(code) == 3 and code.isalpha() and code.upper() not in IS_AGGREGATE
+
+
+def drop_aggregates(df: pd.DataFrame, code_col: str = "code") -> pd.DataFrame:
+    """Return a copy of *df* with rows whose *code_col* fails ``is_country`` removed."""
+    mask = df[code_col].apply(is_country)
+    return df.loc[mask].reset_index(drop=True)

puremacro/_http.py

@@ -0,0 +1,198 @@
+"""Shared HTTP helpers for puremacro fetchers and connectors.
+
+This module is the single canonical home of ``safe_get_bytes``,
+``safe_get_text``, ``safe_get_json`` and the supporting ``USER_AGENT``
++ ``DEFAULT_TIMEOUT`` constants. Promoted from
+``puremacro.narrative.sources._http`` in 0.6.0 so all fetchers
+(narrative connectors, ``puremacro.fetch.*``, instrument loaders)
+share one hardened path.
+
+See ``puremacro/narrative/sources/RETRY_POLICY.md`` for the
+contract every consumer adheres to: 30s default timeout, one-shot
+SSL fallback for older endpoints with stale CA bundles, optional
+keyword-only ``user_agent=`` override (added in 0.4.1) for endpoints
+behind a WAF that blocks the default agent string.
+"""
+from __future__ import annotations
+
+import json
+import ssl
+import urllib.error
+import urllib.request
+
+
+USER_AGENT = "Mozilla/5.0 (puremacro/narrative)"
+DEFAULT_TIMEOUT = 30.0
+
+
+def _request(url: str, timeout: float, user_agent: str | None = None) -> bytes:
+    ua = user_agent or USER_AGENT
+    req = urllib.request.Request(url, headers={"User-Agent": ua})
+    try:
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            return resp.read()
+    except (urllib.error.URLError, ssl.SSLError):
+        # One-shot fallback: some public endpoints (older OECD / IMF /
+        # ministry sites) ship certificates that Python's bundled CA
+        # store does not validate. Retry once with verification off.
+        # See RETRY_POLICY.md §3 for why we do not loop further.
+        ctx = ssl._create_unverified_context()
+        with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp:
+            return resp.read()
+
+
+def safe_get_bytes(url: str, timeout: float = DEFAULT_TIMEOUT,
+                   *, user_agent: str | None = None) -> bytes:
+    """Fetch ``url`` and return raw bytes. SSL fallback applied once.
+
+    ``user_agent`` overrides the default ``Mozilla/5.0 (puremacro/narrative)``
+    UA — needed for endpoints behind a WAF that blocks scripted clients.
+    """
+    return _request(url, timeout, user_agent=user_agent)
+
+
+def safe_get_text(url: str, timeout: float = DEFAULT_TIMEOUT,
+                  *, user_agent: str | None = None) -> str:
+    """Fetch ``url`` and return UTF-8 text (decode errors ignored).
+
+    See ``safe_get_bytes`` for the ``user_agent=`` semantics and
+    ``RETRY_POLICY.md §7`` for the WAF-bypass pattern.
+    """
+    return _request(url, timeout, user_agent=user_agent).decode(
+        "utf-8", errors="ignore",
+    )
+
+
+def safe_get_json(url: str, timeout: float = DEFAULT_TIMEOUT,
+                  *, user_agent: str | None = None) -> dict:
+    """Fetch ``url`` and return decoded JSON.
+
+    Empty / whitespace-only bodies return ``{}`` rather than raising,
+    matching the existing API-connector behaviour (e.g. GDELT v2 rate
+    limits sometimes return blank pages).
+
+    See ``safe_get_bytes`` for the ``user_agent=`` semantics and
+    ``RETRY_POLICY.md §7`` for the WAF-bypass pattern.
+    """
+    text = safe_get_text(url, timeout, user_agent=user_agent)
+    if not text.strip():
+        return {}
+    return json.loads(text)
+
+
+def post_json(url: str, payload: dict, *, timeout: float = DEFAULT_TIMEOUT,
+              headers: dict | None = None) -> dict:
+    """POST ``payload`` as JSON and return the decoded JSON response.
+
+    urllib-only (Pyodide-safe). HTTP errors propagate (not retried); a
+    transport/SSL error retries once with verification off, matching
+    ``_request``. Used by the local-LLM HTTP engine (Ollama / OpenAI-compatible).
+    """
+    data = json.dumps(payload).encode("utf-8")
+    hdrs = {"Content-Type": "application/json", "User-Agent": USER_AGENT}
+    if headers:
+        hdrs.update(headers)
+    req = urllib.request.Request(url, data=data, headers=hdrs, method="POST")
+    try:
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            return json.loads(resp.read().decode("utf-8"))
+    except urllib.error.HTTPError:
+        raise
+    except (urllib.error.URLError, ssl.SSLError):
+        ctx = ssl._create_unverified_context()
+        with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp:
+            return json.loads(resp.read().decode("utf-8"))
+
+
+# ---------------------------------------------------------------------------
+# Opt-in cached + rate-limited variants (0.60.0)
+#
+# Callers who want caching switch from ``safe_get_text`` to
+# ``safe_get_text_cached``. The existing helpers are unchanged.
+# ---------------------------------------------------------------------------
+
+import os
+import time
+import urllib.parse
+
+from ._http_cache import default_cache_dir, cache_read, cache_write
+
+
+# Per-host last-fetch timestamps (monotonic seconds).
+_HOST_LAST_FETCH: dict[str, float] = {}
+
+
+def _reset_throttle_state() -> None:
+    """Clear per-host throttle state. Test helper."""
+    _HOST_LAST_FETCH.clear()
+
+
+def _throttle(host: str, rate_limit_seconds: float) -> None:
+    """Sleep until ``rate_limit_seconds`` have elapsed since the last
+    fetch to ``host``. No-op on first call to a host."""
+    if rate_limit_seconds <= 0:
+        return
+    now = time.monotonic()
+    last = _HOST_LAST_FETCH.get(host)
+    if last is not None:
+        wait = rate_limit_seconds - (now - last)
+        if wait > 0:
+            time.sleep(wait)
+    _HOST_LAST_FETCH[host] = time.monotonic()
+
+
+def _host_of(url: str) -> str:
+    return urllib.parse.urlparse(url).netloc.lower()
+
+
+def safe_get_bytes_cached(
+    url: str,
+    timeout: float = DEFAULT_TIMEOUT,
+    *,
+    user_agent: str | None = None,
+    ttl_seconds: int = 30 * 24 * 3600,
+    rate_limit_seconds: float = 0.5,
+) -> bytes:
+    """Like ``safe_get_bytes`` but cached on disk.
+
+    Cache root: ``$PUREMACRO_HTTP_CACHE_DIR`` or ``~/.cache/puremacro/http``.
+    Bypass entirely with ``PUREMACRO_HTTP_NO_CACHE=1``.
+    """
+    bypass = os.environ.get("PUREMACRO_HTTP_NO_CACHE") == "1"
+    cache_dir = default_cache_dir()
+    if not bypass:
+        hit = cache_read(cache_dir, url, ttl_seconds=ttl_seconds)
+        if hit is not None:
+            return hit
+    _throttle(_host_of(url), rate_limit_seconds)
+    body = _request(url, timeout, user_agent=user_agent)
+    if not bypass:
+        cache_write(cache_dir, url, body)
+    return body
+
+
+def safe_get_text_cached(
+    url: str,
+    timeout: float = DEFAULT_TIMEOUT,
+    *,
+    user_agent: str | None = None,
+    ttl_seconds: int = 30 * 24 * 3600,
+    rate_limit_seconds: float = 0.5,
+) -> str:
+    """Like ``safe_get_text`` but cached on disk. See
+    ``safe_get_bytes_cached`` for env-var semantics.
+    """
+    return safe_get_bytes_cached(
+        url, timeout,
+        user_agent=user_agent,
+        ttl_seconds=ttl_seconds,
+        rate_limit_seconds=rate_limit_seconds,
+    ).decode("utf-8", errors="ignore")
+
+
+__all__ = [
+    "USER_AGENT", "DEFAULT_TIMEOUT",
+    "safe_get_bytes", "safe_get_text", "safe_get_json",
+    "post_json",
+    "safe_get_bytes_cached", "safe_get_text_cached",
+]