fpu-barometer-admin 0.3.0__py3-none-any.whl

fpu_barometer_admin/connectors/vdem_connector.py

@@ -0,0 +1,165 @@
+"""VDEM connector — GitHub release asset style for vdeminstitute/vdemdata."""
+
+from __future__ import annotations
+
+import io
+import logging
+
+import pyreadr
+import requests
+import pandas as pd
+
+from fpu_barometer_admin.connectors.base_connector import (
+    NoNewSourceArtifact,
+    SourceArtifactSet,
+    SourceValidationError,
+)
+from fpu_barometer_admin.storage.objects import ObjectStorage
+
+
+logger = logging.getLogger(__name__)
+
+VDEM_GITHUB_OWNER = "vdeminstitute"
+VDEM_GITHUB_REPO = "vdemdata"
+VDEM_RAW_FILE_PATH = "data/vdem.RData"
+VDEM_API_LATEST_RELEASE = (
+    f"https://api.github.com/repos/{VDEM_GITHUB_OWNER}/{VDEM_GITHUB_REPO}/releases/latest"
+)
+VDEM_RAW_BASE_URL = (
+    f"https://raw.githubusercontent.com/{VDEM_GITHUB_OWNER}/{VDEM_GITHUB_REPO}"
+)
+
+VDEM_TIMEOUT_SECONDS = 120
+VDEM_EXPECTED_PYREADR_KEYS = {"vdem"}
+
+
+class VdemConnector:
+    """Connector for VDEM annual GitHub release RData assets."""
+
+    dataset = "vdem"
+
+    def __init__(self, object_storage: ObjectStorage):
+        self.object_storage = object_storage
+
+    def fetch(
+        self,
+        *,
+        run_id: str,
+        artifact_id: str,
+        since_watermark: str | None = None,
+    ) -> SourceArtifactSet:
+        release_info = self._latest_release_info()
+        tag_name = release_info["tag_name"]
+        etag = release_info.get("etag", tag_name)
+
+        watermark_after = self._compound_watermark(tag_name, etag=etag)
+
+        if since_watermark is not None and since_watermark == watermark_after:
+            raise NoNewSourceArtifact(
+                f"VDEM release {tag_name} already processed",
+                watermark_after=watermark_after,
+            )
+
+        raw_url = f"{VDEM_RAW_BASE_URL}/{tag_name}/{VDEM_RAW_FILE_PATH}"
+        logger.info("Fetching VDEM RData from %s", raw_url)
+        response = requests.get(raw_url, timeout=VDEM_TIMEOUT_SECONDS)
+        if response.status_code == 404:
+            raise SourceValidationError(
+                f"VDEM RData not found at {raw_url}",
+                metadata={
+                    "dataset": self.dataset,
+                    "tag": tag_name,
+                    "expected_path": VDEM_RAW_FILE_PATH,
+                },
+            )
+        response.raise_for_status()
+
+        try:
+            parsed = pyreadr.read_r(io.BytesIO(response.content))
+        except Exception as exc:
+            raise SourceValidationError(
+                f"Failed to parse VDEM RData: {exc}",
+                metadata={"dataset": self.dataset, "tag": tag_name},
+            ) from exc
+
+        matched_keys = VDEM_EXPECTED_PYREADR_KEYS & set(parsed.keys())
+        if not matched_keys:
+            available_keys = list(parsed.keys())
+            raise SourceValidationError(
+                f"VDEM RData missing expected key. Available: {available_keys}",
+                metadata={
+                    "dataset": self.dataset,
+                    "tag": tag_name,
+                    "expected_keys": list(VDEM_EXPECTED_PYREADR_KEYS),
+                    "available_keys": available_keys,
+                },
+            )
+
+        df = parsed[list(matched_keys)[0]]
+        if not isinstance(df, pd.DataFrame) or df.empty:
+            raise SourceValidationError(
+                "VDEM RData produced empty DataFrame",
+                metadata={"dataset": self.dataset, "tag": tag_name},
+            )
+
+        df = self._storage_normalized(df)
+
+        logical_path = f"source_artifacts/{self.dataset}/{artifact_id}/vdem.parquet"
+        self.object_storage.write_dataframe(logical_path, df, fail_if_exists=True)
+        return SourceArtifactSet(
+            dataset=self.dataset,
+            artifact_id=artifact_id,
+            logical_path=logical_path,
+            watermark_after=watermark_after,
+        )
+
+    def _storage_normalized(self, df: pd.DataFrame) -> pd.DataFrame:
+        """Normalize RData-loaded columns so they survive Parquet round-trip."""
+
+        normalized = df.copy()
+        for column in normalized.select_dtypes(include="object").columns:
+            normalized[column] = normalized[column].map(self._source_value)
+        return normalized
+
+    def _source_value(self, value: object) -> object:
+        if value is None:
+            return None
+        try:
+            if pd.isna(value):
+                return None
+        except (TypeError, ValueError):
+            pass
+        if isinstance(value, (list, tuple, dict)):
+            import json
+
+            return json.dumps(value, ensure_ascii=False)
+        return str(value)
+
+    def _latest_release_info(self) -> dict:
+        """Fetch the latest GitHub release metadata."""
+
+        response = requests.get(
+            VDEM_API_LATEST_RELEASE, timeout=VDEM_TIMEOUT_SECONDS
+        )
+        if response.status_code == 404:
+            raise SourceValidationError(
+                "VDEM GitHub repository not found or has no releases",
+                metadata={"dataset": self.dataset, "repo": f"{VDEM_GITHUB_OWNER}/{VDEM_GITHUB_REPO}"},
+            )
+        response.raise_for_status()
+        info = response.json()
+        if "tag_name" not in info:
+            raise SourceValidationError(
+                "VDEM GitHub release response missing tag_name",
+                metadata={
+                    "dataset": self.dataset,
+                    "response_keys": list(info.keys()),
+                },
+            )
+        info["etag"] = response.headers.get("etag", "")
+        return info
+
+    def _compound_watermark(self, tag_name: str, *, etag: str) -> str:
+        """Build a compound watermark from release identity."""
+
+        return f"{VDEM_GITHUB_OWNER}/{VDEM_GITHUB_REPO}@{tag_name}:{VDEM_RAW_FILE_PATH}:{etag}"

fpu_barometer_admin/handlers/__init__.py

@@ -0,0 +1,6 @@
+"""Backend HTTP handler adapters.
+
+Azure Functions and local development HTTP adapters belong here and should stay
+thin: parse shared fpu_barometer models, read through storage, and serialize shared fpu_barometer
+response models.
+"""