fpu-barometer-admin 0.3.0__py3-none-any.whl

fpu_barometer_admin/connectors/__init__.py

@@ -0,0 +1,56 @@
+"""Source data connectors for FPU admin."""
+
+from .base_connector import (
+    BaseConnector,
+    NoNewSourceArtifact,
+    SourceArtifact,
+    SourceArtifactSet,
+    SourceValidationError,
+)
+
+__all__ = [
+    "AcledConnector",
+    "BaseConnector",
+    "CpjConnector",
+    "ErtConnector",
+    "GdeltConnector",
+    "MFRRConnector",
+    "NoNewSourceArtifact",
+    "RrConnector",
+    "SourceArtifact",
+    "SourceArtifactSet",
+    "SourceValidationError",
+    "VdemConnector",
+]
+
+
+def __getattr__(name: str):
+    if name == "AcledConnector":
+        from .acled_connector import AcledConnector
+
+        return AcledConnector
+    if name == "CpjConnector":
+        from .cpj_connector import CpjConnector
+
+        return CpjConnector
+    if name == "ErtConnector":
+        from .ert_connector import ErtConnector
+
+        return ErtConnector
+    if name == "MFRRConnector":
+        from .mfrr_connector import MFRRConnector
+
+        return MFRRConnector
+    if name == "GdeltConnector":
+        from .gdelt_connector import GdeltConnector
+
+        return GdeltConnector
+    if name == "RrConnector":
+        from .rr_connector import RrConnector
+
+        return RrConnector
+    if name == "VdemConnector":
+        from .vdem_connector import VdemConnector
+
+        return VdemConnector
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

fpu_barometer_admin/connectors/acled_connector.py

@@ -0,0 +1,77 @@
+"""ACLED connector."""
+
+from __future__ import annotations
+
+from io import BytesIO
+from pathlib import Path
+
+import pandas as pd
+
+from fpu_barometer_admin.connectors.base_connector import SourceArtifactSet, SourceValidationError
+from fpu_barometer_admin.connectors.static_sources import static_source_for_dataset
+from fpu_barometer_admin.storage.objects import ObjectStorage
+
+
+class AcledConnector:
+    """Connector for the current static ACLED Event source file."""
+
+    dataset = "acled"
+    required_columns = {
+        "event_id",
+        "event_date",
+        "year",
+        "event_type",
+        "country",
+        "latitude",
+        "longitude",
+        "fatalities",
+    }
+
+    def __init__(self, object_storage: ObjectStorage):
+        self.object_storage = object_storage
+
+    def fetch(
+        self,
+        *,
+        run_id: str,
+        artifact_id: str,
+        since_watermark: str | None = None,
+    ) -> SourceArtifactSet:
+        """Fetch the deployed ACLED static source from logical storage."""
+
+        source_file = static_source_for_dataset(self.dataset)
+        payload = self.object_storage.read_bytes(source_file.logical_path)
+        df = pd.read_csv(BytesIO(payload), low_memory=False)
+        return self._write_source_artifact(
+            df,
+            source_name=Path(source_file.logical_path).stem,
+            run_id=run_id,
+            artifact_id=artifact_id,
+        )
+
+    def _write_source_artifact(
+        self,
+        df: pd.DataFrame,
+        *,
+        source_name: str,
+        run_id: str,
+        artifact_id: str,
+    ) -> SourceArtifactSet:
+        missing = self.required_columns - set(df.columns)
+        if missing:
+            missing_columns = sorted(missing)
+            raise SourceValidationError(
+                f"ACLED static source missing required columns: {missing_columns}",
+                metadata={"dataset": self.dataset, "missing_columns": missing_columns},
+            )
+
+        logical_path = (
+            f"source_artifacts/{self.dataset}/{artifact_id}/{source_name}.parquet"
+        )
+        self.object_storage.write_dataframe(logical_path, df, fail_if_exists=True)
+
+        return SourceArtifactSet(
+            dataset=self.dataset,
+            artifact_id=artifact_id,
+            logical_path=logical_path,
+        )

fpu_barometer_admin/connectors/base_connector.py

@@ -0,0 +1,60 @@
+"""Connector interfaces and Source Artifact Set models."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Protocol
+
+
+class SourceValidationError(ValueError):
+    """Raised when a Connector cannot validate a source as a Source Artifact."""
+
+    def __init__(self, message: str, *, metadata: dict[str, Any] | None = None):
+        super().__init__(message)
+        self.metadata = dict(metadata or {})
+
+
+class NoNewSourceArtifact(RuntimeError):
+    """Raised when an incremental Connector finds no new Source Artifact."""
+
+    def __init__(self, message: str, *, watermark_after: str | None = None):
+        super().__init__(message)
+        self.watermark_after = watermark_after
+
+
+@dataclass(frozen=True)
+class SourceArtifactSet:
+    """Explicit reference to the accepted Source Artifact for one dataset update."""
+
+    dataset: str
+    artifact_id: str
+    logical_path: str
+    watermark_after: str | None = None
+
+
+class BaseConnector(Protocol):
+    """Generic connector interface."""
+
+    dataset: str
+
+    def fetch(
+        self,
+        *,
+        run_id: str,
+        artifact_id: str,
+        since_watermark: str | None = None,
+    ) -> SourceArtifactSet:
+        """Fetch and validate source data, returning explicit artifact references."""
+
+
+# Backward-compatible name for older imports during the tracer refactor.
+SourceArtifact = SourceArtifactSet
+
+
+__all__ = [
+    "BaseConnector",
+    "NoNewSourceArtifact",
+    "SourceArtifact",
+    "SourceArtifactSet",
+    "SourceValidationError",
+]

fpu_barometer_admin/connectors/cpj_connector.py

@@ -0,0 +1,92 @@
+"""CPJ (Committee to Protect Journalists) connector."""
+
+from __future__ import annotations
+
+import logging
+
+import requests
+import pandas as pd
+
+from fpu_barometer_admin.connectors.base_connector import SourceArtifactSet, SourceValidationError
+from fpu_barometer_admin.storage.objects import ObjectStorage
+
+logger = logging.getLogger(__name__)
+
+CPJ_API_BASE = "https://cpj.org/wp-json/cpj-datamanager/v1"
+CPJ_PEOPLE_LIST_ENDPOINT = f"{CPJ_API_BASE}/people_list"
+CPJ_TIMEOUT_SECONDS = 120
+CPJ_PAGE_SIZE = 10000
+
+# Columns required in every CPJ API response row
+CPJ_REQUIRED_COLUMNS = {
+    "fullName",
+    "organizations",
+    "location",
+    "status",
+    "startDisplay",
+    "type",
+    "mtpage",
+}
+
+
+class CpjConnector:
+    """Connector for the CPJ (Committee to Protect Journalists) data API."""
+
+    dataset = "cpj"
+    required_columns = CPJ_REQUIRED_COLUMNS
+
+    def __init__(self, object_storage: ObjectStorage):
+        self.object_storage = object_storage
+
+    def fetch(
+        self,
+        *,
+        run_id: str,
+        artifact_id: str,
+        since_watermark: str | None = None,
+    ) -> SourceArtifactSet:
+        """
+        Fetch all CPJ records in a single API call using the CSV-export mode
+        (which returns JSON with all records, bypassing rate limits).
+        """
+
+        response = requests.get(
+            CPJ_PEOPLE_LIST_ENDPOINT,
+            params={
+                "export": "csv",
+                "limit": str(CPJ_PAGE_SIZE),
+            },
+            timeout=CPJ_TIMEOUT_SECONDS,
+        )
+        response.raise_for_status()
+        payload = response.json()
+        all_rows = payload.get("data", [])
+
+        if not all_rows:
+            raise SourceValidationError(
+                "No CPJ data returned",
+                metadata={"dataset": self.dataset},
+            )
+
+        df = pd.DataFrame(all_rows)
+
+        # Fill empty status fields with "Unknown"
+        df["status"] = df["status"].fillna("").replace("", "Unknown")
+
+        # Validate required columns exist in the combined DataFrame
+        missing = self.required_columns - set(df.columns)
+        if missing:
+            missing_columns = sorted(missing)
+            raise SourceValidationError(
+                f"CPJ data missing required columns: {missing_columns}",
+                metadata={"dataset": self.dataset, "missing_columns": missing_columns},
+            )
+
+        logical_path = f"source_artifacts/{self.dataset}/{artifact_id}/cpj.parquet"
+        self.object_storage.write_dataframe(logical_path, df, fail_if_exists=True)
+
+        return SourceArtifactSet(
+            dataset=self.dataset,
+            artifact_id=artifact_id,
+            logical_path=logical_path,
+        )

fpu_barometer_admin/connectors/ert_connector.py

@@ -0,0 +1,134 @@
+"""ERT connector — GitHub release asset style for vdeminstitute/ERT."""
+
+from __future__ import annotations
+
+import io
+import logging
+
+import pandas as pd
+import requests
+
+from fpu_barometer_admin.connectors.base_connector import (
+    NoNewSourceArtifact,
+    SourceArtifactSet,
+    SourceValidationError,
+)
+from fpu_barometer_admin.storage.objects import ObjectStorage
+
+
+logger = logging.getLogger(__name__)
+
+ERT_GITHUB_OWNER = "vdeminstitute"
+ERT_GITHUB_REPO = "ERT"
+ERT_RAW_FILE_PATH = "inst/ert.csv"
+ERT_API_LATEST_RELEASE = (
+    f"https://api.github.com/repos/{ERT_GITHUB_OWNER}/{ERT_GITHUB_REPO}/releases/latest"
+)
+ERT_RAW_BASE_URL = (
+    f"https://raw.githubusercontent.com/{ERT_GITHUB_OWNER}/{ERT_GITHUB_REPO}"
+)
+
+ERT_TIMEOUT_SECONDS = 120
+ERT_REQUIRED_COLUMNS = {"country_text_id", "country_name", "year"}
+
+
+class ErtConnector:
+    """Connector for ERT annual GitHub release CSV assets."""
+
+    dataset = "ert"
+
+    def __init__(self, object_storage: ObjectStorage):
+        self.object_storage = object_storage
+
+    def fetch(
+        self,
+        *,
+        run_id: str,
+        artifact_id: str,
+        since_watermark: str | None = None,
+    ) -> SourceArtifactSet:
+        release_info = self._latest_release_info()
+        tag_name = release_info["tag_name"]
+        etag = release_info.get("etag", tag_name)
+
+        watermark_after = self._compound_watermark(tag_name, etag=etag)
+
+        if since_watermark is not None and since_watermark == watermark_after:
+            raise NoNewSourceArtifact(
+                f"ERT release {tag_name} already processed",
+                watermark_after=watermark_after,
+            )
+
+        raw_url = f"{ERT_RAW_BASE_URL}/{tag_name}/{ERT_RAW_FILE_PATH}"
+        logger.info("Fetching ERT CSV from %s", raw_url)
+        response = requests.get(raw_url, timeout=ERT_TIMEOUT_SECONDS)
+        if response.status_code == 404:
+            raise SourceValidationError(
+                f"ERT CSV not found at {raw_url}",
+                metadata={
+                    "dataset": self.dataset,
+                    "tag": tag_name,
+                    "expected_path": ERT_RAW_FILE_PATH,
+                },
+            )
+        response.raise_for_status()
+
+        df = pd.read_csv(io.BytesIO(response.content), low_memory=False)
+        if df.empty:
+            raise SourceValidationError(
+                "ERT CSV produced empty DataFrame",
+                metadata={"dataset": self.dataset, "tag": tag_name},
+            )
+
+        missing = ERT_REQUIRED_COLUMNS - set(df.columns)
+        if missing:
+            missing_columns = sorted(missing)
+            raise SourceValidationError(
+                f"ERT source missing required columns: {missing_columns}",
+                metadata={
+                    "dataset": self.dataset,
+                    "tag": tag_name,
+                    "missing_columns": missing_columns,
+                },
+            )
+
+        logical_path = f"source_artifacts/{self.dataset}/{artifact_id}/ert.parquet"
+        self.object_storage.write_dataframe(logical_path, df, fail_if_exists=True)
+        return SourceArtifactSet(
+            dataset=self.dataset,
+            artifact_id=artifact_id,
+            logical_path=logical_path,
+            watermark_after=watermark_after,
+        )
+
+    def _latest_release_info(self) -> dict:
+        """Fetch the latest GitHub release metadata."""
+
+        response = requests.get(
+            ERT_API_LATEST_RELEASE, timeout=ERT_TIMEOUT_SECONDS
+        )
+        if response.status_code == 404:
+            raise SourceValidationError(
+                "ERT GitHub repository not found or has no releases",
+                metadata={
+                    "dataset": self.dataset,
+                    "repo": f"{ERT_GITHUB_OWNER}/{ERT_GITHUB_REPO}",
+                },
+            )
+        response.raise_for_status()
+        info = response.json()
+        if "tag_name" not in info:
+            raise SourceValidationError(
+                "ERT GitHub release response missing tag_name",
+                metadata={
+                    "dataset": self.dataset,
+                    "response_keys": list(info.keys()),
+                },
+            )
+        info["etag"] = response.headers.get("etag", "")
+        return info
+
+    def _compound_watermark(self, tag_name: str, *, etag: str) -> str:
+        """Build a compound watermark from release identity."""
+
+        return f"{ERT_GITHUB_OWNER}/{ERT_GITHUB_REPO}@{tag_name}:{ERT_RAW_FILE_PATH}:{etag}"