fpu-barometer-admin 0.3.0__py3-none-any.whl

fpu_barometer_admin/connectors/gdelt_connector.py

@@ -0,0 +1,403 @@
+"""GDELT connector."""
+
+from __future__ import annotations
+
+import io
+import logging
+import os
+import re
+import zipfile
+from dataclasses import dataclass
+import pandas as pd
+import requests
+
+
+logger = logging.getLogger(__name__)
+
+from fpu_barometer_admin.connectors.base_connector import (
+    NoNewSourceArtifact,
+    SourceArtifactSet,
+    SourceValidationError,
+)
+from fpu_barometer_admin.storage.objects import ObjectStorage
+
+
+GDELT_MASTER_FILE_LIST = "http://data.gdeltproject.org/gdeltv2/masterfilelist.txt"
+GDELT_TIMEOUT_SECONDS = 60
+GDELT_EVENT_CODES = {
+    "091",
+    "092",
+    "110",
+    "111",
+    "112",
+    "1121",
+    "1122",
+    "1123",
+    "1125",
+    "113",
+    "114",
+    "115",
+    "116",
+    "1233",
+    "124",
+    "1234",
+    "1245",
+    "130",
+    "131",
+    "1321",
+    "138",
+    "1384",
+    "139",
+    "170",
+    "171",
+    "1711",
+    "1712",
+    "172",
+    "1721",
+    "173",
+    "174",
+    "176",
+    "180",
+    "181",
+    "182",
+    "1821",
+    "1822",
+    "1823",
+    "183",
+    "1831",
+    "1832",
+    "1833",
+    "1834",
+    "185",
+    "186",
+}
+
+
+@dataclass(frozen=True)
+class _ManifestEntry:
+    timestamp: str
+    url: str
+
+
+class GdeltConnector:
+    """Connector for bounded incremental GDELT Event export refreshes."""
+
+    dataset = "gdelt"
+    source_columns = (
+        "GlobalEventID",
+        "SQLDATE",
+        "MonthYear",
+        "Year",
+        "FractionDate",
+        "Actor1Code",
+        "Actor1Name",
+        "Actor1CountryCode",
+        "Actor1KnownGroupCode",
+        "Actor1EthnicCode",
+        "Actor1Religion1Code",
+        "Actor1Religion2Code",
+        "Actor1Type1Code",
+        "Actor1Type2Code",
+        "Actor1Type3Code",
+        "Actor2Code",
+        "Actor2Name",
+        "Actor2CountryCode",
+        "Actor2KnownGroupCode",
+        "Actor2EthnicCode",
+        "Actor2Religion1Code",
+        "Actor2Religion2Code",
+        "Actor2Type1Code",
+        "Actor2Type2Code",
+        "Actor2Type3Code",
+        "IsRootEvent",
+        "EventCode",
+        "EventBaseCode",
+        "EventRootCode",
+        "QuadClass",
+        "GoldsteinScale",
+        "NumMentions",
+        "NumSources",
+        "NumArticles",
+        "AvgTone",
+        "Actor1Geo_Type",
+        "Actor1Geo_FullName",
+        "Actor1Geo_CountryCode",
+        "Actor1Geo_ADM1Code",
+        "Actor1Geo_ADM2Code",
+        "Actor1Geo_Lat",
+        "Actor1Geo_Long",
+        "Actor1Geo_FeatureID",
+        "Actor2Geo_Type",
+        "Actor2Geo_FullName",
+        "Actor2Geo_CountryCode",
+        "Actor2Geo_ADM1Code",
+        "Actor2Geo_ADM2Code",
+        "Actor2Geo_Lat",
+        "Actor2Geo_Long",
+        "Actor2Geo_FeatureID",
+        "ActionGeo_Type",
+        "ActionGeo_FullName",
+        "ActionGeo_CountryCode",
+        "ActionGeo_ADM1Code",
+        "ActionGeo_ADM2Code",
+        "ActionGeo_Lat",
+        "ActionGeo_Long",
+        "ActionGeo_FeatureID",
+        "DATEADDED",
+        "SOURCEURL",
+    )
+    required_columns = {
+        "GlobalEventID",
+        "SQLDATE",
+        "Actor1Name",
+        "Actor1CountryCode",
+        "Actor1Type1Code",
+        "Actor2Name",
+        "Actor2CountryCode",
+        "Actor2Type1Code",
+        "EventCode",
+        "ActionGeo_CountryCode",
+        "ActionGeo_Lat",
+        "ActionGeo_Long",
+        "DATEADDED",
+        "SOURCEURL",
+    }
+    media_terms = ("media", "journalist", "reporter", "press", "news")
+    media_type_codes = {"MED", "JRN"}
+
+    def __init__(
+        self,
+        object_storage: ObjectStorage,
+        *,
+        manifest_url: str = GDELT_MASTER_FILE_LIST,
+        timeout_seconds: int = GDELT_TIMEOUT_SECONDS,
+        max_files_per_run: int | None = None,
+        bootstrap_file_limit: int | None = None,
+    ):
+        self.object_storage = object_storage
+        self.manifest_url = manifest_url
+        self.timeout_seconds = timeout_seconds
+        self.max_files_per_run = max_files_per_run or _positive_int_env(
+            "GDELT_MAX_FILES_PER_RUN"
+        )
+        self.bootstrap_file_limit = (
+            bootstrap_file_limit
+            if bootstrap_file_limit is not None
+            else _positive_int_env("GDELT_BOOTSTRAP_FILE_LIMIT", default=1)
+        )
+
+    def fetch(
+        self,
+        *,
+        run_id: str,
+        artifact_id: str,
+        since_watermark: str | None = None,
+    ) -> SourceArtifactSet:
+        entries = self._manifest_entries()
+        selected = self._select_entries(entries, since_watermark=since_watermark)
+        if not selected:
+            raise NoNewSourceArtifact("GDELT manifest has no newer event export files")
+
+        frames: list[pd.DataFrame] = []
+        latest_successful_timestamp: str | None = None
+        earliest_skipped_timestamp: str | None = None
+        for entry in selected:
+            try:
+                frame = self._read_remote_zip_csv(entry.url)
+                frames.append(frame)
+                latest_successful_timestamp = entry.timestamp
+            except requests.exceptions.HTTPError as exc:
+                if exc.response is not None and exc.response.status_code == 404:
+                    logger.warning(
+                        "GDELT export file not yet available, skipping: %s",
+                        entry.url,
+                    )
+                    if earliest_skipped_timestamp is None:
+                        earliest_skipped_timestamp = entry.timestamp
+                    continue
+                raise
+
+        if not frames:
+            raise NoNewSourceArtifact(
+                "GDELT export files listed in manifest are not yet available",
+                watermark_after=since_watermark,
+            )
+
+        # Determine watermark so we never skip data due to transient 404s.
+        # If any file returned 404, only advance the watermark to the last
+        # selected entry BEFORE the first gap.  Files at and after the gap
+        # will be retried on the next run (reprocessing already-seen files
+        # is safe because the processor deduplicates by event_id against
+        # the baseline processed dataset).
+        if earliest_skipped_timestamp is not None:
+            watermark_after = since_watermark or ""
+            for entry in selected:
+                if entry.timestamp >= earliest_skipped_timestamp:
+                    break
+                watermark_after = entry.timestamp
+        else:
+            watermark_after = selected[-1].timestamp
+
+        return self._write_source_artifact(
+            frames,
+            artifact_id=artifact_id,
+            watermark_after=watermark_after,
+            empty_message="GDELT export files contained no Barometer-relevant Event rows",
+        )
+
+    def _manifest_entries(self) -> list[_ManifestEntry]:
+        response = requests.get(self.manifest_url, timeout=self.timeout_seconds)
+        response.raise_for_status()
+        entries: list[_ManifestEntry] = []
+        for line in response.text.splitlines():
+            match = re.search(r"(https?://\S+/(\d{14})\.export\.CSV\.zip)", line)
+            if match:
+                entries.append(
+                    _ManifestEntry(timestamp=match.group(2), url=match.group(1))
+                )
+        return sorted(
+            {entry.url: entry for entry in entries}.values(),
+            key=lambda entry: entry.timestamp,
+        )
+
+    def _select_entries(
+        self, entries: list[_ManifestEntry], *, since_watermark: str | None
+    ) -> list[_ManifestEntry]:
+        if since_watermark:
+            candidates = [
+                entry for entry in entries if entry.timestamp > since_watermark
+            ]
+        else:
+            candidates = entries[-self.bootstrap_file_limit :]
+        candidates = sorted(candidates, key=lambda entry: entry.timestamp)
+        if self.max_files_per_run is not None:
+            candidates = candidates[: self.max_files_per_run]
+        return candidates
+
+    def _write_source_artifact(
+        self,
+        frames: list[pd.DataFrame],
+        *,
+        artifact_id: str,
+        watermark_after: str,
+        empty_message: str,
+    ) -> SourceArtifactSet:
+        source = pd.concat(frames, ignore_index=True) if frames else pd.DataFrame()
+        self._validate_source_columns(source)
+        source = self._filter_relevant_rows(source)
+        if source.empty:
+            raise NoNewSourceArtifact(empty_message, watermark_after=watermark_after)
+
+        logical_path = f"source_artifacts/{self.dataset}/{artifact_id}/gdelt.parquet"
+        self.object_storage.write_dataframe(logical_path, source, fail_if_exists=True)
+        return SourceArtifactSet(
+            dataset=self.dataset,
+            artifact_id=artifact_id,
+            logical_path=logical_path,
+            watermark_after=watermark_after,
+        )
+
+    def _read_remote_zip_csv(self, url: str) -> pd.DataFrame:
+        response = requests.get(url, timeout=self.timeout_seconds)
+        response.raise_for_status()
+        return self._read_zip_csv_bytes(response.content, source_label=url)
+
+    def _read_zip_csv_bytes(self, payload: bytes, *, source_label: str) -> pd.DataFrame:
+        try:
+            with zipfile.ZipFile(io.BytesIO(payload)) as archive:
+                members = [
+                    name for name in archive.namelist() if name.upper().endswith("CSV")
+                ]
+                if not members:
+                    raise SourceValidationError(
+                        f"GDELT zip contains no CSV member: {source_label}",
+                        metadata={
+                            "dataset": self.dataset,
+                            "source_url": source_label,
+                            "expected_shape": "zip_csv_member",
+                        },
+                    )
+                with archive.open(members[0]) as handle:
+                    frame = self._read_gdelt_csv(handle, source_label=source_label)
+        except zipfile.BadZipFile as exc:
+            raise SourceValidationError(
+                f"GDELT source is not a valid zip: {source_label}",
+                metadata={
+                    "dataset": self.dataset,
+                    "source_url": source_label,
+                    "expected_shape": "zip_file",
+                },
+            ) from exc
+        return frame
+
+    def _read_gdelt_csv(self, source, *, source_label: str) -> pd.DataFrame:
+        frame = pd.read_csv(
+            source,
+            sep="\t",
+            header=None,
+            dtype=str,
+            keep_default_na=False,
+        )
+        if frame.shape[1] != len(self.source_columns):
+            raise SourceValidationError(
+                f"GDELT source has {frame.shape[1]} columns; expected {len(self.source_columns)}",
+                metadata={
+                    "dataset": self.dataset,
+                    "source_url": source_label,
+                    "actual_columns": frame.shape[1],
+                    "expected_columns": len(self.source_columns),
+                },
+            )
+        frame.columns = self.source_columns
+        return frame
+
+    def _validate_source_columns(self, source: pd.DataFrame) -> None:
+        missing = self.required_columns - set(source.columns)
+        if missing:
+            missing_columns = sorted(missing)
+            raise SourceValidationError(
+                f"GDELT source missing required columns: {missing_columns}",
+                metadata={"dataset": self.dataset, "missing_columns": missing_columns},
+            )
+
+    def _filter_relevant_rows(self, source: pd.DataFrame) -> pd.DataFrame:
+        filtered = source[
+            source["EventCode"].astype(str).isin(GDELT_EVENT_CODES)
+        ].copy()
+        if filtered.empty:
+            return filtered
+        return filtered.loc[self._media_relevance_mask(filtered)].reset_index(drop=True)
+
+    def _media_relevance_mask(self, source: pd.DataFrame) -> pd.Series:
+        mask = pd.Series(False, index=source.index)
+        for column in (
+            "Actor1Type1Code",
+            "Actor1Type2Code",
+            "Actor1Type3Code",
+            "Actor2Type1Code",
+            "Actor2Type2Code",
+            "Actor2Type3Code",
+        ):
+            if column in source:
+                mask = mask | source[column].astype(str).str.upper().isin(
+                    self.media_type_codes
+                )
+        for column in ("Actor1Name", "Actor2Name", "Actor1Code", "Actor2Code"):
+            if column in source:
+                lowered = source[column].astype(str).str.lower()
+                for term in self.media_terms:
+                    mask = mask | lowered.str.contains(term, regex=False)
+        return mask
+
+
+def _positive_int_env(name: str, default: int | None = None) -> int | None:
+    raw = os.environ.get(name)
+    if raw is None or raw == "":
+        return default
+    value = int(raw)
+    if value <= 0:
+        raise ValueError(f"{name} must be a positive integer")
+    return value
+
+
+__all__ = ["GDELT_EVENT_CODES", "GDELT_MASTER_FILE_LIST", "GdeltConnector"]

fpu_barometer_admin/connectors/mfrr_connector.py

@@ -0,0 +1,171 @@
+"""MFRR connector."""
+
+from __future__ import annotations
+
+import json
+
+import requests
+import pandas as pd
+
+from fpu_barometer_admin.connectors.base_connector import SourceArtifactSet, SourceValidationError
+from fpu_barometer_admin.storage.objects import ObjectStorage
+
+
+MFRR_API_ENDPOINT = "https://www.mapmf.org/meili/multi-search"
+MFRR_API_TOKEN = "c129ca42527c52965c80099ab1a869f40de8ec3b698d1e361b0cf7402c6d48a1"
+MFRR_PAGE_SIZE = 1000
+MFRR_TIMEOUT_SECONDS = 60
+MFRR_LIMIT = 999999999999
+
+MFRR_ATTRIBUTES_TO_RETRIEVE = [
+    "id",
+    "title",
+    "content",
+    "country",
+    "date",
+    "year",
+    "published_at",
+    "published_at_date",
+    "has_date_of_incidence",
+    "type_of_incident",
+    "top_type_of_incident",
+    "type_of_incident_leaves",
+    "attacked_count",
+    "coe_link",
+    "news_source_links",
+    "internet_source_links",
+    "gender",
+    "type_of_journalist_or_media_actor",
+    "source_of_incident",
+    "context_of_incident",
+    "subjects",
+    "who_was_attacked",
+    "region_names",
+    "region_ids",
+    "consolidated_count",
+    "specific_topic",
+    "project",
+    "_geo_lat",
+    "_geo_lng",
+    "_geo.lat",
+    "_geo.lng",
+    "_geo",
+]
+
+
+class MFRRConnector:
+    """Connector for the MFRR API Event source."""
+
+    dataset = "mfrr"
+    required_columns = {
+        "id",
+        "country",
+        "date",
+        "year",
+        "published_at",
+        "published_at_date",
+        "type_of_incident",
+        "top_type_of_incident",
+        "type_of_incident_leaves",
+        "attacked_count",
+    }
+
+    def __init__(self, object_storage: ObjectStorage):
+        self.object_storage = object_storage
+
+    def fetch(
+        self,
+        *,
+        run_id: str,
+        artifact_id: str,
+        since_watermark: str | None = None,
+    ) -> SourceArtifactSet:
+        response = requests.post(
+            MFRR_API_ENDPOINT,
+            headers={"Authorization": f"Bearer {MFRR_API_TOKEN}"},
+            json={
+                "queries": [
+                    {
+                        "indexUid": "alerts",
+                        "q": "",
+                        "sort": ["timestamp:desc"],
+                        "limit": MFRR_LIMIT,
+                        "offset": 0,
+                        "attributesToRetrieve": MFRR_ATTRIBUTES_TO_RETRIEVE,
+                    }
+                ]
+            },
+            timeout=MFRR_TIMEOUT_SECONDS,
+        )
+        response.raise_for_status()
+        payload = response.json()
+        hits = self._hits_from_response(payload)
+        df = pd.DataFrame(hits)
+        missing = self.required_columns - set(df.columns)
+        if missing:
+            missing_columns = sorted(missing)
+            raise SourceValidationError(
+                f"MFRR API source missing required columns: {missing_columns}",
+                metadata={"dataset": self.dataset, "missing_columns": missing_columns},
+            )
+        df = self._storage_normalized(df)
+
+        logical_path = f"source_artifacts/{self.dataset}/{artifact_id}/mfrr.parquet"
+        self.object_storage.write_dataframe(logical_path, df, fail_if_exists=True)
+        return SourceArtifactSet(
+            dataset=self.dataset,
+            artifact_id=artifact_id,
+            logical_path=logical_path,
+        )
+
+    def _storage_normalized(self, df: pd.DataFrame) -> pd.DataFrame:
+        """Normalize source-shaped API values so mixed JSON-ish columns survive Parquet."""
+
+        normalized = df.copy()
+        for column in normalized.select_dtypes(include="object").columns:
+            normalized[column] = normalized[column].map(self._source_value)
+        return normalized
+
+    def _source_value(self, value: object) -> object:
+        if value is None:
+            return None
+        try:
+            if pd.isna(value):
+                return None
+        except (TypeError, ValueError):
+            pass
+        if isinstance(value, (list, tuple, dict)):
+            return json.dumps(value, ensure_ascii=False)
+        return str(value)
+
+    def _hits_from_response(self, payload: object) -> list[dict]:
+        if not isinstance(payload, dict):
+            raise SourceValidationError(
+                "MFRR API response must be a JSON object",
+                metadata={"dataset": self.dataset, "expected_shape": "json_object"},
+            )
+        results = payload.get("results")
+        if not isinstance(results, list) or not results:
+            raise SourceValidationError(
+                "MFRR API response results must be a non-empty list",
+                metadata={
+                    "dataset": self.dataset,
+                    "expected_shape": "non_empty_results_list",
+                },
+            )
+        first = results[0]
+        if not isinstance(first, dict) or "hits" not in first:
+            raise SourceValidationError(
+                "MFRR API first result must contain hits",
+                metadata={
+                    "dataset": self.dataset,
+                    "expected_shape": "first_result_with_hits",
+                },
+            )
+        hits = first["hits"]
+        if not isinstance(hits, list):
+            raise SourceValidationError(
+                "MFRR API hits must be a list",
+                metadata={"dataset": self.dataset, "expected_shape": "hits_list"},
+            )
+        return hits

fpu_barometer_admin/connectors/rr_connector.py

@@ -0,0 +1,84 @@
+"""RR (Reporters Respond) connector."""
+
+from __future__ import annotations
+
+from io import BytesIO
+from pathlib import Path
+
+import pandas as pd
+
+from fpu_barometer_admin.connectors.base_connector import SourceArtifactSet, SourceValidationError
+from fpu_barometer_admin.connectors.static_sources import static_source_for_dataset
+from fpu_barometer_admin.storage.objects import ObjectStorage
+
+
+class RrConnector:
+    """Connector for the static RR (Reporters Respond) source file."""
+
+    dataset = "rr"
+    required_columns = {
+        "Submission Date",
+        "Nationality:",
+        "Current location:",
+        "Region",
+        "Year",
+        "Decision",
+        "Considered under",
+        "Prevention or Protection",
+        "Type of Assistance",
+        "Legal Threat",
+        "EUR Amount",
+        "Number of Journalists Supported",
+        "I am applying to Reporters Respond as:",
+        "How do you identify as?",
+        "Submission ID",
+    }
+
+    def __init__(self, object_storage: ObjectStorage):
+        self.object_storage = object_storage
+
+    def fetch(
+        self,
+        *,
+        run_id: str,
+        artifact_id: str,
+        since_watermark: str | None = None,
+    ) -> SourceArtifactSet:
+        """Fetch the deployed RR static source from logical storage."""
+
+        source_file = static_source_for_dataset(self.dataset)
+        payload = self.object_storage.read_bytes(source_file.logical_path)
+        df = pd.read_csv(BytesIO(payload), low_memory=False)
+        return self._write_source_artifact(
+            df,
+            source_name=Path(source_file.logical_path).stem,
+            run_id=run_id,
+            artifact_id=artifact_id,
+        )
+
+    def _write_source_artifact(
+        self,
+        df: pd.DataFrame,
+        *,
+        source_name: str,
+        run_id: str,
+        artifact_id: str,
+    ) -> SourceArtifactSet:
+        missing = self.required_columns - set(df.columns)
+        if missing:
+            missing_columns = sorted(missing)
+            raise SourceValidationError(
+                f"RR static source missing required columns: {missing_columns}",
+                metadata={"dataset": self.dataset, "missing_columns": missing_columns},
+            )
+
+        logical_path = (
+            f"source_artifacts/{self.dataset}/{artifact_id}/{source_name}.parquet"
+        )
+        self.object_storage.write_dataframe(logical_path, df, fail_if_exists=True)
+
+        return SourceArtifactSet(
+            dataset=self.dataset,
+            artifact_id=artifact_id,
+            logical_path=logical_path,
+        )

fpu_barometer_admin/connectors/static_sources.py

@@ -0,0 +1,41 @@
+"""Static source file declarations for dataset Connectors."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class StaticSourceFile:
+    """A deploy-time static source file copied into logical storage."""
+
+    dataset: str
+    local_path: Path
+    logical_path: str
+
+
+STATIC_SOURCE_FILES: tuple[StaticSourceFile, ...] = (
+    StaticSourceFile(
+        dataset="acled",
+        local_path=Path("data") / "static" / "acled.csv",
+        logical_path="source_artifacts/acled/acled.csv",
+    ),
+    StaticSourceFile(
+        dataset="rr",
+        local_path=Path("data") / "static" / "rr.csv",
+        logical_path="source_artifacts/rr/rr.csv",
+    ),
+)
+
+
+def static_source_for_dataset(dataset: str) -> StaticSourceFile:
+    """Return the configured static source file for a dataset."""
+
+    for source_file in STATIC_SOURCE_FILES:
+        if source_file.dataset == dataset:
+            return source_file
+    raise ValueError(f"Dataset {dataset!r} has no configured static source file")
+
+
+__all__ = ["STATIC_SOURCE_FILES", "StaticSourceFile", "static_source_for_dataset"]