fpu-barometer-admin 0.3.0__py3-none-any.whl

fpu_barometer_admin/processors/acled_processor.py

@@ -0,0 +1,263 @@
+"""ACLED processor."""
+
+from __future__ import annotations
+
+import pandas as pd
+
+from fpu_barometer_admin.connectors import SourceArtifactSet
+from fpu_barometer_admin.schemas.event import (
+    CountryCanonicalizer,
+    EventSchemaValidationError,
+    canonical_event_id,
+    incident_tags,
+    validate_event_dataframe,
+)
+from fpu_barometer_admin.storage.objects import ObjectStorage
+
+
+class AcledProcessor:
+    """Transform ACLED Source Artifacts into canonical Event data."""
+
+    dataset = "acled"
+
+    def __init__(self, object_storage: ObjectStorage):
+        self.object_storage = object_storage
+        self.country_canonicalizer = CountryCanonicalizer()
+
+    def process(
+        self,
+        artifact_set: SourceArtifactSet,
+        *args: object,
+        **kwargs: object,
+    ) -> pd.DataFrame:
+        if artifact_set.dataset != self.dataset:
+            raise ValueError(
+                f"ACLED processor cannot process dataset {artifact_set.dataset!r}"
+            )
+        source = self.object_storage.read_dataframe(artifact_set.logical_path)
+        self._validate_source_columns(source)
+
+        processed = pd.DataFrame(index=source.index)
+        processed["event_id"] = source["event_id"].map(
+            lambda value: canonical_event_id(self.dataset, value)
+        )
+        processed["dataset"] = self.dataset
+
+        countries = source.apply(self._canonical_country, axis=1)
+        unresolved = countries.isna()
+        if unresolved.any():
+            unresolved_values = (
+                source.loc[unresolved, "country"].dropna().astype(str).unique()[:5]
+            )
+            raise EventSchemaValidationError(
+                "ACLED Event rows have unresolved countries: "
+                f"{list(unresolved_values)}"
+            )
+        processed["iso3"] = countries.map(lambda country: country.iso3)
+        processed["country_name"] = countries.map(lambda country: country.country_name)
+
+        dates = self._parse_event_dates(source["event_date"])
+        if dates.isna().any():
+            bad_dates = source.loc[dates.isna(), "event_date"].head().tolist()
+            raise EventSchemaValidationError(
+                f"ACLED Event rows have invalid event_date values: {bad_dates}"
+            )
+        processed["date"] = dates.dt.date
+        processed["date_precision"] = self._date_precision(source)
+        processed["year"] = dates.dt.year.astype("Int64")
+
+        incident_values = source.apply(self._incident_values, axis=1)
+        processed["type_of_incident"] = incident_values.map(lambda values: values[0])
+        processed["top_type_of_incident"] = incident_values.map(
+            lambda values: values[1]
+        )
+        processed["type_of_incident_leaves"] = incident_values.map(
+            lambda values: values[2]
+        )
+
+        processed["processed_at"] = pd.Timestamp.now(tz="UTC")
+        processed["n_people_affected"] = self._people_affected(source)
+
+        processed["region"] = source.get("admin1")
+        processed["latitude"] = self._numeric(source.get("latitude"), source.index)
+        processed["longitude"] = self._numeric(source.get("longitude"), source.index)
+        processed["description"] = source.get("notes")
+        processed["source_url"] = pd.NA
+        processed["attacked_count"] = self._attacked_count(source)
+        processed["gender"] = pd.NA
+        processed["media_role"] = pd.NA
+        processed["perpetrator_type"] = pd.NA
+        self._validate_coordinates(processed)
+        return validate_event_dataframe(processed)
+
+    def _validate_source_columns(self, source: pd.DataFrame) -> None:
+        required = {"event_id", "event_date", "country", "event_type", "fatalities"}
+        missing = required - set(source.columns)
+        if missing:
+            raise EventSchemaValidationError(
+                f"ACLED Source Artifact missing processor columns: {sorted(missing)}"
+            )
+
+    def _canonical_country(self, row: pd.Series):
+        return self.country_canonicalizer.from_values(
+            iso_numeric=row.get("iso"), country_name=row.get("country")
+        )
+
+    def _parse_event_dates(self, values: pd.Series) -> pd.Series:
+        month_map = {
+            "Januar": "January",
+            "Februar": "February",
+            "März": "March",
+            "April": "April",
+            "Mai": "May",
+            "Juni": "June",
+            "Juli": "July",
+            "August": "August",
+            "September": "September",
+            "Oktober": "October",
+            "November": "November",
+            "Dezember": "December",
+        }
+        normalized = values.astype(str)
+        for german, english in month_map.items():
+            normalized = normalized.str.replace(german, english, regex=False)
+        return pd.to_datetime(normalized, format="%d-%B-%Y", errors="coerce")
+
+    def _date_precision(self, source: pd.DataFrame) -> pd.Series:
+        if "time_precision" not in source.columns:
+            return pd.Series(["day"] * len(source), index=source.index, dtype="object")
+        precision = pd.to_numeric(source["time_precision"], errors="coerce")
+        return precision.map({1: "day", 2: "month", 3: "year"}).fillna("day")
+
+
+    def _incident_values(
+        self, row: pd.Series
+    ) -> tuple[list[str], list[str], list[str]]:
+        text = " ".join(
+            str(row.get(column, ""))
+            for column in (
+                "disorder_type",
+                "event_type",
+                "sub_event_type",
+                "notes",
+                "tags",
+            )
+            if row.get(column) is not None and not pd.isna(row.get(column))
+        ).lower()
+        top: list[str] = []
+        leaves: list[str] = []
+
+        if self._flag_enabled(row.get("arrested_mentioned")):
+            top.append("Legal incident")
+            leaves.append("Arrest/detention/imprisonment")
+        if self._flag_enabled(row.get("abducted_mentioned")) or self._flag_enabled(row.get("disappeared_mentioned")):
+            top.append("Physical assault")
+            leaves.append("Abduction/kidnapping")
+        if "sexual" in text:
+            top.append("Physical assault")
+            leaves.append("Sexual assault")
+        if any(
+            token in text
+            for token in ("attack", "assault", "violence")
+        ):
+            top.append("Physical assault")
+            fatalities = pd.to_numeric(
+                pd.Series([row.get("fatalities")]), errors="coerce"
+            ).iloc[0]
+            if pd.notna(fatalities) and fatalities > 0:
+                leaves.append("Death (physical assault resulting in death)")
+        if self._flag_enabled(row.get("killed_mentioned")):
+            top.append("Physical assault")
+            leaves.append("Death (physical assault resulting in death)")
+        if self._flag_enabled(row.get("tortured_mentioned")):
+            top.append("Physical assault")
+            leaves.append("Injury (physical assault resulting in injury)")
+
+        if any(token in text for token in ("property", "equipment", "raid")):
+            top.append("Attack to property")
+            if "equipment" in text:
+                leaves.append("Equipment")
+            elif "raid" in text:
+                leaves.append("Raid")
+            else:
+                leaves.append("Property (incl. houses, cars, personal belongings)")
+        if any(token in text for token in ("threat", "intimidat")):
+            top.append("Verbal attack")
+            leaves.append("Threatening")
+        if any(token in text for token in ("harass", "insult", "discredit")):
+            top.append("Verbal attack")
+            leaves.append("Insult / harassment / discredit")
+        if any(
+            token in text
+            for token in ("protest", "blocked", "censor", "journalistic activity")
+        ):
+            top.append("Interference")
+            leaves.append("Blocked journalistic activity")
+
+        if not top:
+            top.append("Unknown")
+        if not leaves:
+            leaves.append("Unknown")
+        return incident_tags(top=top, leaves=leaves)
+
+    def _flag_enabled(self, value: object) -> bool:
+        if value is None or pd.isna(value):
+            return False
+        if isinstance(value, str):
+            return value.strip().lower() in {"1", "true", "yes", "y"}
+        return bool(
+            pd.to_numeric(pd.Series([value]), errors="coerce").fillna(0).iloc[0]
+        )
+
+    def _people_affected(self, source: pd.DataFrame) -> pd.Series:
+        counts = self._affected_count_components(source).sum(axis=1)
+        return counts.clip(lower=1).astype("Int64")
+
+    def _attacked_count(self, source: pd.DataFrame) -> pd.Series:
+        counts = self._affected_count_components(source)
+        if counts.empty:
+            return pd.Series([pd.NA] * len(source), index=source.index, dtype="Int64")
+        return counts.sum(axis=1).astype("Int64")
+
+    def _affected_count_components(self, source: pd.DataFrame) -> pd.DataFrame:
+        components = pd.DataFrame(index=source.index)
+        fatalities = pd.to_numeric(source["fatalities"], errors="coerce").fillna(0)
+        if "n_killed" in source.columns:
+            components["n_killed"] = pd.concat(
+                [pd.to_numeric(source["n_killed"], errors="coerce").fillna(0), fatalities],
+                axis=1,
+            ).max(axis=1)
+        else:
+            components["n_killed"] = fatalities
+        for column in (
+            "n_tortured",
+            "n_abducted",
+            "n_arrested",
+            "n_disappeared",
+        ):
+            if column in source.columns:
+                components[column] = pd.to_numeric(source[column], errors="coerce").fillna(0)
+        return components
+
+    # def _media_role(self, source: pd.DataFrame) -> pd.Series:
+    #     for column in ("assoc_actor_2", "assoc_actor_1", "actor2"):
+    #         if column in source.columns:
+    #             return source[column]
+    #     return pd.Series([pd.NA] * len(source), index=source.index, dtype="object")
+
+    def _numeric(self, values: pd.Series | None, index: pd.Index) -> pd.Series:
+        if values is None:
+            return pd.Series([pd.NA] * len(index), index=index, dtype="Float64")
+        return pd.to_numeric(
+            values.astype(str).str.replace(",", ".", regex=False), errors="coerce"
+        )
+
+    def _validate_coordinates(self, processed: pd.DataFrame) -> None:
+        latitude = processed["latitude"].dropna()
+        longitude = processed["longitude"].dropna()
+        if ((latitude < -90) | (latitude > 90)).any():
+            raise EventSchemaValidationError("ACLED Event latitude outside [-90, 90]")
+        if ((longitude < -180) | (longitude > 180)).any():
+            raise EventSchemaValidationError(
+                "ACLED Event longitude outside [-180, 180]"
+            )

fpu_barometer_admin/processors/base_processor.py

@@ -0,0 +1,23 @@
+"""Shared processor interfaces."""
+
+from __future__ import annotations
+
+from typing import Protocol
+
+import pandas as pd
+
+from fpu_barometer_admin.connectors.base_connector import SourceArtifactSet
+
+
+class BaseProcessor(Protocol):
+    """Generic processor interface."""
+
+    dataset: str
+
+    def process(
+        self,
+        artifact_set: SourceArtifactSet,
+        *args: object,
+        **kwargs: object,
+    ) -> pd.DataFrame:
+        """Transform a Source Artifact Set into a canonical DataFrame."""

fpu_barometer_admin/processors/cpj_processor.py

@@ -0,0 +1,147 @@
+"""CPJ (Committee to Protect Journalists) processor."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import pandas as pd
+
+from fpu_barometer_admin.connectors import SourceArtifactSet
+from fpu_barometer_admin.schemas.event import (
+    CountryCanonicalizer,
+    EventSchemaValidationError,
+    canonical_event_id,
+    incident_tags,
+    validate_event_dataframe,
+)
+from fpu_barometer_admin.storage.objects import ObjectStorage
+
+# Map CPJ status values to Barometer incident taxonomy.
+# Each entry maps status -> (top_type, leaf_type)
+_CPJ_STATUS_INCIDENT_MAP: dict[str, tuple[str, str]] = {
+    "Killed": ("Physical assault", "Death (physical assault resulting in death)"),
+    "Imprisoned": ("Legal incident", "Arrest/detention/imprisonment"),
+    "Missing": ("Legal incident", "Other"),
+    "Unknown": ("Unknown", "Unknown"),
+}
+
+# Source columns required by the processor (subset of what the connector validates).
+_PROCESSOR_REQUIRED_COLUMNS = {
+    "fullName",
+    "location",
+    "status",
+    "startDisplay",
+}
+
+# Override mappings for CPJ country names that the canonicalizer can't resolve.
+_CPJ_COUNTRY_OVERRIDES: dict[str, str] = {
+    "Yugoslavia": "Serbia",
+    "Serbia and Montenegro": "Serbia",
+    "Ethopia": "Ethiopia",
+}
+
+
+class CpjProcessor:
+    """Transform CPJ Source Artifacts into canonical Event data."""
+
+    dataset = "cpj"
+
+    def __init__(self, object_storage: ObjectStorage):
+        self.object_storage = object_storage
+        self.country_canonicalizer = CountryCanonicalizer()
+        self._cached_source: pd.DataFrame | None = None
+
+    def process(
+        self,
+        artifact_set: SourceArtifactSet,
+        *args: object,
+        **kwargs: object,
+    ) -> pd.DataFrame:
+        if artifact_set.dataset != self.dataset:
+            raise ValueError(
+                f"CPJ processor cannot process dataset {artifact_set.dataset!r}"
+            )
+
+        source = self.object_storage.read_dataframe(artifact_set.logical_path)
+        self._validate_source_columns(source)
+
+        processed = pd.DataFrame(index=source.index)
+        # Use row index as suffix to make event_id unique (mtpage can have duplicates)
+        processed["event_id"] = source.index.to_series().apply(
+            lambda idx: canonical_event_id(self.dataset, f"row-{idx}")
+        )
+        processed["dataset"] = self.dataset
+
+        # Map country names to ISO3
+        countries = source["location"].map(self._canonical_country)
+        unresolved = countries.isna()
+        if unresolved.any():
+            unresolved_values = (
+                source.loc[unresolved, "location"]
+                .dropna()
+                .astype(str)
+                .unique()[:5]
+                .tolist()
+            )
+            raise EventSchemaValidationError(
+                "CPJ Event rows have unresolved countries: "
+                f"{unresolved_values}"
+            )
+        processed["iso3"] = countries.map(lambda c: c.iso3)
+        processed["country_name"] = countries.map(lambda c: c.country_name)
+
+        # Parse dates from "Month DD, YYYY" format
+        dates = pd.to_datetime(source["startDisplay"], errors="coerce")
+        if dates.isna().any():
+            bad_dates = source.loc[dates.isna(), "startDisplay"].head().tolist()
+            raise EventSchemaValidationError(
+                f"CPJ Event rows have invalid date values: {bad_dates}"
+            )
+        processed["date"] = dates.dt.date
+        processed["date_precision"] = "day"
+        processed["year"] = dates.dt.year.astype("Int64")
+
+        # Map incident types based on CPJ status
+        incident_values = source["status"].map(self._incident_values)
+        processed["type_of_incident"] = incident_values.map(lambda v: v[0])
+        processed["top_type_of_incident"] = incident_values.map(lambda v: v[1])
+        processed["type_of_incident_leaves"] = incident_values.map(lambda v: v[2])
+
+        processed["processed_at"] = pd.Timestamp.now(tz="UTC")
+        processed["n_people_affected"] = 1  # Each CPJ entry is one person
+
+        processed["region"] = pd.NA
+        processed["latitude"] = pd.NA
+        processed["longitude"] = pd.NA
+        processed["description"] = source.get("organizations", pd.NA)
+        processed["source_url"] = source["mtpage"]
+        processed["attacked_count"] = pd.NA
+        processed["gender"] = pd.NA
+        processed["media_role"] = source.get("type", pd.NA)
+        processed["perpetrator_type"] = source.get("sourcesOfFire", pd.NA)
+
+        return validate_event_dataframe(processed)
+
+    def _validate_source_columns(self, source: pd.DataFrame) -> None:
+        missing = _PROCESSOR_REQUIRED_COLUMNS - set(source.columns)
+        if missing:
+            raise EventSchemaValidationError(
+                f"CPJ Source Artifact missing processor columns: {sorted(missing)}"
+            )
+
+    def _canonical_country(self, country_name: object) -> Any | None:
+        if country_name is None or pd.isna(country_name):
+            return None
+        name = str(country_name).strip()
+        # Apply CPJ-specific overrides before canonical lookup
+        name = _CPJ_COUNTRY_OVERRIDES.get(name, name)
+        return self.country_canonicalizer.from_values(country_name=name)
+
+    def _incident_values(
+        self, status: object
+    ) -> tuple[list[str], list[str], list[str]]:
+        raw = str(status).strip() if status and not pd.isna(status) else ""
+        mapped = _CPJ_STATUS_INCIDENT_MAP.get(raw)
+        if mapped:
+            return incident_tags(top=[mapped[0]], leaves=[mapped[1]])
+        return incident_tags(top=["Unknown"], leaves=["Unknown"])

fpu_barometer_admin/processors/ert_processor.py

@@ -0,0 +1,72 @@
+"""ERT processor — transforms CSV source artifacts into canonical Predictor data."""
+
+from __future__ import annotations
+
+import logging
+
+import pandas as pd
+
+from fpu_barometer_admin.connectors.base_connector import SourceArtifactSet
+from fpu_barometer_admin.schemas.predictor import (
+    PREDICTOR_CORE_COLUMNS,
+    canonicalize_predictor_dataframe,
+    validate_predictor_dataframe,
+)
+from fpu_barometer_admin.storage.objects import ObjectStorage
+
+
+logger = logging.getLogger(__name__)
+
+
+class ErtProcessor:
+    """Transform ERT Source Artifacts into canonical Predictor data.
+
+    The processor drops the source ``Unnamed: 0`` column from processed output
+    and passes ``strict=False`` to canonicalization so that unresolved historical
+    country rows and duplicate-collapse rows produce warnings rather than errors.
+    """
+
+    dataset = "ert"
+
+    def __init__(self, object_storage: ObjectStorage):
+        self.object_storage = object_storage
+
+    def process(
+        self,
+        artifact_set: SourceArtifactSet,
+        *args: object,
+        **kwargs: object,
+    ) -> pd.DataFrame:
+        if artifact_set.dataset != self.dataset:
+            raise ValueError(
+                f"ERT processor cannot process dataset {artifact_set.dataset!r}"
+            )
+        source = self.object_storage.read_dataframe(artifact_set.logical_path)
+
+        canonical, report = canonicalize_predictor_dataframe(
+            source,
+            dataset=self.dataset,
+            iso3_column="country_text_id",
+            country_name_column="country_name",
+            year_column="year",
+            strict=False,
+        )
+
+        if "Unnamed: 0" in canonical.columns:
+            canonical = canonical.drop(columns=["Unnamed: 0"])
+
+        if report.unresolved_country_rows:
+            logger.warning(
+                "ERT canonicalization dropped %d unresolved country rows: %s",
+                report.unresolved_country_rows,
+                report.unresolved_source_values,
+            )
+        if report.duplicate_country_year_rows_dropped:
+            logger.warning(
+                "ERT canonicalization dropped %d duplicate country-year rows: %s",
+                report.duplicate_country_year_rows_dropped,
+                report.duplicate_source_values_dropped,
+            )
+
+        result = validate_predictor_dataframe(canonical)
+        return result