fpu-barometer-admin 0.3.0__py3-none-any.whl

fpu_barometer_admin/processors/gdelt_processor.py

@@ -0,0 +1,260 @@
+"""GDELT processor."""
+
+from __future__ import annotations
+
+import warnings
+
+import pandas as pd
+
+from fpu_barometer_admin.connectors import SourceArtifactSet
+from fpu_barometer_admin.schemas.event import (
+    EVENT_COLUMNS,
+    CountryCanonicalizer,
+    EventSchemaValidationError,
+    canonical_event_id,
+    incident_tags,
+    validate_event_dataframe,
+)
+from fpu_barometer_admin.storage.objects import ObjectStorage
+
+
+_GDELT_INCIDENT_MAP: dict[str, tuple[str, str]] = {
+    "091": ("Legal incident", "Investigation"),
+    "092": ("Legal incident", "Investigation"),
+    "110": ("Verbal attack", "Insult / harassment / discredit"),
+    "111": ("Verbal attack", "Insult / harassment / discredit"),
+    "112": ("Verbal attack", "Insult / harassment / discredit"),
+    "1121": ("Verbal attack", "Insult / harassment / discredit"),
+    "1122": ("Verbal attack", "Insult / harassment / discredit"),
+    "1123": ("Verbal attack", "Insult / harassment / discredit"),
+    "1125": ("Verbal attack", "Insult / harassment / discredit"),
+    "113": ("Verbal attack", "Insult / harassment / discredit"),
+    "114": ("Legal incident", "Complaint"),
+    "115": ("Legal incident", "Civil legal actions / lawsuits"),
+    "116": ("Legal incident", "Conviction"),
+    "1233": ("Interference", "Blocked journalistic activity"),
+    "124": ("Interference", "Blocked journalistic activity"),
+    "1234": ("Interference", "Blocked journalistic activity"),
+    "1245": ("Interference", "Blocked journalistic activity"),
+    "130": ("Verbal attack", "Threatening"),
+    "131": ("Verbal attack", "Threatening"),
+    "1321": ("Verbal attack", "Threatening"),
+    "138": ("Verbal attack", "Threatening"),
+    "1384": ("Verbal attack", "Threatening"),
+    "139": ("Verbal attack", "Threatening"),
+    "170": ("Interference", "Other"),
+    "171": ("Attack to property", "Property (incl. houses, cars, personal belongings)"),
+    "1711": ("Attack to property", "Property (incl. houses, cars, personal belongings)"),
+    "1712": ("Attack to property", "Property (incl. houses, cars, personal belongings)"),
+    "172": ("Interference", "Administrative or financial interference"),
+    "1721": ("Interference", "Administrative or financial interference"),
+    "173": ("Legal incident", "Arrest/detention/imprisonment"),
+    "174": ("Legal incident", "Travel ban"),
+    "176": ("Attack to property", "Hacking/DDoS"),
+    "180": ("Physical assault", "Other"),
+    "181": ("Physical assault", "Abduction/kidnapping"),
+    "182": ("Physical assault", "Without injury (physical assault not resulting in injury)"),
+    "1821": ("Physical assault", "Sexual assault"),
+    "1822": ("Physical assault", "Injury (physical assault resulting in injury)"),
+    "1823": ("Physical assault", "Death (physical assault resulting in death)"),
+    "183": ("Physical assault", "Injury (physical assault resulting in injury)"),
+    "1831": ("Physical assault", "Injury (physical assault resulting in injury)"),
+    "1832": ("Physical assault", "Injury (physical assault resulting in injury)"),
+    "1833": ("Physical assault", "Injury (physical assault resulting in injury)"),
+    "1834": ("Physical assault", "Injury (physical assault resulting in injury)"),
+    "185": ("Physical assault", "Injury (physical assault resulting in injury)"),
+    "186": ("Physical assault", "Death (physical assault resulting in death)"),
+}
+
+
+class GdeltProcessor:
+    """Transform GDELT Source Artifacts into canonical Event data."""
+
+    dataset = "gdelt"
+    required_columns = {
+        "GlobalEventID",
+        "SQLDATE",
+        "Actor1Name",
+        "Actor1CountryCode",
+        "Actor2Name",
+        "Actor2CountryCode",
+        "EventCode",
+        "ActionGeo_CountryCode",
+        "ActionGeo_FullName",
+        "ActionGeo_Lat",
+        "ActionGeo_Long",
+        "SOURCEURL",
+    }
+
+    def __init__(self, object_storage: ObjectStorage):
+        self.object_storage = object_storage
+        self.country_canonicalizer = CountryCanonicalizer()
+
+    def process(
+        self,
+        artifact_set: SourceArtifactSet,
+        *args: object,
+        baseline_processed: pd.DataFrame | None = None,
+        **kwargs: object,
+    ) -> pd.DataFrame:
+        if artifact_set.dataset != self.dataset:
+            raise ValueError(
+                f"GDELT processor cannot process dataset {artifact_set.dataset!r}"
+            )
+        source = self.object_storage.read_dataframe(artifact_set.logical_path)
+        self._validate_source_columns(source)
+
+        countries = source.apply(self._country, axis=1)
+        valid_country = countries.notna()
+        if (~valid_country).any():
+            samples = source.loc[~valid_country, ["ActionGeo_CountryCode", "Actor1CountryCode", "Actor2CountryCode"]].head().to_dict("records")
+            warnings.warn(
+                f"GDELT dropped {(~valid_country).sum()} Event rows with unresolved countries: {samples}",
+                RuntimeWarning,
+                stacklevel=2,
+            )
+
+        dates = source["SQLDATE"].map(self._date)
+        valid_date = dates.map(lambda value: pd.notna(value))
+        if (~valid_date).any():
+            samples = source.loc[~valid_date, "SQLDATE"].head().tolist()
+            warnings.warn(
+                f"GDELT dropped {(~valid_date).sum()} Event rows with invalid dates: {samples}",
+                RuntimeWarning,
+                stacklevel=2,
+            )
+
+        coordinates = source.apply(self._coordinates, axis=1)
+        valid_coordinates = coordinates.map(lambda value: value is not None)
+        if (~valid_coordinates).any():
+            samples = source.loc[~valid_coordinates, ["ActionGeo_Lat", "ActionGeo_Long"]].head().to_dict("records")
+            warnings.warn(
+                f"GDELT dropped {(~valid_coordinates).sum()} Event rows with invalid coordinates: {samples}",
+                RuntimeWarning,
+                stacklevel=2,
+            )
+
+        keep = valid_country & valid_date & valid_coordinates
+        source = source.loc[keep].copy()
+        countries = countries.loc[keep]
+        dates = dates.loc[keep]
+        coordinates = coordinates.loc[keep]
+        if source.empty:
+            return validate_event_dataframe(pd.DataFrame(columns=EVENT_COLUMNS))
+
+        processed = pd.DataFrame(index=source.index)
+        processed["event_id"] = source["GlobalEventID"].map(
+            lambda value: canonical_event_id(self.dataset, value)
+        )
+        processed = processed.loc[~processed["event_id"].duplicated(keep="last")].copy()
+        source = source.loc[processed.index]
+        countries = countries.loc[processed.index]
+        dates = dates.loc[processed.index]
+        coordinates = coordinates.loc[processed.index]
+
+        processed["dataset"] = self.dataset
+        processed["iso3"] = countries.map(lambda country: country.iso3)
+        processed["country_name"] = countries.map(lambda country: country.country_name)
+        processed["date"] = dates.map(lambda value: value.date())
+        processed["date_precision"] = "day"
+        processed["year"] = dates.map(lambda value: int(value.year)).astype("Int64")
+        tags = source["EventCode"].map(self._incident_tags)
+        processed["type_of_incident"] = tags.map(lambda value: value[0])
+        processed["top_type_of_incident"] = tags.map(lambda value: value[1])
+        processed["type_of_incident_leaves"] = tags.map(lambda value: value[2])
+        processed["processed_at"] = pd.Timestamp.now(tz="UTC")
+        processed["n_people_affected"] = 1
+        processed["region"] = source["ActionGeo_FullName"].map(self._blank_to_na)
+        processed["latitude"] = coordinates.map(lambda value: value[0])
+        processed["longitude"] = coordinates.map(lambda value: value[1])
+        processed["description"] = source.apply(self._description, axis=1)
+        processed["source_url"] = source["SOURCEURL"].map(self._source_url)
+        processed["attacked_count"] = pd.NA
+        processed["gender"] = pd.NA
+        processed["media_role"] = pd.NA
+        processed["perpetrator_type"] = pd.NA
+
+        processed = processed.loc[:, EVENT_COLUMNS].reset_index(drop=True)
+        if baseline_processed is not None and not baseline_processed.empty:
+            processed = pd.concat(
+                [baseline_processed.loc[:, EVENT_COLUMNS], processed],
+                ignore_index=True,
+            )
+            processed = processed.loc[~processed["event_id"].duplicated(keep="last")]
+            processed = processed.loc[:, EVENT_COLUMNS].reset_index(drop=True)
+        return validate_event_dataframe(processed)
+
+    def _validate_source_columns(self, source: pd.DataFrame) -> None:
+        missing = self.required_columns - set(source.columns)
+        if missing:
+            raise EventSchemaValidationError(
+                f"GDELT Source Artifact missing processor columns: {sorted(missing)}"
+            )
+
+    def _country(self, row: pd.Series):
+        # GDELT ActionGeo country codes are FIPS-ish, so prefer actor ISO3-like values
+        # when available; fall back to location name.
+        for value in (row.get("Actor2CountryCode"), row.get("Actor1CountryCode")):
+            if not self._is_blank(value):
+                country = self.country_canonicalizer.from_values(iso3=value)
+                if country is not None:
+                    return country
+        if not self._is_blank(row.get("ActionGeo_FullName")):
+            return self.country_canonicalizer.from_values(
+                country_name=row.get("ActionGeo_FullName")
+            )
+        return None
+
+    def _date(self, value: object) -> pd.Timestamp | None:
+        text = str(value).strip()
+        if not re_match_yyyymmdd(text):
+            return None
+        parsed = pd.to_datetime(text, format="%Y%m%d", errors="coerce")
+        return parsed if pd.notna(parsed) else None
+
+    def _coordinates(self, row: pd.Series) -> tuple[float | object, float | object] | None:
+        latitude = pd.to_numeric(pd.Series([row.get("ActionGeo_Lat")]), errors="coerce").iloc[0]
+        longitude = pd.to_numeric(pd.Series([row.get("ActionGeo_Long")]), errors="coerce").iloc[0]
+        if pd.isna(latitude) and pd.isna(longitude):
+            return (pd.NA, pd.NA)
+        if pd.isna(latitude) or pd.isna(longitude):
+            return None
+        if latitude < -90 or latitude > 90 or longitude < -180 or longitude > 180:
+            return None
+        return (float(latitude), float(longitude))
+
+    def _incident_tags(self, event_code: object) -> tuple[list[str], list[str], list[str]]:
+        top, leaf = _GDELT_INCIDENT_MAP.get(str(event_code), ("Unknown", "Unknown"))
+        return incident_tags(top=[top], leaves=[leaf])
+
+    def _description(self, row: pd.Series) -> object:
+        event_code = str(row.get("EventCode", "")).strip()
+        actor1 = str(row.get("Actor1Name", "")).strip()
+        actor2 = str(row.get("Actor2Name", "")).strip()
+        parts = [part for part in (actor1, actor2, f"GDELT EventCode {event_code}" if event_code else "") if part]
+        return " | ".join(parts) if parts else pd.NA
+
+    def _source_url(self, value: object) -> object:
+        if self._is_blank(value):
+            return pd.NA
+        return str(value).strip()
+
+    def _blank_to_na(self, value: object) -> object:
+        return pd.NA if self._is_blank(value) else str(value).strip()
+
+    def _is_blank(self, value: object) -> bool:
+        if value is None:
+            return True
+        if isinstance(value, str):
+            return not value.strip() or value.strip().lower() in {"nan", "none", "null"}
+        try:
+            return bool(pd.isna(value))
+        except (TypeError, ValueError):
+            return False
+
+
+def re_match_yyyymmdd(value: str) -> bool:
+    return len(value) == 8 and value.isdigit()
+
+
+__all__ = ["GdeltProcessor"]

fpu_barometer_admin/processors/mfrr_processor.py

@@ -0,0 +1,327 @@
+"""MFRR processor."""
+
+from __future__ import annotations
+
+import ast
+import html
+import json
+import re
+import warnings
+from collections.abc import Iterable
+
+import pandas as pd
+
+from fpu_barometer_admin.connectors import SourceArtifactSet
+from fpu_barometer_admin.schemas.event import (
+    EVENT_COLUMNS,
+    INCIDENT_TAXONOMY,
+    LEAF_INCIDENT_TYPES,
+    TOP_INCIDENT_TYPES,
+    CountryCanonicalizer,
+    EventSchemaValidationError,
+    canonical_event_id,
+    incident_tags,
+    validate_event_dataframe,
+)
+from fpu_barometer_admin.storage.objects import ObjectStorage
+
+
+_LEAF_TO_TOP = {
+    leaf: top for top, leaves in INCIDENT_TAXONOMY.items() for leaf in leaves
+}
+
+
+class MFRRProcessor:
+    """Transform MFRR Source Artifacts into canonical Event data."""
+
+    dataset = "mfrr"
+    required_columns = {
+        "id",
+        "country",
+        "date",
+        "year",
+        "published_at",
+        "published_at_date",
+        "type_of_incident",
+        "top_type_of_incident",
+        "type_of_incident_leaves",
+        "attacked_count",
+    }
+
+    def __init__(self, object_storage: ObjectStorage):
+        self.object_storage = object_storage
+        self.country_canonicalizer = CountryCanonicalizer()
+
+    def process(
+        self,
+        artifact_set: SourceArtifactSet,
+        *args: object,
+        **kwargs: object,
+    ) -> pd.DataFrame:
+        if artifact_set.dataset != self.dataset:
+            raise ValueError(
+                f"MFRR processor cannot process dataset {artifact_set.dataset!r}"
+            )
+        source = self.object_storage.read_dataframe(artifact_set.logical_path)
+        self._validate_source_columns(source)
+
+        countries = source["country"].map(
+            lambda value: self.country_canonicalizer.from_values(country_name=value)
+        )
+        valid_country = countries.notna()
+        if (~valid_country).any():
+            samples = source.loc[~valid_country, "country"].dropna().astype(str).unique()[:5]
+            warnings.warn(
+                f"MFRR dropped {(~valid_country).sum()} Event rows with unresolved countries: {list(samples)}",
+                RuntimeWarning,
+                stacklevel=2,
+            )
+
+        dates = source.apply(self._canonical_date, axis=1)
+        valid_date = dates.map(lambda value: value is not None)
+        if (~valid_date).any():
+            samples = source.loc[~valid_date, ["date", "year", "published_at_date", "published_at"]].head().to_dict("records")
+            warnings.warn(
+                f"MFRR dropped {(~valid_date).sum()} Event rows with invalid dates: {samples}",
+                RuntimeWarning,
+                stacklevel=2,
+            )
+
+        keep = valid_country & valid_date
+        source = source.loc[keep].copy()
+        countries = countries.loc[keep]
+        dates = dates.loc[keep]
+
+        processed = pd.DataFrame(index=source.index)
+        processed["event_id"] = source["id"].map(
+            lambda value: canonical_event_id(self.dataset, value)
+        )
+        processed["dataset"] = self.dataset
+        processed["iso3"] = countries.map(lambda country: country.iso3)
+        processed["country_name"] = countries.map(lambda country: country.country_name)
+        processed["date"] = dates.map(lambda value: value[0])
+        processed["date_precision"] = dates.map(lambda value: value[1])
+        processed["year"] = dates.map(lambda value: value[2]).astype("Int64")
+
+        incident_values = source.apply(self._incident_values, axis=1)
+        processed["type_of_incident"] = incident_values.map(lambda values: values[0])
+        processed["top_type_of_incident"] = incident_values.map(lambda values: values[1])
+        processed["type_of_incident_leaves"] = incident_values.map(lambda values: values[2])
+
+        processed["processed_at"] = pd.Timestamp.now(tz="UTC")
+        processed["n_people_affected"] = self._people_affected(source)
+        processed["region"] = source.apply(self._region, axis=1)
+        processed["latitude"] = source.apply(lambda row: self._coordinate(row, "lat"), axis=1)
+        processed["longitude"] = source.apply(lambda row: self._coordinate(row, "lng"), axis=1)
+        processed["description"] = source.apply(self._description, axis=1)
+        processed["source_url"] = source.apply(self._source_url, axis=1)
+        processed["attacked_count"] = self._attacked_count(source)
+        processed["gender"] = source["gender"].map(self._semicolon_join) if "gender" in source else pd.NA
+        processed["media_role"] = source["type_of_journalist_or_media_actor"].map(self._semicolon_join) if "type_of_journalist_or_media_actor" in source else pd.NA
+        processed["perpetrator_type"] = source["source_of_incident"].map(self._semicolon_join) if "source_of_incident" in source else pd.NA
+
+        processed = processed.loc[:, EVENT_COLUMNS]
+        self._validate_coordinates(processed)
+        return validate_event_dataframe(processed)
+
+    def _validate_source_columns(self, source: pd.DataFrame) -> None:
+        missing = self.required_columns - set(source.columns)
+        if missing:
+            raise EventSchemaValidationError(
+                f"MFRR Source Artifact missing processor columns: {sorted(missing)}"
+            )
+
+    def _canonical_date(self, row: pd.Series) -> tuple[object, str, int] | None:
+        if self._truthy(row.get("has_date_of_incidence")):
+            incident_date = pd.to_datetime(row.get("date"), errors="coerce")
+            if pd.notna(incident_date):
+                return incident_date.date(), "day", int(incident_date.year)
+
+        year = pd.to_numeric(pd.Series([row.get("year")]), errors="coerce").iloc[0]
+        if pd.notna(year):
+            year_int = int(year)
+            if 1000 <= year_int <= 9999:
+                return pd.Timestamp(year=year_int, month=1, day=1).date(), "year", year_int
+
+        for column in ("published_at_date", "published_at"):
+            value = row.get(column)
+            if column == "published_at" and not self._is_blank(value):
+                numeric = pd.to_numeric(pd.Series([value]), errors="coerce").iloc[0]
+                if pd.notna(numeric):
+                    published = pd.to_datetime(numeric, unit="s", errors="coerce")
+                else:
+                    published = pd.to_datetime(value, errors="coerce")
+            else:
+                published = pd.to_datetime(value, errors="coerce")
+            if pd.notna(published):
+                return published.date(), "publication_day", int(published.year)
+        return None
+
+    def _incident_values(self, row: pd.Series) -> tuple[list[str], list[str], list[str]]:
+        source_top = [value for value in self._list_values(row.get("top_type_of_incident")) if value in TOP_INCIDENT_TYPES]
+        source_leaves = [value for value in self._list_values(row.get("type_of_incident_leaves")) if value in LEAF_INCIDENT_TYPES]
+        for value in self._list_values(row.get("type_of_incident")):
+            if value in TOP_INCIDENT_TYPES:
+                source_top.append(value)
+            if value in LEAF_INCIDENT_TYPES:
+                source_leaves.append(value)
+
+        inferred_top = [_LEAF_TO_TOP[leaf] for leaf in source_leaves if leaf in _LEAF_TO_TOP]
+        top = self._dedupe([*source_top, *inferred_top])
+        leaves = self._dedupe(source_leaves)
+        if top and not leaves:
+            leaves = [self._fallback_leaf_for_top(value) for value in top]
+        if not top or not leaves:
+            top = ["Unknown"]
+            leaves = ["Unknown"]
+        return incident_tags(top=top, leaves=leaves)
+
+    def _fallback_leaf_for_top(self, top: str) -> str:
+        allowed = INCIDENT_TAXONOMY.get(top, ())
+        if "Unknown" in allowed:
+            return "Unknown"
+        if top in allowed:
+            return top
+        return allowed[0] if allowed else "Unknown"
+
+    def _people_affected(self, source: pd.DataFrame) -> pd.Series:
+        counts = pd.to_numeric(source["attacked_count"], errors="coerce")
+        return counts.fillna(1).clip(lower=1).astype("Int64")
+
+    def _attacked_count(self, source: pd.DataFrame) -> pd.Series:
+        return pd.to_numeric(source["attacked_count"], errors="coerce").astype("Int64")
+
+    def _description(self, row: pd.Series) -> object:
+        title = self._clean_text(row.get("title"))
+        content = self._clean_text(row.get("content"))
+        parts = [part for part in (title, content) if part]
+        return "\n\n".join(parts) if parts else pd.NA
+
+    def _source_url(self, row: pd.Series) -> object:
+        for value in [row.get("coe_link"), *self._url_values(row.get("news_source_links")), *self._url_values(row.get("internet_source_links"))]:
+            url = self._clean_text(value)
+            if not url:
+                continue
+            if not re.match(r"^[a-z][a-z0-9+.-]*://", url, flags=re.IGNORECASE):
+                url = f"https://{url}"
+            return url
+        return pd.NA
+
+    def _region(self, row: pd.Series) -> object:
+        region_names = [self._clean_text(value) for value in self._list_values(row.get("region_names"))]
+        region_names = [value for value in region_names if value and not self._is_unknownish(value)]
+        return region_names[2] if len(region_names) >= 3 else pd.NA
+
+    def _coordinate(self, row: pd.Series, axis: str) -> object:
+        flat_column = "_geo_lat" if axis == "lat" else "_geo_lng"
+        dotted_column = "_geo.lat" if axis == "lat" else "_geo.lng"
+        value = row.get(flat_column)
+        if self._is_blank(value):
+            value = row.get(dotted_column)
+        if self._is_blank(value):
+            geo = self._maybe_mapping(row.get("_geo"))
+            if isinstance(geo, dict):
+                value = geo.get("lat" if axis == "lat" else "lng")
+        numeric = pd.to_numeric(pd.Series([value]), errors="coerce").iloc[0]
+        return numeric if pd.notna(numeric) else pd.NA
+
+    def _validate_coordinates(self, processed: pd.DataFrame) -> None:
+        latitude = processed["latitude"].dropna()
+        longitude = processed["longitude"].dropna()
+        if ((latitude < -90) | (latitude > 90)).any():
+            raise EventSchemaValidationError("MFRR Event latitude outside [-90, 90]")
+        if ((longitude < -180) | (longitude > 180)).any():
+            raise EventSchemaValidationError("MFRR Event longitude outside [-180, 180]")
+
+    def _semicolon_join(self, value: object) -> object:
+        values = [self._clean_text(item) for item in self._list_values(value)]
+        values = [item for item in values if item and not self._is_unknownish(item)]
+        return "; ".join(self._dedupe(values)) if values else pd.NA
+
+    def _url_values(self, value: object) -> list[object]:
+        values: list[object] = []
+        for item in self._list_values(value):
+            if isinstance(item, str):
+                values.extend(part for part in re.split(r"[\r\n]+", item) if part.strip())
+            else:
+                values.append(item)
+        return values
+
+    def _list_values(self, value: object) -> list[object]:
+        if self._is_blank(value):
+            return []
+        if isinstance(value, (list, tuple, set)):
+            return list(value)
+        if isinstance(value, str):
+            text = value.strip()
+            if not text:
+                return []
+            if text[:1] in "[({":
+                try:
+                    parsed = ast.literal_eval(text)
+                except (SyntaxError, ValueError):
+                    try:
+                        parsed = json.loads(text)
+                    except json.JSONDecodeError:
+                        return [text]
+                if isinstance(parsed, (list, tuple, set)):
+                    return list(parsed)
+            return [text]
+        if isinstance(value, Iterable) and not isinstance(value, (str, bytes, dict)):
+            return list(value)
+        return [value]
+
+    def _maybe_mapping(self, value: object) -> object:
+        if isinstance(value, dict) or self._is_blank(value):
+            return value
+        if isinstance(value, str) and value.strip()[:1] in "{":
+            for loader in (json.loads, ast.literal_eval):
+                try:
+                    parsed = loader(value)
+                except (json.JSONDecodeError, SyntaxError, ValueError):
+                    continue
+                if isinstance(parsed, dict):
+                    return parsed
+        return value
+
+    def _clean_text(self, value: object) -> str | None:
+        if self._is_blank(value):
+            return None
+        text = str(value)
+        text = re.sub(r"<[^>]+>", " ", text)
+        text = html.unescape(text)
+        text = re.sub(r"\s+", " ", text).strip()
+        text = re.sub(r"\s+([.,;:!?])", r"\1", text)
+        return text or None
+
+    def _is_unknownish(self, value: str) -> bool:
+        text = value.strip().lower()
+        return (
+            text in {"unknown", "nan", "none", "null", "not applicable", "n/a", "na"}
+            or text.startswith("unknown ")
+            or text.endswith(" unknown")
+        )
+
+    def _truthy(self, value: object) -> bool:
+        if self._is_blank(value):
+            return False
+        if isinstance(value, str):
+            return value.strip().lower() in {"1", "true", "yes", "y"}
+        return bool(value)
+
+    def _is_blank(self, value: object) -> bool:
+        if value is None:
+            return True
+        if isinstance(value, str):
+            return not value.strip() or value.strip().lower() in {"nan", "none", "null"}
+        try:
+            return bool(pd.isna(value))
+        except (TypeError, ValueError):
+            return False
+
+    def _dedupe(self, values: Iterable) -> list:
+        result = []
+        for value in values:
+            if value not in result:
+                result.append(value)
+        return result