ccrt 0.1.0__py3-none-any.whl

ccrt/interactive.py

@@ -0,0 +1,160 @@
+"""Small terminal workflow for reviewing candidate links with their source accounts."""
+
+from collections import defaultdict
+
+
+def _short(value, limit=110):
+    value = " ".join(str(value or "").split())
+    return value if len(value) <= limit else value[: limit - 1] + "…"
+
+
+def _date(record):
+    if record.date_start is None:
+        return "unknown date"
+    start, end = record.date_start.isoformat(), record.date_end.isoformat()
+    return start if start == end else f"{start} to {end}"
+
+
+def _place(record):
+    raw = record.raw
+    return _short(raw.get("where_description") or raw.get("location") or raw.get("where_coordinates"))
+
+
+def _coordinates(record):
+    if record.latitude is None or record.longitude is None:
+        return "not supplied"
+    return f"lat {record.latitude}, lon {record.longitude}"
+
+
+def _account(record, emit):
+    raw = record.raw
+    emit(f"{record.source.upper()} {record.source_id} | {_date(record)} | {record.country}")
+    place = raw.get("where_description") or raw.get("location") or raw.get("where_coordinates")
+    emit(f"Place: {place or 'not supplied'}")
+    emit(f"Coordinates: {_coordinates(record)}")
+    emit(f"Actors: {record.actor1 or '?'} / {record.actor2 or '?'}")
+    deaths = f"best {record.total_deaths_best}"
+    if record.total_deaths_low is not None or record.total_deaths_high is not None:
+        deaths += f" (low {record.total_deaths_low}, high {record.total_deaths_high})"
+    emit(f"Event type: {record.event_type or '?'} | reported deaths: {deaths}")
+    emit(f"Date precision: {record.date_precision} | location precision: {record.geo_precision}")
+    for label, key in (
+        ("Notes", "notes"),
+        ("Source articles", "source_article"),
+        ("Original sources", "source_original"),
+        ("Source", "source"),
+    ):
+        if raw.get(key):
+            emit(f"{label}: {raw[key]}")
+
+
+def review_interactive(
+    result, review, *, reviewer, save, left_id=None, show_all=False, ask=input, emit=print
+):
+    """Review UCDP/other left-side groups; call save after every decision.
+
+    ``ask`` and ``emit`` are injectable so the terminal workflow can be tested.
+    The caller owns loading the run, validating the review, and persisting it.
+    """
+    review.validate(result)
+    if not reviewer.strip():
+        raise ValueError("--reviewer needs a nonempty name")
+    grouped = defaultdict(list)
+    for candidate in result.candidates:
+        grouped[candidate.left.source_id].append(candidate)
+    if left_id is not None and left_id not in grouped:
+        raise ValueError(f"no candidate group for left source ID {left_id}")
+    decided = {decision.candidate_id: decision for decision in review.decisions}
+    ids = sorted(
+        (
+            key
+            for key, items in grouped.items()
+            if key == left_id
+            or show_all
+            or any(decided[c.candidate_id].status == "unresolved" for c in items)
+        ),
+        key=lambda key: (-grouped[key][0].score, key),
+    )
+    if not ids:
+        emit("No candidate groups to review. Use --all to revisit decisions.")
+        return review
+
+    emit("Scores rank nearby records; coordinates may represent shared place labels.")
+    emit("Compare both source accounts before deciding.")
+    position = ids.index(left_id) if left_id is not None else 0
+    while True:
+        items = grouped[ids[position]]
+        left = items[0].left
+        emit(f"\nGroup {position + 1}/{len(ids)} | {result.left.source.upper()} {left.source_id}")
+        _account(left, emit)
+        for number, candidate in enumerate(items, 1):
+            right = candidate.right
+            status = decided[candidate.candidate_id].status
+            emit(
+                f"  {number}. [{status}] {right.source_id} | score {candidate.score:.3f}"
+                f" | {candidate.days_apart} days | {candidate.distance_km:.1f} km"
+                f" | deaths {right.total_deaths_best} | {_date(right)} | {_place(right)}"
+                f" | {_coordinates(right)}"
+            )
+        try:
+            choice = ask("Number to inspect; n next, p previous, q quit: ").strip().lower()
+        except (EOFError, KeyboardInterrupt):
+            emit("\nReview saved. Re-run the command to continue.")
+            return review
+        if choice == "q":
+            break
+        if choice == "n":
+            if position == len(ids) - 1:
+                emit("End of review queue; returning to the first group.")
+            position = (position + 1) % len(ids)
+            continue
+        if choice == "p":
+            if position == 0:
+                emit("Start of review queue; returning to the last group.")
+            position = (position - 1) % len(ids)
+            continue
+        if not choice.isdecimal() or not 1 <= int(choice) <= len(items):
+            emit("Choose a listed number, n, p, or q.")
+            continue
+        candidate = items[int(choice) - 1]
+        decision = decided[candidate.candidate_id]
+        emit("\nLeft account:")
+        _account(candidate.left, emit)
+        emit("\nCandidate account:")
+        _account(candidate.right, emit)
+        emit(f"Coordinate separation: {candidate.distance_km:.3f} km (limit {result.config.max_km:g} km)")
+        emit(
+            f"Date interval gap: {candidate.days_apart} days"
+            f" (limit {result.config.max_days} days) | proximity score: {candidate.score:.3f}"
+        )
+        emit("Location precision codes are source specific, not kilometer uncertainty radii.")
+        emit(f"Current: {decision.status}" + (f" | note: {decision.note}" if decision.note else ""))
+        emit(f"Warnings: {', '.join(candidate.warnings) or 'none'}")
+        try:
+            action = ask("a accept, r reject, u unresolved, b back, q quit: ").strip().lower()
+        except (EOFError, KeyboardInterrupt):
+            emit("\nReview saved. Re-run the command to continue.")
+            return review
+        if action == "q":
+            break
+        if action == "b":
+            continue
+        statuses = {"a": "accepted", "r": "rejected", "u": "unresolved"}
+        if action not in statuses:
+            emit("Choose a, r, u, b, or q.")
+            continue
+        try:
+            note = ask("Reason (optional; Enter keeps existing note): ").strip()
+        except (EOFError, KeyboardInterrupt):
+            emit("\nNo change saved for this candidate.")
+            return review
+        review = review.decide(
+            candidate.candidate_id, statuses[action], reviewer=reviewer, note=note or decision.note
+        )
+        save(review)
+        decided[candidate.candidate_id] = next(
+            d for d in review.decisions if d.candidate_id == candidate.candidate_id
+        )
+        emit(f"Saved {statuses[action]} for {candidate.left.source_id} / {candidate.right.source_id}.")
+    emit("Review saved. Re-run the command to continue.")
+    return review

ccrt/loaders/__init__.py

@@ -0,0 +1,5 @@
+from ccrt.loaders.acled import load_acled_csv
+from ccrt.loaders.base import ValidationError, load_csv
+from ccrt.loaders.ucdp import load_ucdp_csv
+
+__all__ = ["ValidationError", "load_csv", "load_ucdp_csv", "load_acled_csv"]

ccrt/loaders/acled.py

@@ -0,0 +1,69 @@
+"""ACLED event CSV mapping (current lowercase columns, codebook 2024 onward)."""
+
+import calendar
+from datetime import timedelta
+from pathlib import Path
+
+from ccrt.loaders.base import integer, normalize, number, parse_date, read_csv_rows, text
+from ccrt.schema import SourceRecord, canonical_json
+
+REQUIRED = {
+    "event_id_cnty",
+    "event_date",
+    "time_precision",
+    "latitude",
+    "longitude",
+    "geo_precision",
+    "country",
+    "fatalities",
+}
+
+
+def map_record(row, version):
+    day = parse_date(row["event_date"])
+    precision = integer(row["time_precision"])
+    geo = integer(row["geo_precision"])
+    if precision not in {None, 1, 2, 3} or geo not in {None, 1, 2, 3}:
+        raise ValueError("ACLED precision codes must be 1, 2, 3, or blank")
+    start = end = day
+    basis = "reported"
+    if day and precision == 2:
+        start, end = day - timedelta(days=3), day + timedelta(days=3)
+        basis = "inferred-week-envelope"
+    elif day and precision == 3:
+        start = day.replace(day=1)
+        end = day.replace(day=calendar.monthrange(day.year, day.month)[1])
+        basis = "inferred-month-envelope"
+    return SourceRecord(
+        source="acled",
+        source_id=text(row["event_id_cnty"]),
+        version=version,
+        date_start=start,
+        date_end=end,
+        latitude=number(row["latitude"]),
+        longitude=number(row["longitude"]),
+        country=text(row["country"]),
+        date_precision=precision,
+        geo_precision=geo,
+        date_basis=basis,
+        actor1=text(row.get("actor1")),
+        actor2=text(row.get("actor2")),
+        event_type=text(row.get("event_type")),
+        total_deaths_best=integer(row["fatalities"]),
+        raw_json=canonical_json(row),
+    )
+
+
+def load_acled_csv(path, *, version, errors="raise"):
+    """version is a user-supplied export snapshot label, not an ACLED release number."""
+    rows, sha = read_csv_rows(path, REQUIRED)
+    return normalize(
+        rows,
+        source="acled",
+        version=version,
+        mapper=map_record,
+        input_sha256=sha,
+        origin=Path(path).name,
+        errors=errors,
+        metadata={"adapter": "acled-event-v1", "date_policy": "week ±3 days; month envelope"},
+    )

ccrt/loaders/base.py

@@ -0,0 +1,144 @@
+"""Shared validation and CSV mechanics for explicit source adapters."""
+
+from __future__ import annotations
+
+import csv
+import hashlib
+import io
+import re
+from datetime import date, datetime
+from pathlib import Path
+
+from ccrt.schema import Dataset, SourceRecord, ValidationIssue, canonical_json
+
+
+class ValidationError(ValueError):
+    def __init__(self, issues):
+        self.issues = tuple(issues)
+        first = self.issues[0]
+        super().__init__(f"{len(self.issues)} invalid row(s); row {first.row}: {first.message}")
+
+
+def text(value) -> str:
+    return "" if value is None else str(value).strip()
+
+
+def integer(value) -> int | None:
+    if value is None or text(value) == "":
+        return None
+    if isinstance(value, bool) or not re.fullmatch(r"-?\d+", text(value)):
+        raise ValueError("expected an integer")
+    return int(value)
+
+
+def number(value) -> float | None:
+    return None if value is None or text(value) == "" else float(value)
+
+
+def parse_date(value) -> date | None:
+    value = text(value)
+    if not value:
+        return None
+    if re.fullmatch(r"\d{4}-\d{2}-\d{2}", value):
+        return date.fromisoformat(value)
+    # UCDP CSV exports can contain an ISO timestamp; reject ambiguous locale dates.
+    if re.fullmatch(r"\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})?", value):
+        return datetime.fromisoformat(value.replace("Z", "+00:00")).date()
+    raise ValueError("expected an ISO date (YYYY-MM-DD) or ISO timestamp")
+
+
+def read_csv_rows(path, required):
+    payload = Path(path).read_bytes()
+    reader = csv.DictReader(io.StringIO(payload.decode("utf-8-sig"), newline=""))
+    fields = reader.fieldnames or []
+    if len(fields) != len(set(fields)):
+        raise ValueError("duplicate CSV column names")
+    missing = set(required) - set(fields)
+    if missing:
+        raise ValueError(f"missing columns: {', '.join(sorted(missing))}")
+    rows = []
+    for row in reader:
+        if None in row or any(v is None for v in row.values()):
+            raise ValueError(f"row {reader.line_num}: CSV field count does not match header")
+        rows.append((reader.line_num, row))
+    return rows, hashlib.sha256(payload).hexdigest()
+
+
+def normalize(rows, *, source, version, mapper, input_sha256, origin="", errors="raise", metadata=None):
+    if errors not in {"raise", "collect"}:
+        raise ValueError("errors must be 'raise' or 'collect'")
+    if not isinstance(version, str) or not version.strip():
+        raise ValueError("an explicit source version or export snapshot label is required")
+    records, issues, seen = [], [], set()
+    for line, row in rows:
+        raw = canonical_json(row)
+        try:
+            record = mapper(row, version)
+            if record.key in seen:
+                raise ValueError(f"duplicate source ID: {record.source_id}")
+            seen.add(record.key)
+            records.append(record)
+            estimates = [
+                value
+                for value in (record.total_deaths_low, record.total_deaths_best, record.total_deaths_high)
+                if value is not None
+            ]
+            if len(estimates) > 1 and estimates != sorted(estimates):
+                issues.append(
+                    ValidationIssue(
+                        line,
+                        "source total death estimates are not ordered low <= best <= high; preserved as reported",
+                        "warning",
+                        raw,
+                    )
+                )
+            if record.date_start is None or record.latitude is None or record.longitude is None:
+                issues.append(
+                    ValidationIssue(
+                        line, "missing date or coordinate; retained but cannot match", "warning", raw
+                    )
+                )
+        except (ValueError, TypeError, KeyError, OverflowError) as exc:
+            issues.append(ValidationIssue(line, str(exc), "error", raw))
+    failures = [i for i in issues if i.severity == "error"]
+    if failures and errors == "raise":
+        raise ValidationError(failures)
+    return Dataset(
+        source, version, tuple(records), input_sha256, origin, tuple(issues), canonical_json(metadata or {})
+    )
+
+
+def load_csv(path, source, *, version="user-supplied", errors="raise") -> Dataset:
+    """Import the original minimal CSV format; casualty bounds mean TOTAL deaths."""
+    rows, sha = read_csv_rows(path, {"source_id", "date", "latitude", "longitude"})
+
+    def mapper(row, version):
+        day = parse_date(row["date"])
+        low = integer(row.get("fatalities_min"))
+        high = integer(row.get("fatalities_max"))
+        if low is not None and high is not None and low > high:
+            raise ValueError("fatalities_min cannot exceed fatalities_max")
+        return SourceRecord(
+            source,
+            text(row["source_id"]),
+            version,
+            day,
+            day,
+            number(row["latitude"]),
+            number(row["longitude"]),
+            text(row.get("country")),
+            total_deaths_low=low,
+            total_deaths_high=high,
+            raw_json=canonical_json(row),
+        )
+
+    return normalize(
+        rows,
+        source=source,
+        version=version,
+        mapper=mapper,
+        input_sha256=sha,
+        origin=Path(path).name,
+        errors=errors,
+        metadata={"adapter": "generic-v1"},
+    )

ccrt/loaders/ucdp.py

@@ -0,0 +1,75 @@
+"""UCDP GED 25.1/26.1 field mapping. Other versions require independent validation."""
+
+from pathlib import Path
+
+from ccrt.loaders.base import integer, normalize, number, parse_date, read_csv_rows, text
+from ccrt.schema import SourceRecord, canonical_json, digest
+
+REQUIRED = {"id", "date_start", "date_end", "latitude", "longitude", "country"}
+SUPPORTED_VERSIONS = {"25.1", "26.1"}
+
+
+def check_version(version):
+    if version not in SUPPORTED_VERSIONS:
+        raise ValueError("supported UCDP GED versions: 25.1, 26.1")
+
+
+def map_record(row, version):
+    missing = REQUIRED - row.keys()
+    if missing:
+        raise ValueError(f"missing UCDP fields: {', '.join(sorted(missing))}")
+    date_precision = integer(row.get("date_prec"))
+    geo_precision = integer(row.get("where_prec"))
+    if date_precision is not None and date_precision not in range(1, 6):
+        raise ValueError("UCDP date_prec must be 1 through 5")
+    if geo_precision is not None and geo_precision not in range(1, 8):
+        raise ValueError("UCDP where_prec must be 1 through 7")
+    return SourceRecord(
+        source="ucdp",
+        source_id=text(row["id"]),
+        version=version,
+        date_start=parse_date(row["date_start"]),
+        date_end=parse_date(row["date_end"]),
+        latitude=number(row["latitude"]),
+        longitude=number(row["longitude"]),
+        country=text(row["country"]),
+        date_precision=date_precision,
+        geo_precision=geo_precision,
+        actor1=text(row.get("side_a")),
+        actor2=text(row.get("side_b")),
+        event_type=text(row.get("type_of_violence")),
+        total_deaths_low=integer(row.get("low")),
+        total_deaths_best=integer(row.get("best")),
+        total_deaths_high=integer(row.get("high")),
+        civilian_deaths=integer(row.get("deaths_civilians")),
+        raw_json=canonical_json(row),
+    )
+
+
+def load_ucdp_csv(path, *, version, errors="raise"):
+    check_version(version)
+    rows, sha = read_csv_rows(path, REQUIRED)
+    return normalize(
+        rows,
+        source="ucdp",
+        version=version,
+        mapper=map_record,
+        input_sha256=sha,
+        origin=Path(path).name,
+        errors=errors,
+        metadata={"adapter": "ucdp-ged-v1"},
+    )
+
+
+def from_api_records(records, *, version, metadata=None, errors="raise"):
+    check_version(version)
+    return normalize(
+        enumerate(records, 1),
+        source="ucdp",
+        version=version,
+        mapper=map_record,
+        input_sha256=digest(records),
+        origin="ucdp-api",
+        errors=errors,
+        metadata={"adapter": "ucdp-ged-v1", **(metadata or {})},
+    )

ccrt/merge.py

@@ -0,0 +1,186 @@
+"""Inspectable candidate generation. No automatic assignment or casualty aggregation."""
+
+from __future__ import annotations
+
+from bisect import bisect_left, bisect_right
+from dataclasses import dataclass, field
+from functools import cached_property
+from math import asin, cos, isfinite, radians, sin, sqrt
+from types import MappingProxyType
+
+from ccrt.schema import Dataset, SourceRecord, digest
+
+
+@dataclass(frozen=True)
+class MatchConfig:
+    max_days: int = 3
+    max_km: float = 25.0
+    include_coarse_locations: bool = False
+    country_aliases: dict[str, str] = field(default_factory=dict)
+
+    def __post_init__(self):
+        if type(self.max_days) is not int or self.max_days < 0:
+            raise ValueError("max_days must be a nonnegative integer")
+        if (
+            isinstance(self.max_km, bool)
+            or not isinstance(self.max_km, (int, float))
+            or not isfinite(self.max_km)
+            or self.max_km <= 0
+        ):
+            raise ValueError("max_km must be finite and positive")
+        if type(self.include_coarse_locations) is not bool:
+            raise ValueError("include_coarse_locations must be boolean")
+        if any(
+            not isinstance(k, str) or not isinstance(v, str) or not k.strip() or not v.strip()
+            for k, v in self.country_aliases.items()
+        ):
+            raise ValueError("country_aliases must map nonempty strings to nonempty strings")
+        object.__setattr__(
+            self,
+            "country_aliases",
+            MappingProxyType(
+                {k.strip().casefold(): v.strip().casefold() for k, v in self.country_aliases.items()}
+            ),
+        )
+
+    def to_dict(self):
+        return {
+            "max_days": self.max_days,
+            "max_km": self.max_km,
+            "include_coarse_locations": self.include_coarse_locations,
+            "country_aliases": dict(self.country_aliases),
+        }
+
+
+@dataclass(frozen=True)
+class Candidate:
+    candidate_id: str
+    left: SourceRecord
+    right: SourceRecord
+    days_apart: int
+    distance_km: float
+    score: float
+    warnings: tuple[str, ...] = ()
+
+
+@dataclass(frozen=True)
+class UnlinkedRecord:
+    record: SourceRecord
+    reason: str
+
+
+@dataclass(frozen=True)
+class Reconciliation:
+    left: Dataset
+    right: Dataset
+    config: MatchConfig
+    candidates: tuple[Candidate, ...]
+    unlinked: tuple[UnlinkedRecord, ...]
+
+    @cached_property
+    def run_id(self):
+        return digest(
+            {
+                "left": self.left.fingerprint,
+                "right": self.right.fingerprint,
+                "config": self.config.to_dict(),
+                "algorithm": "interval-haversine-v1",
+            }
+        )
+
+
+def distance_km(left: SourceRecord, right: SourceRecord) -> float:
+    if any(v is None for v in (left.latitude, left.longitude, right.latitude, right.longitude)):
+        raise ValueError("distance needs complete coordinates")
+    lat1, lat2 = radians(left.latitude), radians(right.latitude)
+    a = (
+        sin((lat2 - lat1) / 2) ** 2
+        + cos(lat1) * cos(lat2) * sin(radians(right.longitude - left.longitude) / 2) ** 2
+    )
+    return 6371.0088 * 2 * asin(min(1.0, sqrt(max(0.0, a))))
+
+
+def _ineligible(record, config):
+    if record.date_start is None:
+        return "missing_date"
+    if record.latitude is None or record.longitude is None:
+        return "missing_coordinates"
+    if (
+        not config.include_coarse_locations
+        and record.source in {"ucdp", "acled"}
+        and record.geo_precision is not None
+        and record.geo_precision >= 3
+    ):
+        return "coarse_location_excluded"
+    return None
+
+
+def _country(record, config):
+    country = record.country.strip().casefold()
+    return config.country_aliases.get(country, country)
+
+
+def reconcile(left: Dataset, right: Dataset, *, config: MatchConfig | None = None) -> Reconciliation:
+    """Return all eligible pairs; a record may occur in several candidates.
+
+    Date distance is the gap between closed intervals. Location precision codes are
+    source-specific categories, not radii. Coarse locations are excluded by default.
+    """
+    config = config or MatchConfig()
+    if left.source == right.source:
+        raise ValueError("reconciliation requires distinct sources")
+    seed = digest(
+        {
+            "left": left.fingerprint,
+            "right": right.fingerprint,
+            "config": config.to_dict(),
+            "algorithm": "interval-haversine-v1",
+        }
+    )
+    eligible_right = sorted(
+        (r for r in right if not _ineligible(r, config)), key=lambda r: (r.date_start, r.key)
+    )
+    starts = [r.date_start.toordinal() for r in eligible_right]
+    prefix_ends = []
+    for record in eligible_right:
+        prefix_ends.append(max(record.date_end.toordinal(), prefix_ends[-1] if prefix_ends else 0))
+    candidates, linked = [], set()
+    for a in sorted(left, key=lambda r: r.key):
+        if _ineligible(a, config):
+            continue
+        lower = a.date_start.toordinal() - config.max_days
+        upper = a.date_end.toordinal() + config.max_days
+        begin = bisect_left(prefix_ends, lower)
+        end = bisect_right(starts, upper)
+        for index in range(begin, end):
+            b = eligible_right[index]
+            if b.date_end.toordinal() < lower:
+                continue
+            ca, cb = _country(a, config), _country(b, config)
+            if ca and cb and ca != cb:
+                continue
+            gap = max(0, (a.date_start - b.date_end).days, (b.date_start - a.date_end).days)
+            km = distance_km(a, b)
+            if km > config.max_km:
+                continue
+            warnings = []
+            if not ca or not cb:
+                warnings.append("country_missing")
+            for side, record in (("left", a), ("right", b)):
+                if record.date_start != record.date_end or record.date_precision != 1:
+                    warnings.append(f"{side}_date_uncertain")
+                if record.geo_precision != 1:
+                    warnings.append(f"{side}_location_uncertain")
+                if record.date_basis != "reported":
+                    warnings.append(f"{side}_date_interval_inferred")
+            score = 1 - (gap / (config.max_days + 1) + km / config.max_km) / 2
+            identifier = digest([seed, a.key, b.key])
+            candidates.append(Candidate(identifier, a, b, gap, km, score, tuple(warnings)))
+            linked.update((a.key, b.key))
+    candidates.sort(key=lambda c: (-c.score, c.left.key, c.right.key))
+    unlinked = tuple(
+        UnlinkedRecord(r, _ineligible(r, config) or "no_candidate")
+        for r in sorted((*left.records, *right.records), key=lambda r: r.key)
+        if r.key not in linked
+    )
+    return Reconciliation(left, right, config, tuple(candidates), unlinked)