ccrt 0.1.0__py3-none-any.whl

ccrt/report.py

@@ -0,0 +1,170 @@
+"""Descriptive review diagnostics for partially or fully reviewed candidates."""
+
+from collections import Counter, defaultdict
+from dataclasses import asdict, dataclass
+
+from ccrt._version import __version__
+from ccrt.review import Review
+from ccrt.schema import digest
+
+
+@dataclass(frozen=True)
+class MultipleAcceptedLinks:
+    source_id: str
+    counterpart_ids: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class FatalityDifference:
+    left_id: str
+    right_id: str
+    left_deaths: int
+    right_deaths: int
+
+
+@dataclass(frozen=True)
+class ReviewReport:
+    run_id: str
+    review_fingerprint: str
+    candidate_count: int
+    reviewed_count: int
+    accepted_count: int
+    rejected_count: int
+    unresolved_count: int
+    reviewed_fraction: float | None
+    reviewed_without_notes: int
+    reviewed_score_min: float | None
+    reviewed_score_max: float | None
+    left_source: str
+    right_source: str
+    left_record_count: int
+    right_record_count: int
+    left_with_candidates: int
+    right_with_candidates: int
+    left_unlinked_reasons: dict[str, int]
+    right_unlinked_reasons: dict[str, int]
+    accepted_left_multiple: tuple[MultipleAcceptedLinks, ...]
+    accepted_right_multiple: tuple[MultipleAcceptedLinks, ...]
+    accepted_fatality_differences: tuple[FatalityDifference, ...]
+    accepted_missing_fatalities: tuple[tuple[str, str], ...]
+
+    def to_dict(self):
+        """JSON-ready diagnostics, with no raw accounts or casualty aggregation."""
+        return {
+            "kind": "review-report",
+            "schema_version": 1,
+            "ccrt_version": __version__,
+            "scope": "generated candidates only; coverage is not matching accuracy",
+            **asdict(self),
+        }
+
+
+def summarize_review(result, review=None):
+    """Summarize decisions without treating unresolved pairs as negative labels.
+
+    Multiple accepted counterparts and differing fatality claims are descriptive
+    checks, not proof of an incorrect link. Missed links outside the candidate
+    set are unknown, so this function does not calculate precision or recall.
+    """
+    review = review if review is not None else Review.create(result)
+    review.validate(result)
+    decisions = {decision.candidate_id: decision for decision in review.decisions}
+    counts = Counter(decision.status for decision in review.decisions)
+    reviewed = [c for c in result.candidates if decisions[c.candidate_id].status != "unresolved"]
+    accepted = [c for c in result.candidates if decisions[c.candidate_id].status == "accepted"]
+    left_links, right_links = defaultdict(set), defaultdict(set)
+    differences, missing = [], []
+    for candidate in accepted:
+        left, right = candidate.left, candidate.right
+        left_links[left.source_id].add(right.source_id)
+        right_links[right.source_id].add(left.source_id)
+        a, b = left.total_deaths_best, right.total_deaths_best
+        if a is None or b is None:
+            missing.append((left.source_id, right.source_id))
+        elif a != b:
+            differences.append(FatalityDifference(left.source_id, right.source_id, a, b))
+
+    def multiple(groups):
+        return tuple(
+            MultipleAcceptedLinks(source_id, tuple(sorted(counterparts)))
+            for source_id, counterparts in sorted(groups.items())
+            if len(counterparts) > 1
+        )
+
+    def unlinked_reasons(source):
+        return dict(sorted(Counter(u.reason for u in result.unlinked if u.record.source == source).items()))
+
+    scores = [c.score for c in reviewed]
+    return ReviewReport(
+        run_id=result.run_id,
+        review_fingerprint=digest(
+            {
+                "run_id": review.run_id,
+                "decisions": sorted((asdict(d) for d in review.decisions), key=lambda d: d["candidate_id"]),
+            }
+        ),
+        candidate_count=len(result.candidates),
+        reviewed_count=len(reviewed),
+        accepted_count=counts["accepted"],
+        rejected_count=counts["rejected"],
+        unresolved_count=counts["unresolved"],
+        reviewed_fraction=len(reviewed) / len(result.candidates) if result.candidates else None,
+        reviewed_without_notes=sum(d.status != "unresolved" and not d.note.strip() for d in review.decisions),
+        reviewed_score_min=min(scores) if scores else None,
+        reviewed_score_max=max(scores) if scores else None,
+        left_source=result.left.source,
+        right_source=result.right.source,
+        left_record_count=len(result.left),
+        right_record_count=len(result.right),
+        left_with_candidates=len({c.left.key for c in result.candidates}),
+        right_with_candidates=len({c.right.key for c in result.candidates}),
+        left_unlinked_reasons=unlinked_reasons(result.left.source),
+        right_unlinked_reasons=unlinked_reasons(result.right.source),
+        accepted_left_multiple=multiple(left_links),
+        accepted_right_multiple=multiple(right_links),
+        accepted_fatality_differences=tuple(sorted(differences, key=lambda d: (d.left_id, d.right_id))),
+        accepted_missing_fatalities=tuple(sorted(missing)),
+    )
+
+
+def format_review_report(report):
+    """Compact CLI summary; the JSON report retains all diagnostic IDs."""
+    fraction = f" ({report.reviewed_fraction:.1%})" if report.reviewed_fraction is not None else ""
+    lines = [
+        f"Reviewed {report.reviewed_count}/{report.candidate_count} candidate links{fraction}",
+        f"{report.accepted_count} accepted, {report.rejected_count} rejected, {report.unresolved_count} unresolved",
+        f"{report.reviewed_without_notes} decided links have no review note",
+        "",
+        "Source records with candidates:",
+        f"  {report.left_source}: {report.left_with_candidates}/{report.left_record_count}",
+        f"  {report.right_source}: {report.right_with_candidates}/{report.right_record_count}",
+    ]
+    for source, reasons in (
+        (report.left_source, report.left_unlinked_reasons),
+        (report.right_source, report.right_unlinked_reasons),
+    ):
+        if reasons:
+            lines.append(f"  {source} unlinked: " + ", ".join(f"{k}={v}" for k, v in reasons.items()))
+    lines.extend(["", "Accepted links with multiple counterparts (allowed):"])
+    for source, groups in (
+        (report.left_source, report.accepted_left_multiple),
+        (report.right_source, report.accepted_right_multiple),
+    ):
+        lines.append(f"  {source}: {len(groups)} records")
+        for group in groups[:5]:
+            lines.append(f"    {group.source_id}: {', '.join(group.counterpart_ids)}")
+        if len(groups) > 5:
+            lines.append(f"    {len(groups) - 5} more; see JSON report")
+    lines.extend(
+        [
+            "",
+            f"Different reported fatality counts: {len(report.accepted_fatality_differences)} accepted links",
+            f"Missing fatality counts: {len(report.accepted_missing_fatalities)} accepted links",
+        ]
+    )
+    if report.reviewed_score_min is not None:
+        lines.append(
+            f"Reviewed score range: {report.reviewed_score_min:.3f} to {report.reviewed_score_max:.3f}"
+        )
+    lines.append("\nCoverage describes generated candidates. Matching accuracy needs exhaustive labels.")
+    return "\n".join(lines)

ccrt/review.py

@@ -0,0 +1,56 @@
+"""Explicit review decisions bound to a reproducible candidate run."""
+
+from dataclasses import dataclass, replace
+from datetime import datetime, timezone
+
+from ccrt.merge import Reconciliation
+
+
+@dataclass(frozen=True)
+class Decision:
+    candidate_id: str
+    status: str = "unresolved"
+    reviewer: str = ""
+    note: str = ""
+    reviewed_at: str = ""
+
+    def __post_init__(self):
+        for field in ("candidate_id", "status", "reviewer", "note", "reviewed_at"):
+            if not isinstance(getattr(self, field), str):
+                raise ValueError(f"{field} must be a string")
+        if self.reviewed_at:
+            try:
+                datetime.fromisoformat(self.reviewed_at.replace("Z", "+00:00"))
+            except ValueError:
+                raise ValueError("reviewed_at must be an ISO timestamp") from None
+        if self.status not in {"accepted", "rejected", "unresolved"}:
+            raise ValueError("status must be accepted, rejected, or unresolved")
+        if self.status != "unresolved" and not self.reviewer.strip():
+            raise ValueError("accepted/rejected decisions need a reviewer")
+
+
+@dataclass(frozen=True)
+class Review:
+    run_id: str
+    decisions: tuple[Decision, ...]
+
+    @classmethod
+    def create(cls, result: Reconciliation):
+        return cls(result.run_id, tuple(Decision(c.candidate_id) for c in result.candidates))
+
+    def validate(self, result: Reconciliation):
+        ids = [d.candidate_id for d in self.decisions]
+        if (
+            self.run_id != result.run_id
+            or len(ids) != len(set(ids))
+            or set(ids) != {c.candidate_id for c in result.candidates}
+        ):
+            raise ValueError("review belongs to a different run or has missing/duplicate/unknown candidates")
+
+    def decide(self, candidate_id, status, *, reviewer, note=""):
+        if candidate_id not in {d.candidate_id for d in self.decisions}:
+            raise ValueError("unknown candidate ID")
+        updated = Decision(candidate_id, status, reviewer, note, datetime.now(timezone.utc).isoformat())
+        return replace(
+            self, decisions=tuple(updated if d.candidate_id == candidate_id else d for d in self.decisions)
+        )

ccrt/schema.py

@@ -0,0 +1,162 @@
+"""Source accounts and provenance; linking accounts never changes these records."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import asdict, dataclass
+from datetime import date
+from functools import cached_property
+from math import isfinite
+
+SCHEMA_VERSION = 1
+
+
+def canonical_json(value: object) -> str:
+    return json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(",", ":"), allow_nan=False)
+
+
+def digest(value: object) -> str:
+    return hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest()
+
+
+@dataclass(frozen=True)
+class SourceRecord:
+    source: str
+    source_id: str
+    version: str
+    date_start: date | None
+    date_end: date | None
+    latitude: float | None = None
+    longitude: float | None = None
+    country: str = ""
+    date_precision: int | None = None
+    geo_precision: int | None = None
+    date_basis: str = "reported"
+    actor1: str = ""
+    actor2: str = ""
+    event_type: str = ""
+    total_deaths_low: int | None = None
+    total_deaths_best: int | None = None
+    total_deaths_high: int | None = None
+    civilian_deaths: int | None = None
+    raw_json: str = "{}"
+
+    def __post_init__(self) -> None:
+        for field in ("source", "source_id", "version"):
+            value = getattr(self, field)
+            if not isinstance(value, str) or not value.strip():
+                raise ValueError(f"{field} must be a nonempty string")
+        for field in ("date_start", "date_end"):
+            value = getattr(self, field)
+            if value is not None and type(value) is not date:
+                raise ValueError(f"{field} must be a date or None")
+        if (self.date_start is None) != (self.date_end is None):
+            raise ValueError("both date bounds must be present or missing")
+        if self.date_start is not None and self.date_start > self.date_end:
+            raise ValueError("date_start exceeds date_end")
+        for field, limit in (("latitude", 90), ("longitude", 180)):
+            value = getattr(self, field)
+            if value is not None and (
+                isinstance(value, bool)
+                or not isinstance(value, (int, float))
+                or not isfinite(value)
+                or not -limit <= value <= limit
+            ):
+                raise ValueError(f"{field} must be finite and within ±{limit}")
+        for field in ("total_deaths_low", "total_deaths_best", "total_deaths_high", "civilian_deaths"):
+            value = getattr(self, field)
+            if value is not None and (type(value) is not int or value < 0):
+                raise ValueError(f"{field} must be a nonnegative integer or None")
+        for field in ("date_precision", "geo_precision"):
+            value = getattr(self, field)
+            if value is not None and (type(value) is not int or value < 1):
+                raise ValueError(f"{field} must be a positive integer or None")
+        if not isinstance(json.loads(self.raw_json), dict):
+            raise ValueError("raw_json must encode an object")
+
+    @property
+    def key(self) -> str:
+        return canonical_json([self.source, self.version, self.source_id])
+
+    @property
+    def raw(self) -> dict:
+        """Return a fresh copy so callers cannot mutate stored provenance."""
+        return json.loads(self.raw_json)
+
+    def to_dict(self) -> dict:
+        result = asdict(self)
+        for field in ("date_start", "date_end"):
+            result[field] = getattr(self, field).isoformat() if getattr(self, field) else None
+        return result
+
+    @classmethod
+    def from_dict(cls, value: dict) -> SourceRecord:
+        values = dict(value)
+        for field in ("date_start", "date_end"):
+            values[field] = date.fromisoformat(values[field]) if values[field] else None
+        return cls(**values)
+
+
+@dataclass(frozen=True)
+class ValidationIssue:
+    row: int
+    message: str
+    severity: str = "error"
+    raw_json: str = "{}"
+
+
+@dataclass(frozen=True)
+class Dataset:
+    source: str
+    version: str
+    records: tuple[SourceRecord, ...]
+    input_sha256: str
+    origin: str = ""
+    issues: tuple[ValidationIssue, ...] = ()
+    metadata_json: str = "{}"
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "records", tuple(self.records))
+        object.__setattr__(self, "issues", tuple(self.issues))
+        if not self.source.strip() or not self.version.strip():
+            raise ValueError("dataset source and version are required")
+        keys = set()
+        for record in self.records:
+            if record.source != self.source or record.version != self.version:
+                raise ValueError("record source/version differs from dataset")
+            if record.key in keys:
+                raise ValueError(f"duplicate record ID: {record.source_id}")
+            keys.add(record.key)
+        if not isinstance(json.loads(self.metadata_json), dict):
+            raise ValueError("metadata_json must encode an object")
+
+    @cached_property
+    def fingerprint(self) -> str:
+        """Content identity includes normalized records, mapping metadata and rejected rows."""
+        return digest(
+            {
+                "source": self.source,
+                "version": self.version,
+                "input_sha256": self.input_sha256,
+                "metadata": json.loads(self.metadata_json),
+                "issues": [asdict(i) for i in self.issues],
+                "records": sorted((r.to_dict() for r in self.records), key=lambda r: r["source_id"]),
+            }
+        )
+
+    def __len__(self) -> int:
+        return len(self.records)
+
+    def __iter__(self):
+        return iter(self.records)
+
+    def to_pandas(self):
+        """Return flat columns with nullable types. Requires the tables extra."""
+        try:
+            import pandas as pd
+        except ImportError:
+            raise ImportError('Install table support with: pip install "ccrt[tables]"') from None
+        return pd.DataFrame(
+            [r.to_dict() for r in self.records], columns=SourceRecord.__dataclass_fields__
+        ).convert_dtypes()

ccrt/storage.py

@@ -0,0 +1,168 @@
+"""Versioned JSON bundles and atomic writes for reproducible analysis."""
+
+import json
+import os
+import tempfile
+from dataclasses import asdict
+from pathlib import Path
+
+from ccrt._version import __version__
+from ccrt.merge import MatchConfig, reconcile
+from ccrt.review import Decision, Review
+from ccrt.schema import SCHEMA_VERSION, Dataset, SourceRecord, ValidationIssue, canonical_json
+
+
+def atomic_write(path, content, *, overwrite=False):
+    path = Path(path)
+    if path.exists() and not overwrite:
+        raise FileExistsError(f"output exists: {path}")
+    with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=path.parent, delete=False) as stream:
+        temporary = Path(stream.name)
+        try:
+            stream.write(content)
+            stream.flush()
+            os.fsync(stream.fileno())
+        except BaseException:
+            temporary.unlink(missing_ok=True)
+            raise
+    try:
+        if overwrite:
+            os.replace(temporary, path)
+        else:
+            # A hard link fails atomically if another writer created the destination.
+            os.link(temporary, path)
+    finally:
+        temporary.unlink(missing_ok=True)
+
+
+def dataset_dict(dataset):
+    return {
+        "schema_version": SCHEMA_VERSION,
+        "ccrt_version": __version__,
+        "kind": "dataset",
+        "source": dataset.source,
+        "version": dataset.version,
+        "input_sha256": dataset.input_sha256,
+        "fingerprint": dataset.fingerprint,
+        "origin": dataset.origin,
+        "metadata_json": dataset.metadata_json,
+        "issues": [asdict(i) for i in dataset.issues],
+        "records": [r.to_dict() for r in dataset.records],
+    }
+
+
+def _check(payload, kind):
+    if (
+        not isinstance(payload, dict)
+        or payload.get("schema_version") != SCHEMA_VERSION
+        or payload.get("kind") != kind
+    ):
+        raise ValueError(f"unsupported CCRT {kind} bundle/schema version")
+
+
+def dataset_from_dict(payload):
+    _check(payload, "dataset")
+    try:
+        dataset = Dataset(
+            payload["source"],
+            payload["version"],
+            tuple(SourceRecord.from_dict(r) for r in payload["records"]),
+            payload["input_sha256"],
+            payload["origin"],
+            tuple(ValidationIssue(**i) for i in payload["issues"]),
+            payload["metadata_json"],
+        )
+        if dataset.fingerprint != payload["fingerprint"]:
+            raise ValueError("dataset fingerprint mismatch")
+        return dataset
+    except (KeyError, TypeError) as exc:
+        raise ValueError("malformed dataset bundle") from exc
+
+
+def save_dataset(dataset, path, *, overwrite=False):
+    atomic_write(path, canonical_json(dataset_dict(dataset)) + "\n", overwrite=overwrite)
+
+
+def load_dataset(path):
+    return dataset_from_dict(json.loads(Path(path).read_text(encoding="utf-8")))
+
+
+def save_run(result, path, *, overwrite=False):
+    payload = {
+        "kind": "run",
+        "schema_version": SCHEMA_VERSION,
+        "ccrt_version": __version__,
+        "run_id": result.run_id,
+        "algorithm": "interval-haversine-v1",
+        "left": dataset_dict(result.left),
+        "right": dataset_dict(result.right),
+        "config": result.config.to_dict(),
+        "candidate_ids": [c.candidate_id for c in result.candidates],
+    }
+    atomic_write(path, canonical_json(payload) + "\n", overwrite=overwrite)
+
+
+def load_run(path):
+    payload = json.loads(Path(path).read_text(encoding="utf-8"))
+    _check(payload, "run")
+    try:
+        if payload["algorithm"] != "interval-haversine-v1":
+            raise ValueError("unsupported matching algorithm")
+        result = reconcile(
+            dataset_from_dict(payload["left"]),
+            dataset_from_dict(payload["right"]),
+            config=MatchConfig(**payload["config"]),
+        )
+        if (
+            result.run_id != payload["run_id"]
+            or [c.candidate_id for c in result.candidates] != payload["candidate_ids"]
+        ):
+            raise ValueError("run fingerprint/candidates differ; regenerate the run")
+        return result
+    except (KeyError, TypeError) as exc:
+        raise ValueError("malformed run bundle") from exc
+
+
+def save_review(review, result, path, *, overwrite=False):
+    review.validate(result)
+    payload = {
+        "kind": "review",
+        "schema_version": SCHEMA_VERSION,
+        "ccrt_version": __version__,
+        "run_id": review.run_id,
+        "decisions": [asdict(d) for d in review.decisions],
+    }
+    atomic_write(path, canonical_json(payload) + "\n", overwrite=overwrite)
+
+
+def load_review(path, result):
+    payload = json.loads(Path(path).read_text(encoding="utf-8"))
+    _check(payload, "review")
+    try:
+        review = Review(payload["run_id"], tuple(Decision(**d) for d in payload["decisions"]))
+    except (KeyError, TypeError) as exc:
+        raise ValueError("malformed review bundle") from exc
+    review.validate(result)
+    return review
+
+
+def atomic_write_bytes(path, content, *, overwrite=False):
+    path = Path(path)
+    if path.exists() and not overwrite:
+        raise FileExistsError(f"output exists: {path}")
+    with tempfile.NamedTemporaryFile(mode="wb", dir=path.parent, delete=False) as stream:
+        temporary = Path(stream.name)
+        try:
+            stream.write(content)
+            stream.flush()
+            os.fsync(stream.fileno())
+        except BaseException:
+            temporary.unlink(missing_ok=True)
+            raise
+    try:
+        if overwrite:
+            os.replace(temporary, path)
+        else:
+            os.link(temporary, path)
+    finally:
+        temporary.unlink(missing_ok=True)