datajig 0.5.0.dev0__py3-none-macosx_11_0_arm64.whl

datajig/__init__.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+from importlib import import_module
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from datajig.api import compare
+    from datajig.models import (
+        DistributionDelta,
+        Finding,
+        PolicyResult,
+        ReviewReport,
+        ReviewStatus,
+        SampleMatch,
+        SampleRecord,
+        Severity,
+    )
+    from datajig.policy import PolicyConfig
+
+__all__ = [
+    "DistributionDelta",
+    "Finding",
+    "PolicyConfig",
+    "PolicyResult",
+    "ReviewReport",
+    "ReviewStatus",
+    "SampleMatch",
+    "SampleRecord",
+    "Severity",
+    "compare",
+]
+
+_MODEL_EXPORTS = frozenset(__all__) - {"PolicyConfig", "compare"}
+
+
+def __getattr__(name: str) -> Any:
+    if name == "compare":
+        return import_module("datajig.api").compare
+    if name == "PolicyConfig":
+        return import_module("datajig.policy").PolicyConfig
+    if name in _MODEL_EXPORTS:
+        return getattr(import_module("datajig.models"), name)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

datajig/agent.py

@@ -0,0 +1,270 @@
+from __future__ import annotations
+
+import json
+from collections import Counter
+from importlib.metadata import PackageNotFoundError, version
+
+from datajig.models import Finding, ReviewReport, Severity, indexed_findings
+
+AGENT_API_VERSION = 1
+DEFAULT_PAGE_SIZE = 50
+MAX_PAGE_SIZE = 200
+MAX_COMPACT_FINDINGS = 20
+MAX_COMPACT_CODES = 50
+MAX_COMPACT_SAMPLE_IDS = 5
+MAX_COMPACT_EVIDENCE_ITEMS = 8
+MAX_COMPACT_SEQUENCE_ITEMS = 5
+MAX_COMPACT_TEXT_CHARS = 240
+MAX_COMPACT_JSON_CHARS = 50_000
+
+
+class FindingNotFoundError(LookupError):
+    """Raised when a report does not contain a requested finding ID."""
+
+
+def capabilities() -> dict[str, object]:
+    return {
+        "agent_api_version": AGENT_API_VERSION,
+        "kind": "capabilities",
+        "tool": {"name": "datajig", "version": _tool_version()},
+        "report_schema_versions": [1],
+        "identity_namespace": "datajig-v1",
+        "commands": [
+            "capabilities",
+            "compare",
+            "explain",
+            "finding",
+            "findings",
+            "snapshot",
+            "snapshot-diff",
+            "snapshot-info",
+        ],
+        "limits": {
+            "default_page_size": DEFAULT_PAGE_SIZE,
+            "max_page_size": MAX_PAGE_SIZE,
+            "default_compact_findings": 10,
+            "max_compact_findings": MAX_COMPACT_FINDINGS,
+            "max_compact_characters": MAX_COMPACT_JSON_CHARS,
+        },
+        "features": {
+            "compact_output": True,
+            "deterministic_finding_ids": True,
+            "filtered_pagination": True,
+            "native_manifest_backend": True,
+            "native_manifest_fallback": False,
+            "read_only_queries": True,
+            "snapshot_manifests": True,
+        },
+    }
+
+
+def list_findings(
+    report: ReviewReport,
+    *,
+    severities: tuple[Severity, ...] = (),
+    codes: tuple[str, ...] = (),
+    offset: int = 0,
+    limit: int = DEFAULT_PAGE_SIZE,
+) -> dict[str, object]:
+    _validate_page(offset, limit)
+    severity_filter = set(severities)
+    code_filter = set(codes)
+    filtered = [
+        (finding_id, finding)
+        for finding_id, finding in indexed_findings(report)
+        if (not severity_filter or finding.severity in severity_filter)
+        and (not code_filter or finding.code in code_filter)
+    ]
+    page = filtered[offset : offset + limit]
+    return {
+        "agent_api_version": AGENT_API_VERSION,
+        "kind": "finding_page",
+        "report": _report_descriptor(report),
+        "filters": {
+            "severities": sorted(item.value for item in severity_filter),
+            "codes": sorted(code_filter),
+        },
+        "page": {
+            "offset": offset,
+            "limit": limit,
+            "returned": len(page),
+            "total": len(filtered),
+            "has_more": offset + len(page) < len(filtered),
+        },
+        "findings": [
+            _finding_payload(report, finding_id, finding)
+            for finding_id, finding in page
+        ],
+    }
+
+
+def get_finding(report: ReviewReport, finding_id: str) -> dict[str, object]:
+    for current_id, finding in indexed_findings(report):
+        if current_id == finding_id:
+            return {
+                "agent_api_version": AGENT_API_VERSION,
+                "kind": "finding",
+                "report": _report_descriptor(report),
+                "finding": _finding_payload(report, current_id, finding),
+            }
+    raise FindingNotFoundError(f"finding {finding_id!r} was not found")
+
+
+def compact_summary(report: ReviewReport, *, limit: int = 10) -> dict[str, object]:
+    if limit < 0 or limit > MAX_COMPACT_FINDINGS:
+        raise ValueError(f"limit must be between 0 and {MAX_COMPACT_FINDINGS}")
+    indexed = indexed_findings(report)
+    severity_counts = Counter(finding.severity.value for _, finding in indexed)
+    code_counts = Counter(finding.code for _, finding in indexed)
+    selected = indexed[:limit]
+    compact_codes = _compact_code_counts(code_counts)
+    compact_findings = [
+        _compact_finding_payload(report, finding_id, finding)
+        for finding_id, finding in selected
+    ]
+    result: dict[str, object] = {
+        "agent_api_version": AGENT_API_VERSION,
+        "kind": "compact_summary",
+        "report": _compact_report_descriptor(report),
+        "counts": {
+            "total": len(indexed),
+            "by_severity": {
+                severity.value: severity_counts.get(severity.value, 0)
+                for severity in sorted(Severity, key=lambda item: item.value)
+            },
+            "by_code": compact_codes,
+        },
+        "code_kinds_total": len(code_counts),
+        "code_kinds_truncated": len(code_counts) > len(compact_codes),
+        "findings": compact_findings,
+        "truncated": len(indexed) > len(compact_findings),
+        "budget_truncated": False,
+    }
+    while compact_findings and _json_size(result) > MAX_COMPACT_JSON_CHARS:
+        compact_findings.pop()
+        result["truncated"] = True
+        result["budget_truncated"] = True
+    return result
+
+
+def _finding_payload(
+    report: ReviewReport,
+    finding_id: str,
+    finding: Finding,
+) -> dict[str, object]:
+    configured = dict(report.policy.effective_policy).get(
+        finding.code, finding.severity.value
+    )
+    effective_severity = (
+        configured if isinstance(configured, str) else finding.severity.value
+    )
+    return {
+        "id": finding_id,
+        "code": finding.code,
+        "severity": finding.severity.value,
+        "effective_severity": effective_severity,
+        "message": finding.message,
+        "sample_ids": sorted(finding.sample_ids),
+        "evidence": dict(sorted(finding.evidence)),
+    }
+
+
+def _report_descriptor(report: ReviewReport) -> dict[str, object]:
+    return {
+        "schema_version": report.schema_version,
+        "baseline": report.baseline,
+        "candidate": report.candidate,
+        "complete": report.complete,
+        "status": report.policy.status.value,
+    }
+
+
+def _compact_report_descriptor(report: ReviewReport) -> dict[str, object]:
+    descriptor = _report_descriptor(report)
+    descriptor["baseline"] = _compact_text(report.baseline)
+    descriptor["candidate"] = _compact_text(report.candidate)
+    return descriptor
+
+
+def _compact_finding_payload(
+    report: ReviewReport,
+    finding_id: str,
+    finding: Finding,
+) -> dict[str, object]:
+    configured = dict(report.policy.effective_policy).get(
+        finding.code, finding.severity.value
+    )
+    effective_severity = (
+        configured if isinstance(configured, str) else finding.severity.value
+    )
+    sample_ids = sorted(finding.sample_ids)
+    evidence_items = sorted(finding.evidence)[:MAX_COMPACT_EVIDENCE_ITEMS]
+    return {
+        "id": finding_id,
+        "code": _compact_text(finding.code),
+        "severity": finding.severity.value,
+        "effective_severity": effective_severity,
+        "message": _compact_text(finding.message),
+        "message_truncated": len(finding.message) > MAX_COMPACT_TEXT_CHARS,
+        "sample_ids": [
+            _compact_text(item) for item in sample_ids[:MAX_COMPACT_SAMPLE_IDS]
+        ],
+        "sample_ids_total": len(sample_ids),
+        "sample_ids_truncated": len(sample_ids) > MAX_COMPACT_SAMPLE_IDS,
+        "evidence": {
+            _compact_text(key): _compact_evidence_value(value)
+            for key, value in evidence_items
+        },
+        "evidence_total": len(finding.evidence),
+        "evidence_truncated": len(finding.evidence) > len(evidence_items),
+    }
+
+
+def _compact_code_counts(counts: Counter[str]) -> dict[str, int]:
+    compact: dict[str, int] = {}
+    ordered = sorted(counts.items(), key=lambda item: (-item[1], item[0]))
+    for code, count in ordered[:MAX_COMPACT_CODES]:
+        key = _compact_text(code)
+        compact[key] = compact.get(key, 0) + count
+    return dict(sorted(compact.items()))
+
+
+def _compact_evidence_value(value: object) -> object:
+    if isinstance(value, tuple):
+        return {
+            "items": [
+                _compact_evidence_value(item)
+                for item in value[:MAX_COMPACT_SEQUENCE_ITEMS]
+            ],
+            "total": len(value),
+            "truncated": len(value) > MAX_COMPACT_SEQUENCE_ITEMS,
+        }
+    if isinstance(value, str):
+        return _compact_text(value)
+    return value
+
+
+def _compact_text(value: str) -> str:
+    if len(value) <= MAX_COMPACT_TEXT_CHARS:
+        return value
+    return f"{value[: MAX_COMPACT_TEXT_CHARS - 1]}…"
+
+
+def _json_size(value: object) -> int:
+    return len(
+        json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+    )
+
+
+def _validate_page(offset: int, limit: int) -> None:
+    if offset < 0:
+        raise ValueError("offset must be non-negative")
+    if limit < 1 or limit > MAX_PAGE_SIZE:
+        raise ValueError(f"limit must be between 1 and {MAX_PAGE_SIZE}")
+
+
+def _tool_version() -> str:
+    try:
+        return version("datajig")
+    except PackageNotFoundError:
+        return "0+unknown"

datajig/analyzers/__init__.py

@@ -0,0 +1,2 @@
+"""Semantic analyzers for dataset review findings."""
+

datajig/analyzers/changes.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+from datajig.matching import MatchResult
+from datajig.models import Finding, Severity
+
+
+def analyze_changes(result: MatchResult) -> tuple[Finding, ...]:
+    before_by_path = {item.relative_path: item for item in result.baseline}
+    after_by_path = {item.relative_path: item for item in result.candidate}
+    findings: list[Finding] = list(result.findings)
+
+    for match in result.matches:
+        if match.confidence == "ambiguous":
+            continue
+        before = before_by_path[match.baseline_id]
+        after = after_by_path[match.candidate_id]
+        sample_ids = (before.relative_path, after.relative_path)
+        if before.split != after.split:
+            findings.append(_finding("SPLIT_CHANGED", "sample changed split", sample_ids))
+        if before.label != after.label:
+            findings.append(_finding("LABEL_CHANGED", "sample changed label", sample_ids))
+        if before.logical_path != after.logical_path:
+            findings.append(_finding("SAMPLE_MOVED", "sample path changed", sample_ids))
+        if before.content_hash != after.content_hash:
+            code = (
+                "PROBABLE_REENCODE"
+                if match.distance is not None and match.distance <= result.phash_threshold
+                else "CONTENT_CHANGED"
+            )
+            message = (
+                "sample bytes changed but perceptual content is similar"
+                if code == "PROBABLE_REENCODE"
+                else "sample content changed"
+            )
+            findings.append(_finding(code, message, sample_ids, distance=match.distance))
+
+    for item in result.unmatched_candidate:
+        findings.append(
+            _finding("SAMPLE_ADDED", "sample was added", (item.relative_path,))
+        )
+    for item in result.unmatched_baseline:
+        findings.append(
+            _finding("SAMPLE_REMOVED", "sample was removed", (item.relative_path,))
+        )
+    findings.sort(key=lambda item: (item.code, item.sample_ids))
+    return tuple(findings)
+
+
+def _finding(
+    code: str,
+    message: str,
+    sample_ids: tuple[str, ...],
+    *,
+    distance: int | None = None,
+) -> Finding:
+    evidence = () if distance is None else (("perceptual_distance", distance),)
+    return Finding(code, Severity.INFO, message, sample_ids, evidence)

datajig/analyzers/distribution.py

@@ -0,0 +1,114 @@
+from __future__ import annotations
+
+from collections import Counter
+from dataclasses import dataclass
+
+from datajig.inventory import InventoryResult
+from datajig.models import DistributionDelta, SampleRecord
+
+SIZE_BUCKET_BOUNDARIES = (256, 512, 1024)
+ASPECT_RATIO_BOUNDARIES = (0.8, 1.25)
+BUCKET_BOUNDARIES = (
+    "size: <256, 256-511, 512-1023, >=1024, unknown",
+    "aspect_ratio: portrait <0.8, square 0.8-1.25, landscape >1.25, unknown",
+)
+
+
+@dataclass(frozen=True, slots=True)
+class DistributionBucket:
+    dimension: str
+    key: str
+    count: int
+    proportion: float
+
+
+@dataclass(frozen=True, slots=True)
+class DistributionSummary:
+    total: int
+    buckets: tuple[DistributionBucket, ...]
+    bucket_boundaries: tuple[str, ...] = BUCKET_BOUNDARIES
+
+
+def summarize_distribution(inventory: InventoryResult) -> DistributionSummary:
+    total = len(inventory.records)
+    counts: Counter[tuple[str, str]] = Counter()
+    for record in inventory.records:
+        for dimension, key in _dimensions(record):
+            counts[(dimension, key)] += 1
+    buckets = tuple(
+        DistributionBucket(
+            dimension,
+            key,
+            count,
+            count / total if total else 0.0,
+        )
+        for (dimension, key), count in sorted(counts.items())
+    )
+    return DistributionSummary(total, buckets)
+
+
+def compare_distributions(
+    before: DistributionSummary, after: DistributionSummary
+) -> tuple[DistributionDelta, ...]:
+    before_map = {(item.dimension, item.key): item for item in before.buckets}
+    after_map = {(item.dimension, item.key): item for item in after.buckets}
+    deltas: list[DistributionDelta] = []
+    for dimension, key in sorted(before_map.keys() | after_map.keys()):
+        before_item = before_map.get((dimension, key))
+        after_item = after_map.get((dimension, key))
+        before_count = before_item.count if before_item else 0
+        after_count = after_item.count if after_item else 0
+        before_proportion = before_item.proportion if before_item else 0.0
+        after_proportion = after_item.proportion if after_item else 0.0
+        percentage_delta = (
+            ((after_proportion - before_proportion) / before_proportion) * 100
+            if before_proportion
+            else None
+        )
+        deltas.append(
+            DistributionDelta(
+                dimension,
+                key,
+                before_count,
+                after_count,
+                before_proportion,
+                after_proportion,
+                percentage_delta,
+            )
+        )
+    return tuple(deltas)
+
+
+def _dimensions(record: SampleRecord) -> tuple[tuple[str, str], ...]:
+    return (
+        ("split", record.split),
+        ("label", record.label),
+        ("width", _size_bucket(record.width)),
+        ("height", _size_bucket(record.height)),
+        ("aspect_ratio", _aspect_bucket(record.width, record.height)),
+        ("format", record.media_format or "unknown"),
+        ("channels", str(record.channels) if record.channels is not None else "unknown"),
+    )
+
+
+def _size_bucket(value: int | None) -> str:
+    if value is None:
+        return "unknown"
+    if value < SIZE_BUCKET_BOUNDARIES[0]:
+        return "<256"
+    if value < SIZE_BUCKET_BOUNDARIES[1]:
+        return "256-511"
+    if value < SIZE_BUCKET_BOUNDARIES[2]:
+        return "512-1023"
+    return ">=1024"
+
+
+def _aspect_bucket(width: int | None, height: int | None) -> str:
+    if width is None or height is None or height == 0:
+        return "unknown"
+    ratio = width / height
+    if ratio < ASPECT_RATIO_BOUNDARIES[0]:
+        return "portrait"
+    if ratio <= ASPECT_RATIO_BOUNDARIES[1]:
+        return "square"
+    return "landscape"

datajig/analyzers/leakage.py

@@ -0,0 +1,125 @@
+from __future__ import annotations
+
+from collections import defaultdict
+
+from datajig.inventory import InventoryResult
+from datajig.models import Finding, SampleRecord, Severity
+from datajig.phash_index import PhashIndex
+
+
+def analyze_leakage(
+    inventory: InventoryResult, threshold: int
+) -> tuple[Finding, ...]:
+    if threshold < 0:
+        raise ValueError("threshold must be non-negative")
+    findings: list[Finding] = []
+    by_digest: dict[str, list[SampleRecord]] = defaultdict(list)
+    for record in inventory.records:
+        by_digest[record.content_hash].append(record)
+    for digest in sorted(by_digest):
+        group = sorted(by_digest[digest], key=lambda item: item.relative_path)
+        if len(group) < 2:
+            continue
+        sample_ids = tuple(item.relative_path for item in group)
+        exact_evidence = (("group_size", len(group)), ("content_hash", digest))
+        findings.append(
+            Finding(
+                "EXACT_DUPLICATE_GROUP",
+                Severity.WARNING,
+                "identical image bytes appear more than once",
+                sample_ids,
+                exact_evidence,
+            )
+        )
+        if len({item.split for item in group}) > 1:
+            findings.append(
+                Finding(
+                    "CROSS_SPLIT_EXACT_LEAKAGE",
+                    Severity.ERROR,
+                    "identical image bytes appear across splits",
+                    sample_ids,
+                    exact_evidence,
+                )
+            )
+
+    for group in _near_components(inventory.records, threshold):
+        sample_ids = tuple(item.relative_path for item in group)
+        near_evidence = (("group_size", len(group)), ("threshold", threshold))
+        findings.append(
+            Finding(
+                "NEAR_DUPLICATE_GROUP",
+                Severity.WARNING,
+                "perceptually similar images form a duplicate candidate group",
+                sample_ids,
+                near_evidence,
+            )
+        )
+        if len({item.split for item in group}) > 1:
+            findings.append(
+                Finding(
+                    "CROSS_SPLIT_NEAR_LEAKAGE",
+                    Severity.WARNING,
+                    "perceptually similar images appear across splits",
+                    sample_ids,
+                    near_evidence,
+                )
+            )
+    findings.sort(key=lambda item: (item.code, item.sample_ids))
+    return tuple(findings)
+
+
+def _near_components(
+    records: tuple[SampleRecord, ...], threshold: int
+) -> list[list[SampleRecord]]:
+    records = tuple(sorted(records, key=lambda item: item.relative_path))
+    groups: dict[str, dict[str, list[int]]] = defaultdict(lambda: defaultdict(list))
+    for index, record in enumerate(records):
+        if record.perceptual_hash is not None:
+            groups[record.perceptual_hash][record.content_hash].append(index)
+
+    parents = list(range(len(records)))
+
+    def find(index: int) -> int:
+        while parents[index] != index:
+            parents[index] = parents[parents[index]]
+            index = parents[index]
+        return index
+
+    def union(left: int, right: int) -> None:
+        left_root = find(left)
+        right_root = find(right)
+        if left_root != right_root:
+            parents[right_root] = left_root
+
+    def connect(left_group: list[int], right_group: list[int]) -> None:
+        for index in left_group[1:]:
+            union(left_group[0], index)
+        for index in right_group[1:]:
+            union(right_group[0], index)
+        union(left_group[0], right_group[0])
+
+    phash_index: PhashIndex[str] = PhashIndex()
+    for hash_value in sorted(groups):
+        digest_groups = groups[hash_value]
+        digest_values = sorted(digest_groups)
+        if len(digest_values) > 1:
+            anchor = digest_groups[digest_values[0]]
+            for other_digest in digest_values[1:]:
+                connect(anchor, digest_groups[other_digest])
+        for near_match in phash_index.query(hash_value, threshold):
+            for other_hash in near_match.items:
+                combined = [*digest_groups.items(), *groups[other_hash].items()]
+                anchor_digest, anchor_group = combined[0]
+                for other_digest, other_group in combined[1:]:
+                    if anchor_digest != other_digest:
+                        connect(anchor_group, other_group)
+        phash_index.add(hash_value, hash_value)
+
+    connected: dict[int, list[int]] = defaultdict(list)
+    for record_index in range(len(records)):
+        connected[find(record_index)].append(record_index)
+    return [
+        [records[index] for index in indexes]
+        for _, indexes in sorted(connected.items())
+        if len(indexes) > 1
+    ]