kt-audit 0.1.0__py3-none-any.whl

kt_audit/__init__.py

@@ -0,0 +1,16 @@
+from .audit import audit_frame, audit_path
+from .config import AuditConfig, load_config
+from .errors import AuditError
+from .schema import Finding, Report
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "AuditConfig",
+    "AuditError",
+    "Finding",
+    "Report",
+    "audit_frame",
+    "audit_path",
+    "load_config",
+]

kt_audit/__main__.py

@@ -0,0 +1,4 @@
+from .cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

kt_audit/audit.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import cast
+
+import pandas as pd
+
+from .checks.alignment import check_alignment
+from .checks.multikc import check_multikc
+from .checks.temporal import check_temporal
+from .config import AuditConfig
+from .errors import AuditError
+from .schema import Finding, Report, Verdict, prepare_frame
+
+SCHEMA_VERSION = "0.1"
+_SEVERITY_ORDER = {"error": 0, "warn": 1, "info": 2}
+
+
+def audit_frame(df: pd.DataFrame, config: AuditConfig | None = None) -> Report:
+    config = config or AuditConfig()
+    prepared = prepare_frame(df, config)
+    findings: list[Finding] = []
+    if config.temporal:
+        findings.extend(check_temporal(prepared, config))
+    if config.multikc:
+        findings.extend(check_multikc(prepared, config))
+    metrics: dict[str, dict[str, object]] = {}
+    if config.alignment:
+        alignment_findings, metrics = check_alignment(prepared, config)
+        findings.extend(alignment_findings)
+    findings.sort(key=_finding_sort_key)
+    return Report(
+        schema_version=SCHEMA_VERSION,
+        generated_at=datetime.now(timezone.utc).isoformat(timespec="seconds"),
+        input=_input_summary(prepared),
+        findings=tuple(findings),
+        metrics=metrics,
+        verdict=_verdict(findings),
+    )
+
+
+def audit_path(path: str | Path, config: AuditConfig | None = None) -> Report:
+    path = Path(path)
+    if not path.exists():
+        raise AuditError(f"input not found: {path}")
+    suffix = path.suffix.lower()
+    if suffix in {".jsonl", ".ndjson"}:
+        df = pd.read_json(path, lines=True)
+    elif suffix == ".csv":
+        df = pd.read_csv(path)
+    else:
+        raise AuditError(f"unsupported input format: {suffix}; use .csv or .jsonl")
+    return audit_frame(df, config)
+
+
+def _finding_sort_key(finding: Finding) -> tuple[int, str, str]:
+    return (_SEVERITY_ORDER[finding.severity], finding.check, finding.message)
+
+
+def _verdict(findings: list[Finding]) -> Verdict:
+    severities = {finding.severity for finding in findings}
+    if "error" in severities:
+        return "fail"
+    if "warn" in severities:
+        return "warn"
+    return "pass"
+
+
+def _input_summary(df: pd.DataFrame) -> dict[str, object]:
+    splits = {
+        str(key): int(value) for key, value in df["split"].value_counts().sort_index().items()
+    }
+    has_items = bool(df["item_id"].notna().any())
+    return {
+        "rows": len(df),
+        "users": int(cast(int, df["user_id"].nunique())),
+        "items": int(cast(int, df["item_id"].nunique())) if has_items else None,
+        "folds": int(cast(int, df["_fold"].nunique())),
+        "splits": splits,
+    }

kt_audit/checks/__init__.py

@@ -0,0 +1,5 @@
+from .alignment import check_alignment
+from .multikc import check_multikc
+from .temporal import check_temporal
+
+__all__ = ["check_alignment", "check_multikc", "check_temporal"]

kt_audit/checks/_common.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+from collections.abc import Iterator
+from typing import Any
+
+import pandas as pd
+
+from ..schema import jsonable
+
+
+def iter_folds(df: pd.DataFrame) -> Iterator[tuple[str, pd.DataFrame]]:
+    for fold, group in df.groupby("_fold", sort=True, dropna=False):
+        yield str(fold), group
+
+
+def fold_example(fold: str, example: dict[str, Any]) -> dict[str, object]:
+    values = {key: jsonable(value) for key, value in example.items()}
+    if fold != "__all__":
+        return {"fold": fold, **values}
+    return values
+
+
+def examples_with_fold(
+    frame: pd.DataFrame,
+    columns: tuple[str, ...],
+    max_examples: int,
+) -> tuple[dict[str, object], ...]:
+    examples = []
+    for _, row in frame.head(max_examples).iterrows():
+        values = {column: row[column] for column in columns if column in row.index}
+        examples.append(fold_example(str(row["_fold"]), values))
+    return tuple(examples)
+
+
+def examples_from_frame(
+    frame: pd.DataFrame,
+    columns: tuple[str, ...],
+    max_examples: int,
+    fold: str,
+) -> tuple[dict[str, object], ...]:
+    examples = []
+    for _, row in frame.head(max_examples).iterrows():
+        values = {column: row[column] for column in columns if column in row.index}
+        examples.append(fold_example(fold, values))
+    return tuple(examples)

kt_audit/checks/alignment.py

@@ -0,0 +1,179 @@
+from __future__ import annotations
+
+from typing import cast
+
+import numpy as np
+import pandas as pd
+from sklearn.metrics import accuracy_score, roc_auc_score
+
+from ..config import AuditConfig
+from ..schema import SPLITS, Finding
+from ._common import examples_with_fold, fold_example, iter_folds
+
+
+def check_alignment(
+    df: pd.DataFrame,
+    config: AuditConfig,
+) -> tuple[list[Finding], dict[str, dict[str, object]]]:
+    if "y_pred" not in df.columns:
+        return (
+            [
+                Finding(
+                    "alignment",
+                    "info",
+                    "y_pred is missing; metric checks skipped",
+                    0,
+                    (),
+                )
+            ],
+            {},
+        )
+
+    findings: list[Finding] = []
+    findings.extend(_invalid_findings(df, config))
+    findings.extend(_duplicate_findings(df, config))
+    findings.extend(_constant_findings(df))
+    metrics = _metrics(df)
+    findings.extend(_reported_findings(metrics, config))
+    return findings, metrics
+
+
+def _invalid_findings(df: pd.DataFrame, config: AuditConfig) -> list[Finding]:
+    pred = df["y_pred"].astype("float64")
+    true = df["y_true"].astype("float64")
+    invalid_pred = ~np.isfinite(pred) | (pred < 0) | (pred > 1)
+    invalid_true = ~true.isin([0.0, 1.0])
+    findings: list[Finding] = []
+    if bool(invalid_pred.any()):
+        examples = examples_with_fold(
+            cast(pd.DataFrame, df.loc[invalid_pred]),
+            ("_row", "user_id", "item_id", "y_pred"),
+            config.max_examples,
+        )
+        findings.append(
+            Finding(
+                "alignment",
+                "error",
+                "y_pred has missing, non-finite, or out-of-range values",
+                int(np.count_nonzero(invalid_pred)),
+                examples,
+            )
+        )
+    if bool(invalid_true.any()):
+        examples = examples_with_fold(
+            cast(pd.DataFrame, df.loc[invalid_true]),
+            ("_row", "user_id", "item_id", "y_true"),
+            config.max_examples,
+        )
+        findings.append(
+            Finding(
+                "alignment",
+                "error",
+                "y_true is not binary",
+                int(np.count_nonzero(invalid_true.to_numpy())),
+                examples,
+            )
+        )
+    return findings
+
+
+def _duplicate_findings(df: pd.DataFrame, config: AuditConfig) -> list[Finding]:
+    keys = _key_columns(df)
+    extra = df.duplicated(subset=keys, keep="first")
+    if not bool(extra.any()):
+        return []
+    examples = examples_with_fold(
+        cast(pd.DataFrame, df.loc[df.duplicated(subset=keys, keep=False)]),
+        ("_row", "user_id", "item_id", "timestamp"),
+        config.max_examples,
+    )
+    return [
+        Finding(
+            "alignment",
+            "error",
+            "duplicate interaction keys",
+            int(np.count_nonzero(extra.to_numpy())),
+            examples,
+        )
+    ]
+
+
+def _constant_findings(df: pd.DataFrame) -> list[Finding]:
+    findings: list[Finding] = []
+    for fold, group in iter_folds(df):
+        if group["y_pred"].nunique(dropna=True) <= 1:
+            findings.append(
+                Finding(
+                    "alignment",
+                    "warn",
+                    "predictions are constant within this fold",
+                    1,
+                    (fold_example(fold, {"n": len(group)}),),
+                )
+            )
+    return findings
+
+
+def _key_columns(df: pd.DataFrame) -> list[str]:
+    if bool(df["item_id"].notna().any()):
+        return ["_fold", "user_id", "item_id", "timestamp"]
+    if {"sequence_id", "position"}.issubset(df.columns):
+        return ["_fold", "user_id", "sequence_id", "position"]
+    return ["_fold", "user_id", "timestamp"]
+
+
+def _metrics(df: pd.DataFrame) -> dict[str, dict[str, object]]:
+    valid = df.dropna(subset=["y_true", "y_pred"]).copy()
+    pred = valid["y_pred"].astype("float64")
+    true = valid["y_true"].astype("float64")
+    mask = np.isfinite(pred) & pred.between(0, 1) & true.isin([0.0, 1.0])
+    valid = cast(pd.DataFrame, valid.loc[mask])
+    metrics: dict[str, dict[str, object]] = {}
+    for split in SPLITS:
+        part = cast(pd.DataFrame, valid[valid["split"] == split])
+        if part.empty:
+            continue
+        y = cast(pd.Series, part["y_true"]).astype(int)
+        p = cast(pd.Series, part["y_pred"]).astype(float)
+        auc = float(roc_auc_score(y, p)) if y.nunique() == 2 else None
+        acc = float(accuracy_score(y, (p >= 0.5).astype(int)))
+        metrics[split] = {"auc": auc, "acc": acc, "n": len(part)}
+    return metrics
+
+
+def _reported_findings(
+    metrics: dict[str, dict[str, object]],
+    config: AuditConfig,
+) -> list[Finding]:
+    reported = config.reported_auc
+    if reported is None:
+        return []
+    test_auc = metrics.get("test", {}).get("auc")
+    if not isinstance(test_auc, (int, float)):
+        return [
+            Finding(
+                "alignment",
+                "warn",
+                "reported_auc is set but test AUC cannot be computed",
+                0,
+                (),
+            )
+        ]
+    delta = abs(float(test_auc) - reported)
+    if delta <= config.auc_tolerance:
+        return []
+    return [
+        Finding(
+            "alignment",
+            "error",
+            "reported test AUC differs from recomputed AUC",
+            1,
+            (
+                {
+                    "reported_auc": reported,
+                    "recomputed_auc": float(test_auc),
+                    "delta": delta,
+                },
+            ),
+        )
+    ]

kt_audit/checks/multikc.py

@@ -0,0 +1,160 @@
+from __future__ import annotations
+
+from typing import cast
+
+import numpy as np
+import pandas as pd
+
+from ..config import AuditConfig
+from ..schema import Finding
+from ._common import fold_example, iter_folds
+
+
+def check_multikc(df: pd.DataFrame, config: AuditConfig) -> list[Finding]:
+    findings: list[Finding] = []
+    for fold, group in iter_folds(df):
+        findings.extend(_check_fold(fold, group, config))
+    return findings
+
+
+def _check_fold(fold: str, group: pd.DataFrame, config: AuditConfig) -> list[Finding]:
+    if bool(group["item_id"].isna().all()):
+        return [
+            Finding(
+                "multi_kc",
+                "info",
+                "item_id is missing; multi-KC leakage check skipped",
+                0,
+                (),
+            )
+        ]
+    working = group.dropna(subset=["item_id"])
+    findings = _missing_findings(group)
+    if working.empty:
+        return findings
+    item_splits, item_kcs, kc_splits = _aggregate(working)
+    overlap_items, cross_items = _leaks(item_splits, item_kcs, kc_splits)
+    findings.extend(
+        _leak_findings(
+            overlap_items,
+            cross_items,
+            item_kcs,
+            item_splits,
+            kc_splits,
+            fold,
+            config,
+        )
+    )
+    return findings
+
+
+def _missing_findings(group: pd.DataFrame) -> list[Finding]:
+    missing = int(np.count_nonzero(group["item_id"].isna().to_numpy()))
+    if not missing:
+        return []
+    return [
+        Finding(
+            "multi_kc",
+            "warn",
+            "rows with missing item_id were skipped",
+            missing,
+            (),
+        )
+    ]
+
+
+def _aggregate(working: pd.DataFrame) -> tuple[pd.Series, pd.Series, pd.Series]:
+    item_splits = cast(
+        pd.Series,
+        working.groupby("item_id")["split"].agg(lambda values: tuple(sorted(set(values)))),
+    )
+    item_kcs = cast(pd.Series, working.groupby("item_id")["kc_ids"].agg(_union_kcs))
+    exploded = working.explode("kc_ids")
+    kc_splits = cast(
+        pd.Series,
+        exploded.groupby("kc_ids")["split"].agg(lambda values: frozenset(values)),
+    )
+    return item_splits, item_kcs, kc_splits
+
+
+def _leaks(
+    item_splits: pd.Series,
+    item_kcs: pd.Series,
+    kc_splits: pd.Series,
+) -> tuple[list[str], list[str]]:
+    overlap_items: list[str] = []
+    cross_items: list[str] = []
+    for item_id, kcs in item_kcs.items():
+        splits = set(item_splits[item_id])
+        if not {"train", "test"}.issubset(splits):
+            continue
+        overlap_items.append(str(item_id))
+        train_only = [kc for kc in kcs if set(kc_splits[kc]) == {"train"}]
+        test_only = [kc for kc in kcs if set(kc_splits[kc]) == {"test"}]
+        if train_only and test_only:
+            cross_items.append(str(item_id))
+    return overlap_items, cross_items
+
+
+def _leak_findings(
+    overlap_items: list[str],
+    cross_items: list[str],
+    item_kcs: pd.Series,
+    item_splits: pd.Series,
+    kc_splits: pd.Series,
+    fold: str,
+    config: AuditConfig,
+) -> list[Finding]:
+    findings: list[Finding] = []
+    if overlap_items:
+        examples = tuple(
+            _example(item, item_kcs, item_splits, kc_splits, fold)
+            for item in overlap_items[: config.max_examples]
+        )
+        findings.append(
+            Finding(
+                "multi_kc",
+                "warn",
+                "the same item appears in train and test",
+                len(overlap_items),
+                examples,
+            )
+        )
+    if cross_items:
+        examples = tuple(
+            _example(item, item_kcs, item_splits, kc_splits, fold)
+            for item in cross_items[: config.max_examples]
+        )
+        findings.append(
+            Finding(
+                "multi_kc",
+                "error",
+                "an item has train-only and test-only knowledge components",
+                len(cross_items),
+                examples,
+            )
+        )
+    return findings
+
+
+def _union_kcs(values: pd.Series) -> tuple[str, ...]:
+    return tuple(sorted({kc for value in values for kc in value}))
+
+
+def _example(
+    item_id: str,
+    item_kcs: pd.Series,
+    item_splits: pd.Series,
+    kc_splits: pd.Series,
+    fold: str,
+) -> dict[str, object]:
+    kcs = item_kcs[item_id]
+    return fold_example(
+        fold,
+        {
+            "item_id": item_id,
+            "splits": list(item_splits[item_id]),
+            "kc_ids": list(kcs),
+            "kc_splits": {kc: sorted(kc_splits[kc]) for kc in kcs},
+        },
+    )

kt_audit/checks/temporal.py

@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+from typing import Any, cast
+
+import pandas as pd
+
+from ..config import AuditConfig
+from ..schema import Finding
+from ._common import examples_from_frame, fold_example, iter_folds
+
+
+def check_temporal(df: pd.DataFrame, config: AuditConfig) -> list[Finding]:
+    findings: list[Finding] = []
+    for fold, group in iter_folds(df):
+        findings.extend(_check_fold(fold, group, config))
+    return findings
+
+
+def _check_fold(fold: str, group: pd.DataFrame, config: AuditConfig) -> list[Finding]:
+    train = cast(pd.DataFrame, group[group["split"] == "train"])
+    evaluation = cast(pd.DataFrame, group[group["split"].isin(["valid", "test"])])
+    if train.empty or evaluation.empty:
+        return []
+    findings = _global_findings(fold, train, evaluation)
+    findings.extend(_overlap_findings(fold, group, config))
+    findings.extend(_violation_findings(fold, train, evaluation, config))
+    return findings
+
+
+def _global_findings(
+    fold: str,
+    train: pd.DataFrame,
+    evaluation: pd.DataFrame,
+) -> list[Finding]:
+    train_max: Any = train["timestamp"].max()
+    eval_min: Any = evaluation["timestamp"].min()
+    if train_max < eval_min:
+        return []
+    example = fold_example(fold, {"train_max": train_max, "eval_min": eval_min})
+    return [
+        Finding(
+            "temporal_split",
+            "error",
+            "training events are not strictly before evaluation events",
+            1,
+            (example,),
+        )
+    ]
+
+
+def _overlap_findings(
+    fold: str,
+    group: pd.DataFrame,
+    config: AuditConfig,
+) -> list[Finding]:
+    split_counts = group.groupby("user_id")["split"].nunique()
+    multi = cast(pd.Series, split_counts[split_counts > 1])
+    if multi.empty:
+        return []
+    users = (
+        group[group["user_id"].isin(multi.index.tolist())]
+        .groupby("user_id")["split"]
+        .agg(lambda values: tuple(sorted(set(values))))
+    )
+    examples = tuple(
+        fold_example(fold, {"user_id": user, "splits": splits})
+        for user, splits in users.head(config.max_examples).items()
+    )
+    return [
+        Finding(
+            "temporal_split",
+            "warn",
+            "users appear in more than one split",
+            len(multi),
+            examples,
+        )
+    ]
+
+
+def _violation_findings(
+    fold: str,
+    train: pd.DataFrame,
+    evaluation: pd.DataFrame,
+    config: AuditConfig,
+) -> list[Finding]:
+    train_max_by_user = train.groupby("user_id")["timestamp"].max()
+    eval_min_by_user = evaluation.groupby("user_id")["timestamp"].min()
+    merged = pd.DataFrame({"train_max": train_max_by_user, "eval_min": eval_min_by_user}).dropna()
+    violations = cast(pd.DataFrame, merged[merged["train_max"] >= merged["eval_min"]])
+    if violations.empty:
+        return []
+    examples = examples_from_frame(
+        violations.reset_index(),
+        ("user_id", "train_max", "eval_min"),
+        config.max_examples,
+        fold,
+    )
+    return [
+        Finding(
+            "temporal_split",
+            "error",
+            "a user's training events overlap or follow evaluation events",
+            len(violations),
+            examples,
+        )
+    ]

kt_audit/cli.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+import argparse
+import sys
+from dataclasses import replace
+from pathlib import Path
+
+from . import __version__
+from .audit import audit_path
+from .config import load_config
+from .errors import AuditError
+from .report import report_to_markdown, write_json
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="kt-audit",
+        description="Audit knowledge-tracing evaluation before you trust the AUC.",
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+    run = subparsers.add_parser("run", help="audit a CSV or JSONL file")
+    run.add_argument("input", type=Path, help="input CSV or JSONL")
+    run.add_argument(
+        "--out",
+        type=Path,
+        default=Path("kt-audit-report"),
+        help="output directory (default: kt-audit-report)",
+    )
+    run.add_argument("--config", type=Path, default=None, help="JSON config")
+    run.add_argument("--no-temporal", action="store_true", help="skip temporal split check")
+    run.add_argument("--no-multikc", action="store_true", help="skip multi-KC check")
+    run.add_argument("--no-alignment", action="store_true", help="skip prediction alignment check")
+    parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    try:
+        config = load_config(args.config)
+        config = replace(
+            config,
+            temporal=config.temporal and not args.no_temporal,
+            multikc=config.multikc and not args.no_multikc,
+            alignment=config.alignment and not args.no_alignment,
+        )
+        report = audit_path(args.input, config)
+    except (AuditError, FileNotFoundError, ValueError) as exc:
+        print(f"kt-audit: {exc}", file=sys.stderr)
+        return 2
+
+    args.out.mkdir(parents=True, exist_ok=True)
+    write_json(report, args.out / "report.json")
+    (args.out / "report.md").write_text(report_to_markdown(report), encoding="utf-8")
+    errors = sum(finding.severity == "error" for finding in report.findings)
+    warnings = sum(finding.severity == "warn" for finding in report.findings)
+    print(f"verdict: {report.verdict}")
+    print(f"findings: {errors} error, {warnings} warn")
+    print(f"report: {args.out / 'report.md'}")
+    return 1 if report.verdict == "fail" else 0

kt_audit/config.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, fields
+from pathlib import Path
+from typing import Any, cast
+
+from .errors import AuditError
+
+
+@dataclass(frozen=True)
+class AuditConfig:
+    kc_separator: str = "|"
+    temporal: bool = True
+    multikc: bool = True
+    alignment: bool = True
+    max_examples: int = 3
+    reported_auc: float | None = None
+    auc_tolerance: float = 0.01
+
+    def __post_init__(self) -> None:
+        if not self.kc_separator:
+            raise ValueError("kc_separator must not be empty")
+        if self.max_examples < 1:
+            raise ValueError("max_examples must be >= 1")
+        if self.auc_tolerance < 0:
+            raise ValueError("auc_tolerance must be >= 0")
+        if self.reported_auc is not None and not 0 <= self.reported_auc <= 1:
+            raise ValueError("reported_auc must be in [0, 1]")
+
+
+def load_config(path: str | Path | None) -> AuditConfig:
+    if path is None:
+        return AuditConfig()
+    path = Path(path)
+    try:
+        raw = json.loads(path.read_text(encoding="utf-8"))
+    except FileNotFoundError as exc:
+        raise AuditError(f"config not found: {path}") from exc
+    except json.JSONDecodeError as exc:
+        raise AuditError(f"invalid JSON in {path}: {exc}") from exc
+    if not isinstance(raw, dict):
+        raise AuditError("config must be a JSON object")
+    allowed = {item.name for item in fields(AuditConfig)}
+    unknown = sorted(set(raw) - allowed)
+    if unknown:
+        raise AuditError(f"unknown config keys: {', '.join(unknown)}")
+    try:
+        return AuditConfig(**cast(dict[str, Any], raw))
+    except (TypeError, ValueError) as exc:
+        raise AuditError(f"invalid config: {exc}") from exc

kt_audit/errors.py

@@ -0,0 +1,2 @@
+class AuditError(ValueError):
+    """Raised when the input or configuration cannot be audited."""

kt_audit/report.py

@@ -0,0 +1,115 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from .schema import Finding, Report, jsonable
+
+
+def finding_to_dict(finding: Finding) -> dict[str, object]:
+    return {
+        "check": finding.check,
+        "severity": finding.severity,
+        "message": finding.message,
+        "count": finding.count,
+        "examples": jsonable(finding.examples),
+    }
+
+
+def report_to_dict(report: Report) -> dict[str, object]:
+    return {
+        "schema_version": report.schema_version,
+        "generated_at": report.generated_at,
+        "verdict": report.verdict,
+        "input": jsonable(report.input),
+        "metrics": jsonable(report.metrics),
+        "findings": [finding_to_dict(finding) for finding in report.findings],
+    }
+
+
+def write_json(report: Report, path: Path) -> None:
+    payload = json.dumps(report_to_dict(report), indent=2, ensure_ascii=False)
+    path.write_text(payload + "\n", encoding="utf-8")
+
+
+def report_to_markdown(report: Report) -> str:
+    lines = _header_lines(report)
+    lines.extend(["", "## Findings"])
+    if not report.findings:
+        lines.extend(["", "No findings."])
+    for finding in report.findings:
+        lines.extend(_finding_lines(finding))
+    lines.extend(_metrics_lines(report.metrics))
+    lines.extend(_limitation_lines())
+    return "\n".join(lines) + "\n"
+
+
+def _header_lines(report: Report) -> list[str]:
+    lines = [
+        "# kt-audit report",
+        "",
+        f"**Verdict: {report.verdict.upper()}**",
+        "",
+        f"- generated: {report.generated_at}",
+        f"- schema: {report.schema_version}",
+        f"- rows: {report.input.get('rows')}",
+        f"- users: {report.input.get('users')}",
+        f"- items: {report.input.get('items')}",
+    ]
+    splits = report.input.get("splits")
+    if isinstance(splits, dict):
+        rendered = ", ".join(f"{key}={value}" for key, value in sorted(splits.items()))
+        lines.append(f"- splits: {rendered}")
+    return lines
+
+
+def _finding_lines(finding: Finding) -> list[str]:
+    lines = [
+        "",
+        f"### {finding.severity.upper()} · {finding.check}",
+        "",
+        f"{finding.message} (count: {finding.count})",
+    ]
+    if finding.examples:
+        payload = json.dumps(jsonable(finding.examples), indent=2, ensure_ascii=False)
+        lines.extend(["", "```json", payload, "```"])
+    return lines
+
+
+def _metrics_lines(metrics: dict[str, dict[str, object]]) -> list[str]:
+    if not metrics:
+        return []
+    lines = [
+        "",
+        "## Metrics",
+        "",
+        "| Split | n | AUC | Accuracy |",
+        "| --- | ---: | ---: | ---: |",
+    ]
+    for split in sorted(metrics):
+        metric = metrics[split]
+        lines.append(
+            f"| {split} | {metric.get('n')} | {_format_metric(metric.get('auc'))} "
+            f"| {_format_metric(metric.get('acc'))} |"
+        )
+    return lines
+
+
+def _limitation_lines() -> list[str]:
+    return [
+        "",
+        "## Limitations",
+        "",
+        "- Findings are heuristics, not causal proof.",
+        "- The multi-KC check needs `item_id`; it is skipped when missing.",
+        "- The temporal check assumes timestamps are comparable within a user.",
+        "- A passing report does not prove that a model is valid.",
+    ]
+
+
+def _format_metric(value: object) -> str:
+    if value is None:
+        return "n/a"
+    if isinstance(value, float):
+        return f"{value:.4f}"
+    return str(value)

kt_audit/schema.py

@@ -0,0 +1,197 @@
+from __future__ import annotations
+
+import json
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Literal, cast
+
+import numpy as np
+import pandas as pd
+
+from .config import AuditConfig
+from .errors import AuditError
+
+Severity = Literal["error", "warn", "info"]
+Verdict = Literal["pass", "warn", "fail"]
+SPLITS: tuple[str, ...] = ("train", "valid", "test")
+_SPLIT_ALIASES = {
+    "train": "train",
+    "valid": "valid",
+    "validation": "valid",
+    "val": "valid",
+    "test": "test",
+}
+
+
+@dataclass(frozen=True)
+class Finding:
+    check: str
+    severity: Severity
+    message: str
+    count: int = 0
+    examples: tuple[dict[str, object], ...] = ()
+
+
+@dataclass(frozen=True)
+class Report:
+    schema_version: str
+    generated_at: str
+    input: dict[str, object]
+    findings: tuple[Finding, ...]
+    metrics: dict[str, dict[str, object]]
+    verdict: Verdict
+
+
+def _is_missing(value: object) -> bool:
+    if value is None or value is pd.NA:
+        return True
+    if isinstance(value, (list, tuple, set, dict)):
+        return False
+    try:
+        return bool(pd.isna(value))
+    except (TypeError, ValueError):
+        return False
+
+
+def normalize_split(value: object) -> str:
+    if _is_missing(value):
+        raise AuditError("split is missing")
+    text = str(value).strip().lower()
+    if text not in _SPLIT_ALIASES:
+        raise AuditError(f"invalid split: {value!r}")
+    return _SPLIT_ALIASES[text]
+
+
+def parse_kc_ids(value: object, separator: str) -> tuple[str, ...]:
+    if isinstance(value, str):
+        text = value.strip()
+        if text.startswith(("[", "{")):
+            try:
+                parsed = json.loads(text)
+            except json.JSONDecodeError as exc:
+                raise AuditError(f"invalid kc_ids JSON: {value!r}") from exc
+            if not isinstance(parsed, list):
+                raise AuditError(f"kc_ids JSON must be a list: {value!r}")
+            values = parsed
+        else:
+            values = text.split(separator)
+    elif isinstance(value, (list, tuple, set)):
+        values = list(value)
+    elif _is_missing(value):
+        raise AuditError("kc_ids is missing")
+    else:
+        raise AuditError(f"kc_ids must be a string or list: {value!r}")
+    cleaned = tuple(dict.fromkeys(str(item).strip() for item in values if str(item).strip()))
+    if not cleaned:
+        raise AuditError(f"kc_ids is empty: {value!r}")
+    return cleaned
+
+
+def jsonable(value: object) -> object:
+    if isinstance(value, pd.Timestamp):
+        return cast(pd.Timestamp, value).isoformat()
+    if isinstance(value, np.datetime64):
+        return cast(pd.Timestamp, pd.Timestamp(value)).isoformat()
+    if isinstance(value, (list, tuple, set, frozenset)):
+        return [jsonable(item) for item in value]
+    if isinstance(value, dict):
+        return {str(key): jsonable(item) for key, item in value.items()}
+    if isinstance(value, np.integer):
+        return int(value)
+    if isinstance(value, np.floating):
+        return float(value)
+    if isinstance(value, np.bool_):
+        return bool(value)
+    if _is_missing(value):
+        return None
+    return value
+
+
+def example_from_row(row: pd.Series, columns: tuple[str, ...]) -> dict[str, object]:
+    return {column: jsonable(row[column]) for column in columns if column in row.index}
+
+
+def prepare_frame(df: pd.DataFrame, config: AuditConfig) -> pd.DataFrame:
+    if df.empty:
+        raise AuditError("input has no rows")
+    missing = [column for column in ("user_id", "timestamp", "split", "kc_ids") if column not in df]
+    if missing:
+        raise AuditError(f"missing required columns: {', '.join(missing)}")
+
+    out = df.copy().reset_index(drop=True)
+    out["_row"] = np.arange(1, len(out) + 1)
+    out["user_id"] = _to_string(_series(out, "user_id"), "user_id", required=True)
+    out["split"] = _map_rows(_series(out, "split"), normalize_split, "split")
+    out["kc_ids"] = _map_rows(
+        _series(out, "kc_ids"),
+        lambda value: parse_kc_ids(value, config.kc_separator),
+        "kc_ids",
+    )
+    out["timestamp"] = _parse_timestamps(_series(out, "timestamp"))
+
+    if "item_id" in out.columns:
+        out["item_id"] = _to_string(_series(out, "item_id"), "item_id", required=False)
+    else:
+        out["item_id"] = pd.Series(pd.NA, index=out.index, dtype="string")
+
+    if "fold" in out.columns:
+        out["_fold"] = _to_string(_series(out, "fold"), "fold", required=False).fillna("__all__")
+    else:
+        out["_fold"] = "__all__"
+
+    if "sequence_id" in out.columns:
+        out["sequence_id"] = _to_string(
+            _series(out, "sequence_id"),
+            "sequence_id",
+            required=False,
+        )
+    if "position" in out.columns:
+        out["position"] = pd.to_numeric(_series(out, "position"), errors="coerce")
+
+    has_pred = "y_pred" in out.columns
+    has_true = "y_true" in out.columns
+    if has_pred and not has_true:
+        raise AuditError("y_pred requires y_true")
+    if has_true:
+        out["y_true"] = pd.to_numeric(_series(out, "y_true"), errors="coerce")
+    if has_pred:
+        out["y_pred"] = pd.to_numeric(_series(out, "y_pred"), errors="coerce")
+    return out
+
+
+def _series(df: pd.DataFrame, name: str) -> pd.Series:
+    return cast(pd.Series, df[name])
+
+
+def _to_string(series: pd.Series, name: str, *, required: bool) -> pd.Series:
+    if required and bool(series.isna().any()):
+        raise AuditError(f"{name} contains missing values")
+    return series.astype("string")
+
+
+def _map_rows(
+    series: pd.Series,
+    func: Callable[[object], object],
+    name: str,
+) -> pd.Series:
+    values: list[object] = []
+    for row_number, value in enumerate(series, start=1):
+        try:
+            values.append(func(value))
+        except AuditError as exc:
+            raise AuditError(f"row {row_number} column {name}: {exc}") from exc
+    return pd.Series(values, index=series.index, dtype="object")
+
+
+def _parse_timestamps(series: pd.Series) -> pd.Series:
+    if bool(series.isna().any()):
+        raise AuditError("timestamp contains missing values")
+    if pd.api.types.is_numeric_dtype(series):
+        return series.astype("float64")
+    try:
+        parsed = pd.to_datetime(series, errors="raise", utc=True)
+    except (TypeError, ValueError) as exc:
+        raise AuditError(f"invalid timestamp: {exc}") from exc
+    if bool(parsed.isna().any()):
+        raise AuditError("timestamp contains unparseable values")
+    return parsed

kt_audit-0.1.0.dist-info/METADATA

@@ -0,0 +1,169 @@
+Metadata-Version: 2.5
+Name: kt-audit
+Version: 0.1.0
+Summary: Audit knowledge-tracing evaluation before you trust the AUC.
+Project-URL: Homepage, https://github.com/summeriver13/kt-audit
+Project-URL: Repository, https://github.com/summeriver13/kt-audit
+Project-URL: Issues, https://github.com/summeriver13/kt-audit/issues
+Author: Xia Jiang
+License: MIT
+License-File: LICENSE
+Keywords: benchmark,education,evaluation,knowledge-tracing,leakage
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Requires-Dist: numpy>=1.26
+Requires-Dist: pandas>=2.1
+Requires-Dist: scikit-learn>=1.4
+Provides-Extra: dev
+Requires-Dist: basedpyright>=1.17; extra == 'dev'
+Requires-Dist: pytest-cov>=5; extra == 'dev'
+Requires-Dist: pytest>=8; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# kt-audit
+
+**Before you trust an AUC, audit the evaluation.**
+
+`kt-audit` is a small, opinionated audit tool for knowledge-tracing (KT) experiments. It does not train models. It checks whether the evaluation setup behind a reported AUC is trustworthy.
+
+Status: **alpha (0.1.0)**. The scope is intentionally narrow; see [docs/charter.md](docs/charter.md) for the scope and non-goals.
+
+## Why
+
+KT papers and libraries report AUC, but the evaluation protocol is often under-specified:
+
+- Are train events strictly before validation/test events for each user?
+- Does the same item appear in train and test?
+- When an item has multiple knowledge components (KCs), are some KCs train-only and others test-only?
+- Do the reported metrics match a recomputation from the released predictions?
+
+`kt-audit` turns those questions into a machine-readable report you can attach to a paper artifact, a PR, or a model release.
+
+## Install
+
+Not on PyPI yet. From source:
+
+```bash
+git clone https://github.com/summeriver13/kt-audit
+cd kt-audit
+uv sync
+uv run kt-audit --help
+```
+
+Or with pip:
+
+```bash
+pip install -e .
+```
+
+Requires Python 3.10+.
+
+## Quickstart
+
+```bash
+uv run kt-audit run examples/leaky.csv --out /tmp/kt-audit-report
+```
+
+Output:
+
+```text
+verdict: fail
+findings: 6 error, 2 warn
+report: /tmp/kt-audit-report/report.md
+```
+
+The command writes:
+
+- `report.json` — stable, machine-readable.
+- `report.md` — human-readable, with examples and limitations.
+
+Exit codes: `0` pass/warn, `1` fail, `2` input/config error.
+
+## What it checks
+
+| Check | Catches | Default |
+| --- | --- | --- |
+| `temporal_split` | train timestamps after validation/test; users across splits | error / warn |
+| `multi_kc` | same item with train-only and test-only KCs | error |
+| `alignment` | missing/duplicate/out-of-range predictions; constant predictions; reported AUC mismatch | error / warn |
+
+Each finding has a `severity`, a `count`, and up to three examples.
+
+## Input
+
+One row per interaction or prediction. CSV or JSONL.
+
+| Column | Required | Notes |
+| --- | :--: | --- |
+| `user_id` | yes | non-null |
+| `timestamp` | yes | ISO8601 or numeric; comparable within a user |
+| `split` | yes | `train` / `valid` / `test` |
+| `kc_ids` | yes | pipe-separated KCs or JSON array |
+| `item_id` | no | needed for multi-KC check |
+| `y_true` | no | 0/1; required with `y_pred` |
+| `y_pred` | no | [0,1]; omit to skip metric checks |
+| `fold` | no | checks run per fold |
+| `sequence_id`, `position` | no | reserved for sequence checks |
+
+See [docs/data-contract.md](docs/data-contract.md) for the full contract.
+
+## Python API
+
+```python
+from kt_audit import AuditConfig, audit_path
+
+report = audit_path("events.csv", AuditConfig(reported_auc=0.80))
+print(report.verdict)
+for finding in report.findings:
+    print(finding.severity, finding.check, finding.count)
+```
+
+## Design constraints
+
+- Runtime dependencies: `numpy`, `pandas`, `scikit-learn`.
+- CLI uses `argparse`; config uses JSON.
+- Each check is a pure function over a prepared `DataFrame`.
+- Synthetic examples only; no dataset is bundled.
+- Findings are heuristics, not causal proof.
+
+## Development
+
+```bash
+uv sync --extra dev
+uv run ruff check .
+uv run ruff format --check .
+uv run basedpyright
+uv run pytest --cov=kt_audit --cov-report=term-missing
+bash scripts/wheel-smoke.sh
+```
+
+## Release
+
+See [docs/release.md](docs/release.md). TestPyPI and PyPI publish through GitHub Actions trusted publishing.
+
+## Roadmap
+
+- v0.2: pyKT adapter, bootstrap AUC intervals, HTML report.
+- v0.3: duplicate interactions, cold-start, sequence truncation, label leakage.
+
+## AI disclosure
+
+AI tools are used in development. The maintainer reviews every check definition, test, and release. Findings generated by the tool are deterministic and do not call an LLM.
+
+## Contributing
+
+See [CONTRIBUTING.md](CONTRIBUTING.md). The v0.1 scope is frozen to the three checks above.
+
+## License
+
+MIT. See [LICENSE](LICENSE).

kt_audit-0.1.0.dist-info/RECORD

@@ -0,0 +1,19 @@
+kt_audit/__init__.py,sha256=3YMGd4msQtq1hRtjMpw3tK9BXJ1iRWdE0Syh5qNgvxc,315
+kt_audit/__main__.py,sha256=MHKZ_ae3fSLGTLUUMOx15fWdeOnJSHhq-zslRP5F5Lc,79
+kt_audit/audit.py,sha256=NvRXC8ErLxSuiy3GY-erSaR3DgaiJcqgH0OPINcoxTc,2698
+kt_audit/cli.py,sha256=knCQr9Nf_jJOvcs4QzpgmefDc6fEcVCeb16z2vOfaDM,2452
+kt_audit/config.py,sha256=9dMJiPAA2e-xEluK5dNjEaIVY81yyV1OOGUOsrcthtk,1737
+kt_audit/errors.py,sha256=u_CUNg6CxePNPjPe2n9cHvn1x0YrlHUIa8GCcXcWDHo,98
+kt_audit/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+kt_audit/report.py,sha256=J5IH-6hs-Ki6dHSoRbeuvb2ADfZZZeuCUWFezNKz0Pk,3451
+kt_audit/schema.py,sha256=FZqjFfce_W9DlvPb-QwfmzIVoKzmmCnarKsWd0j6NcU,6460
+kt_audit/checks/__init__.py,sha256=wYQlGZn-einHubcbr7TRMFKo04Uw7s3Nyo9mjPX1iOA,177
+kt_audit/checks/_common.py,sha256=_TL9dre3mXK81AQHaHOFtzCwhl9s3IoMimeVZURZbjI,1334
+kt_audit/checks/alignment.py,sha256=rYipLa5-mc2IW6yqre1swU10Chys0e--NUq14I9jBog,5618
+kt_audit/checks/multikc.py,sha256=GaBj4OUfIOA2oJZCfLxFjakCbMblBUb-U7yFDYZt1UM,4533
+kt_audit/checks/temporal.py,sha256=XaJrMqkBE3yWvyB3rOkUnRm0zWoAexOZpsk3ervTR9k,3193
+kt_audit-0.1.0.dist-info/METADATA,sha256=yiqAS2hb-uhIW5v8XXTijwZVwjnFjg9QQ4CJy1JB_pk,5255
+kt_audit-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+kt_audit-0.1.0.dist-info/entry_points.txt,sha256=0mkO5jFlNxsm2QjnHBNKhsWX6j24XToQi7aD3cEUs34,47
+kt_audit-0.1.0.dist-info/licenses/LICENSE,sha256=Es5McTiFUOG7ZXX6wUvNrgnLJBHt6iZHZjhz9PYbx9g,1066
+kt_audit-0.1.0.dist-info/RECORD,,

kt_audit-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

kt_audit-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+kt-audit = kt_audit.cli:main

kt_audit-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Xia Jiang
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.