kt-audit 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. kt_audit-0.1.0/.gitattributes +3 -0
  2. kt_audit-0.1.0/.github/workflows/ci.yml +36 -0
  3. kt_audit-0.1.0/.github/workflows/publish.yml +74 -0
  4. kt_audit-0.1.0/.gitignore +18 -0
  5. kt_audit-0.1.0/CONTRIBUTING.md +38 -0
  6. kt_audit-0.1.0/LICENSE +21 -0
  7. kt_audit-0.1.0/PKG-INFO +169 -0
  8. kt_audit-0.1.0/README.md +137 -0
  9. kt_audit-0.1.0/docs/charter.md +136 -0
  10. kt_audit-0.1.0/docs/data-contract.md +60 -0
  11. kt_audit-0.1.0/docs/release.md +53 -0
  12. kt_audit-0.1.0/examples/audit.json +4 -0
  13. kt_audit-0.1.0/examples/leaky.csv +11 -0
  14. kt_audit-0.1.0/pyproject.toml +73 -0
  15. kt_audit-0.1.0/scripts/wheel-smoke.sh +42 -0
  16. kt_audit-0.1.0/src/kt_audit/__init__.py +16 -0
  17. kt_audit-0.1.0/src/kt_audit/__main__.py +4 -0
  18. kt_audit-0.1.0/src/kt_audit/audit.py +81 -0
  19. kt_audit-0.1.0/src/kt_audit/checks/__init__.py +5 -0
  20. kt_audit-0.1.0/src/kt_audit/checks/_common.py +45 -0
  21. kt_audit-0.1.0/src/kt_audit/checks/alignment.py +179 -0
  22. kt_audit-0.1.0/src/kt_audit/checks/multikc.py +160 -0
  23. kt_audit-0.1.0/src/kt_audit/checks/temporal.py +106 -0
  24. kt_audit-0.1.0/src/kt_audit/cli.py +61 -0
  25. kt_audit-0.1.0/src/kt_audit/config.py +51 -0
  26. kt_audit-0.1.0/src/kt_audit/errors.py +2 -0
  27. kt_audit-0.1.0/src/kt_audit/py.typed +0 -0
  28. kt_audit-0.1.0/src/kt_audit/report.py +115 -0
  29. kt_audit-0.1.0/src/kt_audit/schema.py +197 -0
  30. kt_audit-0.1.0/tests/conftest.py +60 -0
  31. kt_audit-0.1.0/tests/test_alignment.py +89 -0
  32. kt_audit-0.1.0/tests/test_audit.py +86 -0
  33. kt_audit-0.1.0/tests/test_cli.py +68 -0
  34. kt_audit-0.1.0/tests/test_multikc.py +63 -0
  35. kt_audit-0.1.0/tests/test_public_api.py +99 -0
  36. kt_audit-0.1.0/tests/test_report.py +45 -0
  37. kt_audit-0.1.0/tests/test_schema.py +202 -0
  38. kt_audit-0.1.0/tests/test_temporal.py +61 -0
@@ -0,0 +1,3 @@
1
+ * text=auto eol=lf
2
+ *.sh text eol=lf
3
+ *.bat text eol=crlf
@@ -0,0 +1,36 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ pull_request:
6
+
7
+ jobs:
8
+ test:
9
+ runs-on: ubuntu-latest
10
+ strategy:
11
+ fail-fast: false
12
+ matrix:
13
+ python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
14
+ steps:
15
+ - uses: actions/checkout@v4
16
+ - uses: astral-sh/setup-uv@v6
17
+ - name: Install
18
+ run: uv sync --extra dev --python ${{ matrix.python-version }}
19
+ - name: Lint
20
+ run: |
21
+ uv run ruff check .
22
+ uv run ruff format --check .
23
+ - name: Type check
24
+ run: uv run basedpyright
25
+ - name: Test
26
+ run: uv run pytest --cov=kt_audit --cov-report=term-missing --cov-fail-under=85
27
+ - name: Build
28
+ run: uv build
29
+
30
+ wheel-smoke:
31
+ runs-on: ubuntu-latest
32
+ steps:
33
+ - uses: actions/checkout@v4
34
+ - uses: astral-sh/setup-uv@v6
35
+ - name: Wheel smoke
36
+ run: bash scripts/wheel-smoke.sh
@@ -0,0 +1,74 @@
1
+ name: Publish
2
+
3
+ on:
4
+ push:
5
+ tags:
6
+ - "v[0-9]+.[0-9]+.[0-9]+"
7
+ workflow_dispatch:
8
+
9
+ permissions:
10
+ contents: read
11
+
12
+ jobs:
13
+ build:
14
+ runs-on: ubuntu-latest
15
+ steps:
16
+ - uses: actions/checkout@v4
17
+ with:
18
+ persist-credentials: false
19
+ - uses: astral-sh/setup-uv@v6
20
+ with:
21
+ enable-cache: false
22
+ - name: Build
23
+ run: uv build
24
+ - name: Check tag matches version
25
+ if: startsWith(github.ref, 'refs/tags/v')
26
+ run: |
27
+ version=$(uv run python -c "import kt_audit; print(kt_audit.__version__)")
28
+ test "v$version" = "$GITHUB_REF_NAME"
29
+ - uses: actions/upload-artifact@v4
30
+ with:
31
+ name: dist
32
+ path: dist/
33
+
34
+ publish-testpypi:
35
+ if: github.event_name == 'workflow_dispatch'
36
+ needs: build
37
+ runs-on: ubuntu-latest
38
+ environment:
39
+ name: testpypi
40
+ url: https://test.pypi.org/p/kt-audit
41
+ permissions:
42
+ contents: read
43
+ id-token: write
44
+ steps:
45
+ - uses: astral-sh/setup-uv@v6
46
+ with:
47
+ enable-cache: false
48
+ - uses: actions/download-artifact@v4
49
+ with:
50
+ name: dist
51
+ path: dist/
52
+ - name: Publish to TestPyPI
53
+ run: uv publish --publish-url https://test.pypi.org/legacy/ --trusted-publishing always
54
+
55
+ publish-pypi:
56
+ if: github.event_name == 'push'
57
+ needs: build
58
+ runs-on: ubuntu-latest
59
+ environment:
60
+ name: pypi
61
+ url: https://pypi.org/p/kt-audit
62
+ permissions:
63
+ contents: read
64
+ id-token: write
65
+ steps:
66
+ - uses: astral-sh/setup-uv@v6
67
+ with:
68
+ enable-cache: false
69
+ - uses: actions/download-artifact@v4
70
+ with:
71
+ name: dist
72
+ path: dist/
73
+ - name: Publish to PyPI
74
+ run: uv publish --trusted-publishing always
@@ -0,0 +1,18 @@
1
+ .venv/
2
+ __pycache__/
3
+ *.py[cod]
4
+ .pytest_cache/
5
+ .ruff_cache/
6
+ .basedpyright/
7
+ .coverage
8
+ htmlcov/
9
+ dist/
10
+ build/
11
+ *.egg-info/
12
+ kt-audit-report/
13
+
14
+ # local uv lock (regenerated per environment)
15
+ uv.lock
16
+
17
+ # local scratch
18
+ .tmp/
@@ -0,0 +1,38 @@
1
+ # Contributing
2
+
3
+ `kt-audit` is intentionally small. The v0.1 scope is frozen to three checks:
4
+
5
+ - `temporal_split`
6
+ - `multi_kc`
7
+ - `alignment`
8
+
9
+ A new check needs an issue with a concrete failure mode and a minimal example first. Please do not open a large refactor or a new framework PR.
10
+
11
+ ## Development
12
+
13
+ ```bash
14
+ uv sync --extra dev
15
+ uv run pytest --cov=kt_audit --cov-report=term-missing
16
+ uv run ruff check .
17
+ uv run ruff format --check .
18
+ uv run basedpyright
19
+ ```
20
+
21
+ ## Rules
22
+
23
+ - Keep runtime dependencies to `numpy`, `pandas`, and `scikit-learn`.
24
+ - Add a test for every new branch. Synthetic data only; do not commit restricted datasets.
25
+ - Every finding must have a severity, a count, and bounded examples.
26
+ - Document limitations next to the check, not only in the README.
27
+ - Disclose AI assistance in the PR body. The maintainer reviews every check definition and test.
28
+
29
+ ## Reporting a false positive
30
+
31
+ Open an issue with:
32
+
33
+ 1. the command,
34
+ 2. the input schema (not the full private dataset),
35
+ 3. the actual finding,
36
+ 4. why you think it is wrong.
37
+
38
+ A reproducible counterexample is more useful than a general complaint.
kt_audit-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Xia Jiang
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,169 @@
1
+ Metadata-Version: 2.5
2
+ Name: kt-audit
3
+ Version: 0.1.0
4
+ Summary: Audit knowledge-tracing evaluation before you trust the AUC.
5
+ Project-URL: Homepage, https://github.com/summeriver13/kt-audit
6
+ Project-URL: Repository, https://github.com/summeriver13/kt-audit
7
+ Project-URL: Issues, https://github.com/summeriver13/kt-audit/issues
8
+ Author: Xia Jiang
9
+ License: MIT
10
+ License-File: LICENSE
11
+ Keywords: benchmark,education,evaluation,knowledge-tracing,leakage
12
+ Classifier: Development Status :: 3 - Alpha
13
+ Classifier: Intended Audience :: Science/Research
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.10
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Programming Language :: Python :: 3.13
20
+ Classifier: Programming Language :: Python :: 3.14
21
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
22
+ Requires-Python: >=3.10
23
+ Requires-Dist: numpy>=1.26
24
+ Requires-Dist: pandas>=2.1
25
+ Requires-Dist: scikit-learn>=1.4
26
+ Provides-Extra: dev
27
+ Requires-Dist: basedpyright>=1.17; extra == 'dev'
28
+ Requires-Dist: pytest-cov>=5; extra == 'dev'
29
+ Requires-Dist: pytest>=8; extra == 'dev'
30
+ Requires-Dist: ruff>=0.6; extra == 'dev'
31
+ Description-Content-Type: text/markdown
32
+
33
+ # kt-audit
34
+
35
+ **Before you trust an AUC, audit the evaluation.**
36
+
37
+ `kt-audit` is a small, opinionated audit tool for knowledge-tracing (KT) experiments. It does not train models. It checks whether the evaluation setup behind a reported AUC is trustworthy.
38
+
39
+ Status: **alpha (0.1.0)**. The scope is intentionally narrow; see [docs/charter.md](docs/charter.md) for the scope and non-goals.
40
+
41
+ ## Why
42
+
43
+ KT papers and libraries report AUC, but the evaluation protocol is often under-specified:
44
+
45
+ - Are train events strictly before validation/test events for each user?
46
+ - Does the same item appear in train and test?
47
+ - When an item has multiple knowledge components (KCs), are some KCs train-only and others test-only?
48
+ - Do the reported metrics match a recomputation from the released predictions?
49
+
50
+ `kt-audit` turns those questions into a machine-readable report you can attach to a paper artifact, a PR, or a model release.
51
+
52
+ ## Install
53
+
54
+ Not on PyPI yet. From source:
55
+
56
+ ```bash
57
+ git clone https://github.com/summeriver13/kt-audit
58
+ cd kt-audit
59
+ uv sync
60
+ uv run kt-audit --help
61
+ ```
62
+
63
+ Or with pip:
64
+
65
+ ```bash
66
+ pip install -e .
67
+ ```
68
+
69
+ Requires Python 3.10+.
70
+
71
+ ## Quickstart
72
+
73
+ ```bash
74
+ uv run kt-audit run examples/leaky.csv --out /tmp/kt-audit-report
75
+ ```
76
+
77
+ Output:
78
+
79
+ ```text
80
+ verdict: fail
81
+ findings: 6 error, 2 warn
82
+ report: /tmp/kt-audit-report/report.md
83
+ ```
84
+
85
+ The command writes:
86
+
87
+ - `report.json` — stable, machine-readable.
88
+ - `report.md` — human-readable, with examples and limitations.
89
+
90
+ Exit codes: `0` pass/warn, `1` fail, `2` input/config error.
91
+
92
+ ## What it checks
93
+
94
+ | Check | Catches | Default |
95
+ | --- | --- | --- |
96
+ | `temporal_split` | train timestamps after validation/test; users across splits | error / warn |
97
+ | `multi_kc` | same item with train-only and test-only KCs | error |
98
+ | `alignment` | missing/duplicate/out-of-range predictions; constant predictions; reported AUC mismatch | error / warn |
99
+
100
+ Each finding has a `severity`, a `count`, and up to three examples.
101
+
102
+ ## Input
103
+
104
+ One row per interaction or prediction. CSV or JSONL.
105
+
106
+ | Column | Required | Notes |
107
+ | --- | :--: | --- |
108
+ | `user_id` | yes | non-null |
109
+ | `timestamp` | yes | ISO8601 or numeric; comparable within a user |
110
+ | `split` | yes | `train` / `valid` / `test` |
111
+ | `kc_ids` | yes | pipe-separated KCs or JSON array |
112
+ | `item_id` | no | needed for multi-KC check |
113
+ | `y_true` | no | 0/1; required with `y_pred` |
114
+ | `y_pred` | no | [0,1]; omit to skip metric checks |
115
+ | `fold` | no | checks run per fold |
116
+ | `sequence_id`, `position` | no | reserved for sequence checks |
117
+
118
+ See [docs/data-contract.md](docs/data-contract.md) for the full contract.
119
+
120
+ ## Python API
121
+
122
+ ```python
123
+ from kt_audit import AuditConfig, audit_path
124
+
125
+ report = audit_path("events.csv", AuditConfig(reported_auc=0.80))
126
+ print(report.verdict)
127
+ for finding in report.findings:
128
+ print(finding.severity, finding.check, finding.count)
129
+ ```
130
+
131
+ ## Design constraints
132
+
133
+ - Runtime dependencies: `numpy`, `pandas`, `scikit-learn`.
134
+ - CLI uses `argparse`; config uses JSON.
135
+ - Each check is a pure function over a prepared `DataFrame`.
136
+ - Synthetic examples only; no dataset is bundled.
137
+ - Findings are heuristics, not causal proof.
138
+
139
+ ## Development
140
+
141
+ ```bash
142
+ uv sync --extra dev
143
+ uv run ruff check .
144
+ uv run ruff format --check .
145
+ uv run basedpyright
146
+ uv run pytest --cov=kt_audit --cov-report=term-missing
147
+ bash scripts/wheel-smoke.sh
148
+ ```
149
+
150
+ ## Release
151
+
152
+ See [docs/release.md](docs/release.md). TestPyPI and PyPI publish through GitHub Actions trusted publishing.
153
+
154
+ ## Roadmap
155
+
156
+ - v0.2: pyKT adapter, bootstrap AUC intervals, HTML report.
157
+ - v0.3: duplicate interactions, cold-start, sequence truncation, label leakage.
158
+
159
+ ## AI disclosure
160
+
161
+ AI tools are used in development. The maintainer reviews every check definition, test, and release. Findings generated by the tool are deterministic and do not call an LLM.
162
+
163
+ ## Contributing
164
+
165
+ See [CONTRIBUTING.md](CONTRIBUTING.md). The v0.1 scope is frozen to the three checks above.
166
+
167
+ ## License
168
+
169
+ MIT. See [LICENSE](LICENSE).
@@ -0,0 +1,137 @@
1
+ # kt-audit
2
+
3
+ **Before you trust an AUC, audit the evaluation.**
4
+
5
+ `kt-audit` is a small, opinionated audit tool for knowledge-tracing (KT) experiments. It does not train models. It checks whether the evaluation setup behind a reported AUC is trustworthy.
6
+
7
+ Status: **alpha (0.1.0)**. The scope is intentionally narrow; see [docs/charter.md](docs/charter.md) for the scope and non-goals.
8
+
9
+ ## Why
10
+
11
+ KT papers and libraries report AUC, but the evaluation protocol is often under-specified:
12
+
13
+ - Are train events strictly before validation/test events for each user?
14
+ - Does the same item appear in train and test?
15
+ - When an item has multiple knowledge components (KCs), are some KCs train-only and others test-only?
16
+ - Do the reported metrics match a recomputation from the released predictions?
17
+
18
+ `kt-audit` turns those questions into a machine-readable report you can attach to a paper artifact, a PR, or a model release.
19
+
20
+ ## Install
21
+
22
+ Not on PyPI yet. From source:
23
+
24
+ ```bash
25
+ git clone https://github.com/summeriver13/kt-audit
26
+ cd kt-audit
27
+ uv sync
28
+ uv run kt-audit --help
29
+ ```
30
+
31
+ Or with pip:
32
+
33
+ ```bash
34
+ pip install -e .
35
+ ```
36
+
37
+ Requires Python 3.10+.
38
+
39
+ ## Quickstart
40
+
41
+ ```bash
42
+ uv run kt-audit run examples/leaky.csv --out /tmp/kt-audit-report
43
+ ```
44
+
45
+ Output:
46
+
47
+ ```text
48
+ verdict: fail
49
+ findings: 6 error, 2 warn
50
+ report: /tmp/kt-audit-report/report.md
51
+ ```
52
+
53
+ The command writes:
54
+
55
+ - `report.json` — stable, machine-readable.
56
+ - `report.md` — human-readable, with examples and limitations.
57
+
58
+ Exit codes: `0` pass/warn, `1` fail, `2` input/config error.
59
+
60
+ ## What it checks
61
+
62
+ | Check | Catches | Default |
63
+ | --- | --- | --- |
64
+ | `temporal_split` | train timestamps after validation/test; users across splits | error / warn |
65
+ | `multi_kc` | same item with train-only and test-only KCs | error |
66
+ | `alignment` | missing/duplicate/out-of-range predictions; constant predictions; reported AUC mismatch | error / warn |
67
+
68
+ Each finding has a `severity`, a `count`, and up to three examples.
69
+
70
+ ## Input
71
+
72
+ One row per interaction or prediction. CSV or JSONL.
73
+
74
+ | Column | Required | Notes |
75
+ | --- | :--: | --- |
76
+ | `user_id` | yes | non-null |
77
+ | `timestamp` | yes | ISO8601 or numeric; comparable within a user |
78
+ | `split` | yes | `train` / `valid` / `test` |
79
+ | `kc_ids` | yes | pipe-separated KCs or JSON array |
80
+ | `item_id` | no | needed for multi-KC check |
81
+ | `y_true` | no | 0/1; required with `y_pred` |
82
+ | `y_pred` | no | [0,1]; omit to skip metric checks |
83
+ | `fold` | no | checks run per fold |
84
+ | `sequence_id`, `position` | no | reserved for sequence checks |
85
+
86
+ See [docs/data-contract.md](docs/data-contract.md) for the full contract.
87
+
88
+ ## Python API
89
+
90
+ ```python
91
+ from kt_audit import AuditConfig, audit_path
92
+
93
+ report = audit_path("events.csv", AuditConfig(reported_auc=0.80))
94
+ print(report.verdict)
95
+ for finding in report.findings:
96
+ print(finding.severity, finding.check, finding.count)
97
+ ```
98
+
99
+ ## Design constraints
100
+
101
+ - Runtime dependencies: `numpy`, `pandas`, `scikit-learn`.
102
+ - CLI uses `argparse`; config uses JSON.
103
+ - Each check is a pure function over a prepared `DataFrame`.
104
+ - Synthetic examples only; no dataset is bundled.
105
+ - Findings are heuristics, not causal proof.
106
+
107
+ ## Development
108
+
109
+ ```bash
110
+ uv sync --extra dev
111
+ uv run ruff check .
112
+ uv run ruff format --check .
113
+ uv run basedpyright
114
+ uv run pytest --cov=kt_audit --cov-report=term-missing
115
+ bash scripts/wheel-smoke.sh
116
+ ```
117
+
118
+ ## Release
119
+
120
+ See [docs/release.md](docs/release.md). TestPyPI and PyPI publish through GitHub Actions trusted publishing.
121
+
122
+ ## Roadmap
123
+
124
+ - v0.2: pyKT adapter, bootstrap AUC intervals, HTML report.
125
+ - v0.3: duplicate interactions, cold-start, sequence truncation, label leakage.
126
+
127
+ ## AI disclosure
128
+
129
+ AI tools are used in development. The maintainer reviews every check definition, test, and release. Findings generated by the tool are deterministic and do not call an LLM.
130
+
131
+ ## Contributing
132
+
133
+ See [CONTRIBUTING.md](CONTRIBUTING.md). The v0.1 scope is frozen to the three checks above.
134
+
135
+ ## License
136
+
137
+ MIT. See [LICENSE](LICENSE).
@@ -0,0 +1,136 @@
1
+ # kt-audit charter
2
+
3
+ > 状态:draft,2026-10-20。目标:4 周 MVP,每周 2-4 小时。
4
+
5
+ ## 一句话
6
+
7
+ **Before you trust an AUC, audit the evaluation.**
8
+
9
+ ## 问题
10
+
11
+ 知识追踪(KT)论文和库都在报 AUC,但评估协议不一致:
12
+
13
+ - split 是否按时间?用户是否跨 split?
14
+ - 一道题有多个知识点时,训练集和测试集是否共享了同一道题的不同 KC?
15
+ - question-level 与 KC-level 的预测是否被混用?
16
+ - 报告的 AUC 和用公开预测重算的 AUC 是否一致?
17
+
18
+ pyKT 是事实上的 KT benchmark,但无 CI、几乎无测试、外部 PR 通道已冷。研究者需要一个**轻量、可机器读取、能放进论文 artifact** 的审计工具。
19
+
20
+ ## 用户
21
+
22
+ - 准备投稿或放 artifact 的 KT 研究者。
23
+ - 接 KT 模型做产品原型的工程师。
24
+ - 复现实验、审稿、带学生的研究生。
25
+
26
+ ## 非目标
27
+
28
+ - 不训练 DKT/AKT,不做模型库。
29
+ - 不做 leaderboard,不托管数据集。
30
+ - 不证明因果,只标记可疑模式。
31
+ - v0.1 不做 LLM-KT 评估。
32
+ - 不替代人工判断;报告里的 finding 是证据,不是判决。
33
+
34
+ ## MVP:三个检查
35
+
36
+ | 检查 | 问题 | 默认严重度 |
37
+ | --- | --- | --- |
38
+ | temporal split | 同一用户的 train 事件是否晚于 valid/test?用户是否跨 split? | error / warn |
39
+ | multi-KC leakage | 同一道题是否同时含有 train-only 和 test-only 的 KC? | error |
40
+ | prediction alignment | 是否有缺失、重复、越界、常量预测?重算 AUC 是否与报告一致? | error / warn |
41
+
42
+ ### 1. temporal split
43
+
44
+ - 同一 user 的 `max(train timestamp) >= min(valid/test timestamp)`:error。
45
+ - 同一 user 同时出现在多个 split:warn(可能是随机 split,也可能泄漏)。
46
+ - 全局 split 时间区间重叠:warn。
47
+ - 没有 `timestamp`:跳过并 warn。
48
+
49
+ ### 2. multi-KC leakage
50
+
51
+ - 构建 `item_id -> set(kc_ids)`。
52
+ - 同一道题的 KC 集合里,既有只在 train 出现的 KC,又有只在 test 出现的 KC:error。
53
+ - 同一道题同时出现在 train 和 test:warn。
54
+ - 没有 `item_id`:跳过并 warn。
55
+ - 这是启发式,不是因果证明;报告必须写清。
56
+
57
+ ### 3. prediction alignment
58
+
59
+ - `y_pred` 为 NaN/inf/越界:error。
60
+ - 主键重复(`user_id + item_id + timestamp`):error。
61
+ - 预测常量(标准差为 0):warn。
62
+ - 可选 `reported_auc`:与重算 AUC 差值超过 tolerance:error。
63
+ - 没有 `y_pred`:跳过指标检查。
64
+
65
+ ## 输入契约
66
+
67
+ 一行一个交互/预测。CSV 或 JSONL。
68
+
69
+ | 列 | 必填 | 说明 |
70
+ | --- | :--: | --- |
71
+ | `user_id` | 是 | 用户 ID |
72
+ | `timestamp` | 是 | ISO8601 或数值;同一用户内可比较 |
73
+ | `split` | 是 | `train` / `valid` / `test` |
74
+ | `kc_ids` | 是 | 管道分隔或 JSON 数组 |
75
+ | `item_id` | 否 | 题目 ID;缺失时跳过 multi-KC 检查 |
76
+ | `y_true` | 否 | 0/1;有 `y_pred` 时必填 |
77
+ | `y_pred` | 否 | [0,1];缺失时跳过指标检查 |
78
+ | `fold` | 否 | 交叉验证折 |
79
+ | `sequence_id` | 否 | 序列 ID |
80
+ | `position` | 否 | 序列内位置 |
81
+
82
+ 完整说明见 `docs/data-contract.md`。
83
+
84
+ ## 输出
85
+
86
+ `kt-audit run events.csv --out report/` 生成:
87
+
88
+ - `report.json`:机器可读,带 `schema_version`、`findings`、`metrics`、`verdict`。
89
+ - `report.md`:人读,先给 verdict,再给证据与限制。
90
+
91
+ 退出码:
92
+
93
+ - `0`:pass 或 warn。
94
+ - `1`:fail(至少一个 error)。
95
+ - `2`:输入/配置错误。
96
+
97
+ ## 设计约束
98
+
99
+ - 运行依赖只加 `numpy`、`pandas`、`scikit-learn`。
100
+ - CLI 用标准库 `argparse`,不引入 Typer/Click。
101
+ - 配置用 JSON,不引入 YAML/TOML 解析器。
102
+ - `src/` 布局;包名 `kt_audit`,命令 `kt-audit`。
103
+ - 每个 check 是纯函数:输入 DataFrame + config,输出 Finding 列表。
104
+ - 报告排序稳定,示例截断,避免大文件。
105
+ - 对外 README/文档英文;内部 charter/notes 中文。
106
+
107
+ ## 4 周成功标准
108
+
109
+ - `uv run pytest` 绿,三个 check 都有正反例。
110
+ - `kt-audit run examples/leaky.csv --out /tmp/report` 能生成 JSON + Markdown。
111
+ - 示例报告能抓到三类故意泄漏。
112
+ - GitHub Actions 在 Python 3.10-3.13 上绿。
113
+ - 一个陌生人按 README 能在 5 分钟内跑出报告。
114
+
115
+ **不以下载量、stars、外部 PR 数作为早期成功标准。**
116
+
117
+ ## 后续(v0.2+)
118
+
119
+ - pyKT 输出 adapter。
120
+ - bootstrap AUC 置信区间。
121
+ - HTML 报告。
122
+ - 更多检查:重复交互、冷启动、序列截断、标签泄漏。
123
+ - 真实数据集 case study(注意授权)。
124
+
125
+ ## 风险
126
+
127
+ - 启发式会有假阳性;必须提供 examples 和 limitations。
128
+ - 容易膨胀成“又一个 KT 框架”;v0.1 冻结在三个检查。
129
+ - 数据集不能随便打包;示例用合成数据。
130
+ - 维护时间有限;每周 2-4 小时,不承诺 SLA。
131
+
132
+ ## 许可与治理
133
+
134
+ - MIT。
135
+ - 初始单维护者;PR 欢迎,但不承诺响应时间。
136
+ - AI 可参与开发,但每条 finding 的定义和测试必须有人复核。
@@ -0,0 +1,60 @@
1
+ # Data contract
2
+
3
+ `kt-audit` 接受 **一行一个交互/预测** 的宽表。CSV(UTF-8)或 JSONL;Python API 直接收 `pandas.DataFrame`。
4
+
5
+ ## 列
6
+
7
+ | 列 | 类型 | 必填 | 约束 |
8
+ | --- | --- | :--: | --- |
9
+ | `user_id` | string/int | 是 | 非空;同一用户内 timestamp 可比较 |
10
+ | `timestamp` | ISO8601 / 数值 | 是 | 可解析;不要混用 naive 与 timezone-aware |
11
+ | `split` | string | 是 | `train` / `valid` / `test`;`val`、`validation` 会归一为 `valid` |
12
+ | `kc_ids` | string/list | 是 | 管道分隔或 `["kc_a","kc_b"]`;至少一个 |
13
+ | `item_id` | string/int | 否 | 缺失时跳过 multi-KC 检查 |
14
+ | `y_true` | 0/1 | 否 | 有 `y_pred` 时必填;不能是 NaN |
15
+ | `y_pred` | float | 否 | 有限值,且 `0 <= y_pred <= 1` |
16
+ | `fold` | int/string | 否 | 有 fold 时按 fold 分别跑检查 |
17
+ | `sequence_id` | string/int | 否 | 用于序列级检查(v0.2) |
18
+ | `position` | int | 否 | 序列内位置(v0.2) |
19
+
20
+ ## 解析规则
21
+
22
+ - `split`:转小写并 strip;`val` / `validation` -> `valid`;其他值报错。
23
+ - `kc_ids`:以 `[` 开头按 JSON 数组解析;否则按 `|` 分割、strip、去空。
24
+ - `timestamp`:优先 `pandas.to_datetime`;数值保持数值。若解析后既有 tz-aware 又有 naive,报错。
25
+ - `user_id` / `item_id`:统一转字符串,保留原始值用于报告。
26
+ - `y_true`:只接受 0/1(含 `0.0`/`1.0`)。
27
+ - `y_pred`:必须是有限浮点,范围 [0,1]。
28
+
29
+ ## 主键
30
+
31
+ 按可用列依次选择:
32
+
33
+ 1. `(fold, user_id, item_id, timestamp)`
34
+ 2. `(fold, user_id, timestamp, sequence_id, position)`
35
+ 3. `(fold, user_id, timestamp)`
36
+
37
+ 主键重复记一条 `alignment` error。
38
+
39
+ ## 最小示例
40
+
41
+ ```csv
42
+ user_id,item_id,kc_ids,timestamp,split,y_true,y_pred
43
+ u1,q1,kc_a|kc_b,2024-01-01T10:00:00,train,1,0.80
44
+ u1,q2,kc_b,2024-01-01T10:05:00,train,0,0.30
45
+ u2,q1,kc_a|kc_b,2024-01-02T09:00:00,test,1,0.75
46
+ u2,q3,kc_c,2024-01-02T09:10:00,test,0,0.20
47
+ ```
48
+
49
+ 这个例子里 `q1` 同时出现在 train/test,且 `kc_a` 只在 train、`kc_b` 在两边都有——multi-KC 检查会给出 warn/error,具体看配置。
50
+
51
+ ## pyKT 映射(v0.2)
52
+
53
+ pyKT 的列名不同(`uid`、`questions`、`concepts` 等)。v0.1 不内置 adapter;示例用合成数据,避免数据集授权问题。v0.2 再加 `--adapter pykt`。
54
+
55
+ ## 限制
56
+
57
+ - 时间戳只用于 split 泄漏检查,不推断因果。
58
+ - 没有 `item_id` 时无法检查题目级多 KC 泄漏。
59
+ - 没有 `y_pred` 时只做数据/泄漏检查,不算指标。
60
+ - 跨 fold 的重复不一定是错;默认按 fold 隔离检查。