assessment-lens 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- assessment_lens-0.2.0/.gitignore +10 -0
- assessment_lens-0.2.0/CLAUDE.md +52 -0
- assessment_lens-0.2.0/LICENSE +21 -0
- assessment_lens-0.2.0/PKG-INFO +162 -0
- assessment_lens-0.2.0/README.md +130 -0
- assessment_lens-0.2.0/docs/SCOPING.md +34 -0
- assessment_lens-0.2.0/examples/data-viz-rubric.yaml +23 -0
- assessment_lens-0.2.0/pyproject.toml +72 -0
- assessment_lens-0.2.0/src/assessment_lens/__init__.py +35 -0
- assessment_lens-0.2.0/src/assessment_lens/alignment.py +145 -0
- assessment_lens-0.2.0/src/assessment_lens/assess.py +109 -0
- assessment_lens-0.2.0/src/assessment_lens/bundle.py +108 -0
- assessment_lens-0.2.0/src/assessment_lens/cli.py +145 -0
- assessment_lens-0.2.0/src/assessment_lens/draft_rubric.py +151 -0
- assessment_lens-0.2.0/src/assessment_lens/exceptions.py +19 -0
- assessment_lens-0.2.0/src/assessment_lens/llm.py +92 -0
- assessment_lens-0.2.0/src/assessment_lens/models.py +139 -0
- assessment_lens-0.2.0/src/assessment_lens/report.py +94 -0
- assessment_lens-0.2.0/src/assessment_lens/rubric.py +111 -0
- assessment_lens-0.2.0/tests/__init__.py +0 -0
- assessment_lens-0.2.0/tests/test_alignment.py +89 -0
- assessment_lens-0.2.0/tests/test_cli.py +143 -0
- assessment_lens-0.2.0/tests/test_draft_rubric.py +116 -0
- assessment_lens-0.2.0/tests/test_models.py +58 -0
- assessment_lens-0.2.0/tests/test_rubric.py +74 -0
- assessment_lens-0.2.0/uv.lock +2401 -0
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# CLAUDE.md
|
|
2
|
+
|
|
3
|
+
Guidance for Claude Code when working in this repository.
|
|
4
|
+
|
|
5
|
+
## What this is
|
|
6
|
+
|
|
7
|
+
`assessment-lens` is a **lens** — an assessment-aware *product*, not an
|
|
8
|
+
`-analyser`. Analysers are assessment-agnostic signal generators; lenses sit
|
|
9
|
+
above them and consume their signals. This one maps signals to a rubric as
|
|
10
|
+
**observations, not grades**. The core invariant, enforced in the models:
|
|
11
|
+
|
|
12
|
+
> **The LLM narrates and cites; it never scores.** There is no score/mark/grade/
|
|
13
|
+
> weight field anywhere in `models.py`. A human assigns every mark. If you ever
|
|
14
|
+
> find yourself adding one, stop — that belongs to the lecturer, not the lens.
|
|
15
|
+
|
|
16
|
+
Because it's a lens, there is **no** `lens-contract` / FastAPI / `manifest` /
|
|
17
|
+
`serve` here (those are the *analyser* contract). It's a plain CLI consumer that
|
|
18
|
+
composes **only `bundle-analyser`** (one call per submission subfolder).
|
|
19
|
+
|
|
20
|
+
## Architecture
|
|
21
|
+
|
|
22
|
+
```
|
|
23
|
+
cli.py → argparse: `assess`, `draft-rubric`
|
|
24
|
+
rubric.py → load/validate the structured rubric (YAML/JSON) + ext→content-kind
|
|
25
|
+
assess.py → orchestration: discover submissions, reconcile deliverables, run pipeline
|
|
26
|
+
bundle.py → subprocess wrapper over the bundle-analyser CLI + signal-path resolver
|
|
27
|
+
alignment.py → alignment-check: signals → Observations (evidence + threshold coverage); LLM `narrate` opt-in via --llm
|
|
28
|
+
llm.py → Anthropic plumbing (key resolution, model defaults, `complete`); everything degradable
|
|
29
|
+
report.py → cohort sheet (CSV) + per-student reports (Markdown)
|
|
30
|
+
draft_rubric.py → free-form spec → proposed rubric (LLM-assisted; lecturer reviews before use)
|
|
31
|
+
models.py → the central contract (Rubric/Criterion/Deliverable, Observation/Coverage)
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
The `bundle.run_bundle` / `bundle.get_signal` integration is pinned to
|
|
35
|
+
bundle-analyser's real output schema (verified against the live CLI) — re-verify
|
|
36
|
+
if bundle-analyser's aggregate shape changes.
|
|
37
|
+
|
|
38
|
+
## Toolchain (family standard — run before committing)
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
ruff format . && ruff check . && pytest -v
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
`uv` for env/packaging, `pyproject.toml` only. Tests mirror `src/`.
|
|
45
|
+
|
|
46
|
+
## Provenance
|
|
47
|
+
|
|
48
|
+
Scoping lives in `lens-analysers/docs/ASSESSMENT-LENS-SCOPING.md` and
|
|
49
|
+
[ADR-0001](https://github.com/michael-borck/lens-analysers/blob/main/docs/adr/0001-alignment-lives-in-assessment-lens.md).
|
|
50
|
+
The rubric/storage/LLM-key code is being harvested+adapted from `video-analyser`
|
|
51
|
+
@ `b3ae7e19` (pre-grading-removal) — see `draft_rubric.py` and the scoping doc's
|
|
52
|
+
harvest map. Adapt, don't copy: the old code *scored*; this lens *narrates*.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Michael Borck
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: assessment-lens
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Signals-based assessment — map analyser signals to a rubric as observations, never grades. A human assigns every mark.
|
|
5
|
+
Project-URL: Homepage, https://github.com/michael-borck/assessment-lens
|
|
6
|
+
Project-URL: Repository, https://github.com/michael-borck/assessment-lens
|
|
7
|
+
Project-URL: Issues, https://github.com/michael-borck/assessment-lens/issues
|
|
8
|
+
Author-email: Michael Borck <michael.borck@curtin.edu.au>
|
|
9
|
+
License: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: assessment,edtech,education,feedback,lens,rubric,signals
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Education
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Requires-Python: >=3.11
|
|
19
|
+
Requires-Dist: pydantic>=2.5.0
|
|
20
|
+
Requires-Dist: pyyaml>=6.0
|
|
21
|
+
Requires-Dist: rich>=13.7.0
|
|
22
|
+
Provides-Extra: analysers
|
|
23
|
+
Requires-Dist: bundle-analyser>=0.1.0; extra == 'analysers'
|
|
24
|
+
Provides-Extra: dev
|
|
25
|
+
Requires-Dist: pytest-cov>=4.0.0; extra == 'dev'
|
|
26
|
+
Requires-Dist: pytest>=8.0.0; extra == 'dev'
|
|
27
|
+
Provides-Extra: documents
|
|
28
|
+
Requires-Dist: document-analyser>=0.5.0; extra == 'documents'
|
|
29
|
+
Provides-Extra: llm
|
|
30
|
+
Requires-Dist: anthropic>=0.40.0; extra == 'llm'
|
|
31
|
+
Description-Content-Type: text/markdown
|
|
32
|
+
|
|
33
|
+
# assessment-lens
|
|
34
|
+
|
|
35
|
+
Part of the [lens family](https://github.com/michael-borck/lens-analysers).
|
|
36
|
+
|
|
37
|
+
[](https://www.python.org/downloads/)
|
|
38
|
+
[](https://opensource.org/licenses/MIT)
|
|
39
|
+
|
|
40
|
+
**Signals-based assessment.** Analysers generate signals about student
|
|
41
|
+
submissions; this lens maps those signals to a rubric as **observations, not
|
|
42
|
+
grades**. A lecturer reads the observations, weighs them, and assigns the mark.
|
|
43
|
+
The AI **narrates and cites; it never scores.** A human stays in the loop.
|
|
44
|
+
|
|
45
|
+
> `assessment-lens` is a **lens** (an assessment-aware product), not an
|
|
46
|
+
> `-analyser`. Analysers are assessment-agnostic signal generators; lenses sit
|
|
47
|
+
> above them. It *consumes* analysers (via `bundle-analyser`); it never generates
|
|
48
|
+
> signals. See [ADR-0001](https://github.com/michael-borck/lens-analysers/blob/main/docs/adr/0001-alignment-lives-in-assessment-lens.md).
|
|
49
|
+
|
|
50
|
+
## Why "observations, not grades"
|
|
51
|
+
|
|
52
|
+
LLMs are inconsistent at the precise act of *marking*. So the lens keeps them off
|
|
53
|
+
it. The deterministic signals (from analysers) are the anchor; an **Observation**
|
|
54
|
+
maps a signal or two to a rubric **Criterion** with **cited evidence**, a short
|
|
55
|
+
**narration bound to that evidence**, and a **coverage** (`present` / `partial` /
|
|
56
|
+
`absent`) — which is *"is the evidence there?"*, derived from thresholds, **not a
|
|
57
|
+
mark**. There is deliberately no score/mark/weight field anywhere in the model.
|
|
58
|
+
|
|
59
|
+
## Pipeline
|
|
60
|
+
|
|
61
|
+
```
|
|
62
|
+
Specification ──draft-rubric (LLM, reviewed)──▶ Rubric (criteria + mapping) + deliverables [YAML]
|
|
63
|
+
Submissions root ──discover──▶ one Submission per subfolder
|
|
64
|
+
each Submission folder ──bundle-analyser──▶ Signals
|
|
65
|
+
Signals + Criteria + Deliverables ──alignment-check──▶ Observations
|
|
66
|
+
└─▶ cohort triage sheet + per-student reports
|
|
67
|
+
lecturer reads observations → assigns marks → writes feedback
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
`assess` is **folder-dumb**: one rubric + one folder of submission subfolders.
|
|
71
|
+
Group/individual splits and mark-combining are handled outside (pre/post) or by
|
|
72
|
+
running `assess` per folder.
|
|
73
|
+
|
|
74
|
+
## Two commands
|
|
75
|
+
|
|
76
|
+
- **`assess`** — structured rubric + a submissions folder → observations (cohort
|
|
77
|
+
sheet + per-student reports). Add `--llm` for evidence-bound narration on each
|
|
78
|
+
observation (opt-in; needs the `[llm]` extra + `ANTHROPIC_API_KEY`).
|
|
79
|
+
- **`draft-rubric`** — free-form specification → a *proposed* structured rubric
|
|
80
|
+
for the lecturer to **review/edit before use**. LLM-assisted; reads text/markdown
|
|
81
|
+
directly and `.pdf`/`.docx`/`.pptx` via the `[documents]` extra.
|
|
82
|
+
|
|
83
|
+
## Install
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
# from source (family layout)
|
|
87
|
+
uv venv && source .venv/bin/activate
|
|
88
|
+
uv pip install -e ".[dev]"
|
|
89
|
+
|
|
90
|
+
# pull the whole analyser stack into the same env (for `assess` to run for real):
|
|
91
|
+
uv pip install -e ".[dev,analysers]"
|
|
92
|
+
|
|
93
|
+
# opt-in LLM features (narration + draft-rubric):
|
|
94
|
+
uv pip install -e ".[llm]" # + export ANTHROPIC_API_KEY=...
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
`assess` shells out to the `bundle-analyser` CLI (it must be on `PATH`). The
|
|
98
|
+
`[analysers]` extra installs it; or point `assessment-lens` at an environment
|
|
99
|
+
where it is already installed.
|
|
100
|
+
|
|
101
|
+
## Quick start
|
|
102
|
+
|
|
103
|
+
```bash
|
|
104
|
+
# 1. author a structured rubric (or start from the example)
|
|
105
|
+
cp examples/data-viz-rubric.yaml my-rubric.yaml
|
|
106
|
+
|
|
107
|
+
# 2. lay out submissions: one subfolder per student/group
|
|
108
|
+
# submissions/alice/{report.pdf,demo.mp4} submissions/bob/...
|
|
109
|
+
|
|
110
|
+
# 3. assess (add --llm for narrated observations)
|
|
111
|
+
assessment-lens assess my-rubric.yaml submissions/ -o out/
|
|
112
|
+
# -> out/cohort-sheet.csv (row per submission × criterion, sortable)
|
|
113
|
+
# -> out/reports/<id>.md (per-student observation sheet, no marks)
|
|
114
|
+
|
|
115
|
+
# or draft a rubric from a free-form spec, then review/edit it
|
|
116
|
+
assessment-lens draft-rubric assignment-brief.md -o my-rubric.yaml
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
## Rubric schema (the central contract)
|
|
120
|
+
|
|
121
|
+
```yaml
|
|
122
|
+
assignment: "Data-Viz Project"
|
|
123
|
+
component: individual # plain label; assess ignores it
|
|
124
|
+
expected_deliverables:
|
|
125
|
+
- id: report
|
|
126
|
+
description: "Written report (~2000 words)"
|
|
127
|
+
accepts: [document] # content kinds that satisfy it
|
|
128
|
+
- id: demo
|
|
129
|
+
description: "≤5-min recorded demo"
|
|
130
|
+
accepts: [video]
|
|
131
|
+
rubric:
|
|
132
|
+
- id: critical-thinking
|
|
133
|
+
description: "Evidence of critical engagement / analysis"
|
|
134
|
+
signals_of_interest: [conversation.critical_thinking, reflection.depth] # OPTIONAL mapping
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
`signals_of_interest` is the **signal→criterion mapping**, and it's **optional**.
|
|
138
|
+
Pinned → deterministic coverage. Blank → `alignment-check` selects signals at
|
|
139
|
+
runtime and shows its choice (near-term).
|
|
140
|
+
|
|
141
|
+
## Status
|
|
142
|
+
|
|
143
|
+
**v0.2.** Working today:
|
|
144
|
+
|
|
145
|
+
- ✅ Rubric load/validate; deliverable reconciliation; submission discovery
|
|
146
|
+
- ✅ `assess` orchestration + evidence-bound observations + threshold coverage
|
|
147
|
+
- ✅ cohort sheet (CSV) + per-student reports (Markdown)
|
|
148
|
+
- ✅ `bundle-analyser` integration — verified against the real CLI output schema
|
|
149
|
+
- ✅ LLM narration (`assess --llm`) — narrate-and-cite, bound to evidence; degrades
|
|
150
|
+
to empty notes when the `[llm]` extra/key is missing
|
|
151
|
+
- ✅ `draft-rubric` — proposes a rubric from a free-form spec (always review before use)
|
|
152
|
+
- 📋 Runtime signal selection when `signals_of_interest` is blank — deferred
|
|
153
|
+
|
|
154
|
+
## Development
|
|
155
|
+
|
|
156
|
+
```bash
|
|
157
|
+
ruff format . && ruff check . && pytest -v
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
## License
|
|
161
|
+
|
|
162
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
# assessment-lens
|
|
2
|
+
|
|
3
|
+
Part of the [lens family](https://github.com/michael-borck/lens-analysers).
|
|
4
|
+
|
|
5
|
+
[](https://www.python.org/downloads/)
|
|
6
|
+
[](https://opensource.org/licenses/MIT)
|
|
7
|
+
|
|
8
|
+
**Signals-based assessment.** Analysers generate signals about student
|
|
9
|
+
submissions; this lens maps those signals to a rubric as **observations, not
|
|
10
|
+
grades**. A lecturer reads the observations, weighs them, and assigns the mark.
|
|
11
|
+
The AI **narrates and cites; it never scores.** A human stays in the loop.
|
|
12
|
+
|
|
13
|
+
> `assessment-lens` is a **lens** (an assessment-aware product), not an
|
|
14
|
+
> `-analyser`. Analysers are assessment-agnostic signal generators; lenses sit
|
|
15
|
+
> above them. It *consumes* analysers (via `bundle-analyser`); it never generates
|
|
16
|
+
> signals. See [ADR-0001](https://github.com/michael-borck/lens-analysers/blob/main/docs/adr/0001-alignment-lives-in-assessment-lens.md).
|
|
17
|
+
|
|
18
|
+
## Why "observations, not grades"
|
|
19
|
+
|
|
20
|
+
LLMs are inconsistent at the precise act of *marking*. So the lens keeps them off
|
|
21
|
+
it. The deterministic signals (from analysers) are the anchor; an **Observation**
|
|
22
|
+
maps a signal or two to a rubric **Criterion** with **cited evidence**, a short
|
|
23
|
+
**narration bound to that evidence**, and a **coverage** (`present` / `partial` /
|
|
24
|
+
`absent`) — which is *"is the evidence there?"*, derived from thresholds, **not a
|
|
25
|
+
mark**. There is deliberately no score/mark/weight field anywhere in the model.
|
|
26
|
+
|
|
27
|
+
## Pipeline
|
|
28
|
+
|
|
29
|
+
```
|
|
30
|
+
Specification ──draft-rubric (LLM, reviewed)──▶ Rubric (criteria + mapping) + deliverables [YAML]
|
|
31
|
+
Submissions root ──discover──▶ one Submission per subfolder
|
|
32
|
+
each Submission folder ──bundle-analyser──▶ Signals
|
|
33
|
+
Signals + Criteria + Deliverables ──alignment-check──▶ Observations
|
|
34
|
+
└─▶ cohort triage sheet + per-student reports
|
|
35
|
+
lecturer reads observations → assigns marks → writes feedback
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
`assess` is **folder-dumb**: one rubric + one folder of submission subfolders.
|
|
39
|
+
Group/individual splits and mark-combining are handled outside (pre/post) or by
|
|
40
|
+
running `assess` per folder.
|
|
41
|
+
|
|
42
|
+
## Two commands
|
|
43
|
+
|
|
44
|
+
- **`assess`** — structured rubric + a submissions folder → observations (cohort
|
|
45
|
+
sheet + per-student reports). Add `--llm` for evidence-bound narration on each
|
|
46
|
+
observation (opt-in; needs the `[llm]` extra + `ANTHROPIC_API_KEY`).
|
|
47
|
+
- **`draft-rubric`** — free-form specification → a *proposed* structured rubric
|
|
48
|
+
for the lecturer to **review/edit before use**. LLM-assisted; reads text/markdown
|
|
49
|
+
directly and `.pdf`/`.docx`/`.pptx` via the `[documents]` extra.
|
|
50
|
+
|
|
51
|
+
## Install
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
# from source (family layout)
|
|
55
|
+
uv venv && source .venv/bin/activate
|
|
56
|
+
uv pip install -e ".[dev]"
|
|
57
|
+
|
|
58
|
+
# pull the whole analyser stack into the same env (for `assess` to run for real):
|
|
59
|
+
uv pip install -e ".[dev,analysers]"
|
|
60
|
+
|
|
61
|
+
# opt-in LLM features (narration + draft-rubric):
|
|
62
|
+
uv pip install -e ".[llm]" # + export ANTHROPIC_API_KEY=...
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
`assess` shells out to the `bundle-analyser` CLI (it must be on `PATH`). The
|
|
66
|
+
`[analysers]` extra installs it; or point `assessment-lens` at an environment
|
|
67
|
+
where it is already installed.
|
|
68
|
+
|
|
69
|
+
## Quick start
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
# 1. author a structured rubric (or start from the example)
|
|
73
|
+
cp examples/data-viz-rubric.yaml my-rubric.yaml
|
|
74
|
+
|
|
75
|
+
# 2. lay out submissions: one subfolder per student/group
|
|
76
|
+
# submissions/alice/{report.pdf,demo.mp4} submissions/bob/...
|
|
77
|
+
|
|
78
|
+
# 3. assess (add --llm for narrated observations)
|
|
79
|
+
assessment-lens assess my-rubric.yaml submissions/ -o out/
|
|
80
|
+
# -> out/cohort-sheet.csv (row per submission × criterion, sortable)
|
|
81
|
+
# -> out/reports/<id>.md (per-student observation sheet, no marks)
|
|
82
|
+
|
|
83
|
+
# or draft a rubric from a free-form spec, then review/edit it
|
|
84
|
+
assessment-lens draft-rubric assignment-brief.md -o my-rubric.yaml
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
## Rubric schema (the central contract)
|
|
88
|
+
|
|
89
|
+
```yaml
|
|
90
|
+
assignment: "Data-Viz Project"
|
|
91
|
+
component: individual # plain label; assess ignores it
|
|
92
|
+
expected_deliverables:
|
|
93
|
+
- id: report
|
|
94
|
+
description: "Written report (~2000 words)"
|
|
95
|
+
accepts: [document] # content kinds that satisfy it
|
|
96
|
+
- id: demo
|
|
97
|
+
description: "≤5-min recorded demo"
|
|
98
|
+
accepts: [video]
|
|
99
|
+
rubric:
|
|
100
|
+
- id: critical-thinking
|
|
101
|
+
description: "Evidence of critical engagement / analysis"
|
|
102
|
+
signals_of_interest: [conversation.critical_thinking, reflection.depth] # OPTIONAL mapping
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
`signals_of_interest` is the **signal→criterion mapping**, and it's **optional**.
|
|
106
|
+
Pinned → deterministic coverage. Blank → `alignment-check` selects signals at
|
|
107
|
+
runtime and shows its choice (near-term).
|
|
108
|
+
|
|
109
|
+
## Status
|
|
110
|
+
|
|
111
|
+
**v0.2.** Working today:
|
|
112
|
+
|
|
113
|
+
- ✅ Rubric load/validate; deliverable reconciliation; submission discovery
|
|
114
|
+
- ✅ `assess` orchestration + evidence-bound observations + threshold coverage
|
|
115
|
+
- ✅ cohort sheet (CSV) + per-student reports (Markdown)
|
|
116
|
+
- ✅ `bundle-analyser` integration — verified against the real CLI output schema
|
|
117
|
+
- ✅ LLM narration (`assess --llm`) — narrate-and-cite, bound to evidence; degrades
|
|
118
|
+
to empty notes when the `[llm]` extra/key is missing
|
|
119
|
+
- ✅ `draft-rubric` — proposes a rubric from a free-form spec (always review before use)
|
|
120
|
+
- 📋 Runtime signal selection when `signals_of_interest` is blank — deferred
|
|
121
|
+
|
|
122
|
+
## Development
|
|
123
|
+
|
|
124
|
+
```bash
|
|
125
|
+
ruff format . && ruff check . && pytest -v
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
## License
|
|
129
|
+
|
|
130
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# Scoping
|
|
2
|
+
|
|
3
|
+
The canonical scope for `assessment-lens` lives with the family architecture docs:
|
|
4
|
+
|
|
5
|
+
- **Scope & design** — `lens-analysers/docs/ASSESSMENT-LENS-SCOPING.md`
|
|
6
|
+
(glossary, pipeline, the two commands, rubric + observation schemas,
|
|
7
|
+
composition, principles, MVP boundary, harvest map).
|
|
8
|
+
- **Why a lens, not an analyser** —
|
|
9
|
+
`lens-analysers/docs/adr/0001-alignment-lives-in-assessment-lens.md`.
|
|
10
|
+
- **Signals it consumes** — `lens-analysers/docs/SIGNAL-CATALOGUE.md`.
|
|
11
|
+
|
|
12
|
+
## The one-paragraph version
|
|
13
|
+
|
|
14
|
+
Tools generate signals about student submissions; this lens maps those signals
|
|
15
|
+
to a rubric as **observations, not grades**. The lecturer reads them, weighs
|
|
16
|
+
them, and assigns the mark. The AI **narrates and cites; it never scores** — a
|
|
17
|
+
human stays in the loop. `assess` is folder-dumb: one rubric + one folder of
|
|
18
|
+
submission subfolders, `bundle-analyser` once per subfolder, then
|
|
19
|
+
`alignment-check` turns signals into evidence-bound observations with
|
|
20
|
+
threshold-derived coverage.
|
|
21
|
+
|
|
22
|
+
## Build state vs scope
|
|
23
|
+
|
|
24
|
+
The deterministic spine (rubric → discover → reconcile deliverables → gather
|
|
25
|
+
evidence → coverage → reports) is implemented. The two pieces still to land:
|
|
26
|
+
|
|
27
|
+
1. **LLM narration** (`alignment.narrate`) — the only LLM-touched field
|
|
28
|
+
(`Observation.note`); arrives with the `[llm]` extra.
|
|
29
|
+
2. **`draft-rubric`** — free-form spec → proposed rubric; harvest + adapt the
|
|
30
|
+
rubric/key code from `video-analyser` @ `b3ae7e19`.
|
|
31
|
+
|
|
32
|
+
> Reminder recorded in family memory: `video-analyser` 0.10.0 (grading removed)
|
|
33
|
+
> is on `main` but **held from PyPI** until this lens reborns the grading, so
|
|
34
|
+
> users get a migration path.
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# Example structured rubric for assessment-lens `assess`.
|
|
2
|
+
# This is the central contract — author it by hand for v1 (draft-rubric will
|
|
3
|
+
# later propose one from a free-form spec). `component` is a plain label; assess
|
|
4
|
+
# ignores it. `signals_of_interest` is the OPTIONAL signal->criterion mapping.
|
|
5
|
+
assignment: "Data-Viz Project"
|
|
6
|
+
component: individual
|
|
7
|
+
expected_deliverables:
|
|
8
|
+
- id: report
|
|
9
|
+
description: "Written report (~2000 words)"
|
|
10
|
+
accepts: [document]
|
|
11
|
+
- id: demo
|
|
12
|
+
description: "≤5-min recorded demo"
|
|
13
|
+
accepts: [video]
|
|
14
|
+
rubric:
|
|
15
|
+
- id: critical-thinking
|
|
16
|
+
description: "Evidence of critical engagement / analysis"
|
|
17
|
+
signals_of_interest: [conversation.critical_thinking, reflection.depth]
|
|
18
|
+
- id: technical-quality
|
|
19
|
+
description: "Code quality and correctness"
|
|
20
|
+
signals_of_interest: [code.complexity, code.lint, code.code_level]
|
|
21
|
+
- id: communication
|
|
22
|
+
description: "Clarity of written and spoken communication"
|
|
23
|
+
# no signals pinned -> alignment-check would select at runtime (near-term)
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "assessment-lens"
|
|
7
|
+
version = "0.2.0"
|
|
8
|
+
description = "Signals-based assessment — map analyser signals to a rubric as observations, never grades. A human assigns every mark."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [{ name = "Michael Borck", email = "michael.borck@curtin.edu.au" }]
|
|
13
|
+
keywords = ["assessment", "rubric", "education", "edtech", "signals", "lens", "feedback"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 3 - Alpha",
|
|
16
|
+
"Intended Audience :: Education",
|
|
17
|
+
"License :: OSI Approved :: MIT License",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Programming Language :: Python :: 3.11",
|
|
20
|
+
"Programming Language :: Python :: 3.12",
|
|
21
|
+
]
|
|
22
|
+
# A lens is a *consumer*, not an analyser: no fastapi / uvicorn / lens-contract /
|
|
23
|
+
# manifest here. It reads rubrics, composes bundle-analyser, and renders reports.
|
|
24
|
+
dependencies = [
|
|
25
|
+
"pydantic>=2.5.0",
|
|
26
|
+
"pyyaml>=6.0",
|
|
27
|
+
"rich>=13.7.0",
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
[project.optional-dependencies]
|
|
31
|
+
# bundle-analyser is invoked as a subprocess (the `bundle-analyser` CLI on PATH).
|
|
32
|
+
# Install this extra to pull the whole analyser stack into the same environment;
|
|
33
|
+
# otherwise point assessment-lens at an already-installed bundle-analyser.
|
|
34
|
+
analysers = [
|
|
35
|
+
"bundle-analyser>=0.1.0",
|
|
36
|
+
]
|
|
37
|
+
# Opt-in LLM features: alignment-check narration + draft-rubric (narrate-and-cite;
|
|
38
|
+
# never scores). Anthropic-only for now, matching conversation-analyser.
|
|
39
|
+
llm = [
|
|
40
|
+
"anthropic>=0.40.0",
|
|
41
|
+
]
|
|
42
|
+
# Binary specs (.pdf/.docx/.pptx) for draft-rubric are extracted via the
|
|
43
|
+
# family's canonical extractor.
|
|
44
|
+
documents = [
|
|
45
|
+
"document-analyser>=0.5.0",
|
|
46
|
+
]
|
|
47
|
+
dev = [
|
|
48
|
+
"pytest>=8.0.0",
|
|
49
|
+
"pytest-cov>=4.0.0",
|
|
50
|
+
]
|
|
51
|
+
|
|
52
|
+
[tool.uv.sources]
|
|
53
|
+
bundle-analyser = { path = "../bundle-analyser", editable = true }
|
|
54
|
+
document-analyser = { path = "../document-analyser", editable = true }
|
|
55
|
+
|
|
56
|
+
[project.scripts]
|
|
57
|
+
assessment-lens = "assessment_lens.cli:main"
|
|
58
|
+
|
|
59
|
+
[project.urls]
|
|
60
|
+
Homepage = "https://github.com/michael-borck/assessment-lens"
|
|
61
|
+
Repository = "https://github.com/michael-borck/assessment-lens"
|
|
62
|
+
Issues = "https://github.com/michael-borck/assessment-lens/issues"
|
|
63
|
+
|
|
64
|
+
[tool.hatch.build.targets.wheel]
|
|
65
|
+
packages = ["src/assessment_lens"]
|
|
66
|
+
|
|
67
|
+
[tool.pytest.ini_options]
|
|
68
|
+
testpaths = ["tests"]
|
|
69
|
+
|
|
70
|
+
[tool.ruff]
|
|
71
|
+
line-length = 100
|
|
72
|
+
target-version = "py311"
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""assessment-lens — signals-based assessment for the lens family.
|
|
2
|
+
|
|
3
|
+
A *lens* (assessment-aware product) that consumes analyser signals and maps them
|
|
4
|
+
to a rubric as **observations, not grades**. The AI narrates and cites; a human
|
|
5
|
+
assigns every mark. See docs/SCOPING.md.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from .exceptions import AssessmentLensError
|
|
9
|
+
from .models import (
|
|
10
|
+
AssessmentResult,
|
|
11
|
+
Coverage,
|
|
12
|
+
Criterion,
|
|
13
|
+
DeliverableObservation,
|
|
14
|
+
Evidence,
|
|
15
|
+
ExpectedDeliverable,
|
|
16
|
+
Observation,
|
|
17
|
+
Rubric,
|
|
18
|
+
SubmissionResult,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
__version__ = "0.1.0"
|
|
22
|
+
|
|
23
|
+
__all__ = [
|
|
24
|
+
"AssessmentLensError",
|
|
25
|
+
"AssessmentResult",
|
|
26
|
+
"Coverage",
|
|
27
|
+
"Criterion",
|
|
28
|
+
"DeliverableObservation",
|
|
29
|
+
"Evidence",
|
|
30
|
+
"ExpectedDeliverable",
|
|
31
|
+
"Observation",
|
|
32
|
+
"Rubric",
|
|
33
|
+
"SubmissionResult",
|
|
34
|
+
"__version__",
|
|
35
|
+
]
|