feedforward-practice 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- feedforward_practice-0.1.0/.gitignore +74 -0
- feedforward_practice-0.1.0/PKG-INFO +59 -0
- feedforward_practice-0.1.0/README.md +42 -0
- feedforward_practice-0.1.0/pyproject.toml +38 -0
- feedforward_practice-0.1.0/src/feedforward_practice/__init__.py +13 -0
- feedforward_practice-0.1.0/src/feedforward_practice/api.py +83 -0
- feedforward_practice-0.1.0/src/feedforward_practice/cli.py +82 -0
- feedforward_practice-0.1.0/src/feedforward_practice/contracts/feedback-prompt.md +45 -0
- feedforward_practice-0.1.0/src/feedforward_practice/contracts/ffrubric.schema.json +49 -0
- feedforward_practice-0.1.0/src/feedforward_practice/contracts/levels.json +15 -0
- feedforward_practice-0.1.0/src/feedforward_practice/contracts.py +52 -0
- feedforward_practice-0.1.0/src/feedforward_practice/engine.py +175 -0
- feedforward_practice-0.1.0/src/feedforward_practice/manifest.py +11 -0
- feedforward_practice-0.1.0/src/feedforward_practice/providers.py +99 -0
- feedforward_practice-0.1.0/src/feedforward_practice/run.py +43 -0
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# Virtual Environment
|
|
2
|
+
venv/
|
|
3
|
+
*.venv
|
|
4
|
+
.env
|
|
5
|
+
.venv/ # uv virtual environment
|
|
6
|
+
|
|
7
|
+
# Python bytecode
|
|
8
|
+
__pycache__/
|
|
9
|
+
*.py[cod]
|
|
10
|
+
*$py.class
|
|
11
|
+
|
|
12
|
+
# IDE and editor files
|
|
13
|
+
.vscode/
|
|
14
|
+
.idea/
|
|
15
|
+
*.swp
|
|
16
|
+
*.swo
|
|
17
|
+
*~
|
|
18
|
+
|
|
19
|
+
# Operating System files
|
|
20
|
+
.DS_Store
|
|
21
|
+
Thumbs.db
|
|
22
|
+
|
|
23
|
+
# SQLite database files
|
|
24
|
+
*.db
|
|
25
|
+
*.sqlite3
|
|
26
|
+
*.sqlite
|
|
27
|
+
|
|
28
|
+
# Data directory
|
|
29
|
+
data/
|
|
30
|
+
|
|
31
|
+
# Alembic versions
|
|
32
|
+
alembic/versions/*
|
|
33
|
+
!alembic/versions/.gitkeep
|
|
34
|
+
|
|
35
|
+
# Distribution / packaging
|
|
36
|
+
dist/
|
|
37
|
+
build/
|
|
38
|
+
*.egg-info/
|
|
39
|
+
|
|
40
|
+
# Type checking
|
|
41
|
+
.mypy_cache/
|
|
42
|
+
.dmypy.json
|
|
43
|
+
dmypy.json
|
|
44
|
+
|
|
45
|
+
# Linting/Formatting
|
|
46
|
+
.ruff_cache/
|
|
47
|
+
|
|
48
|
+
# Project specific reference materials
|
|
49
|
+
docs/reference/
|
|
50
|
+
docs/html-mockups/
|
|
51
|
+
docs/svg-mockups/
|
|
52
|
+
docs/llms-ctx.txt
|
|
53
|
+
|
|
54
|
+
# Session keys and local configuration
|
|
55
|
+
.sesskey
|
|
56
|
+
.env
|
|
57
|
+
|
|
58
|
+
# Sensitive files that may exist
|
|
59
|
+
*.log
|
|
60
|
+
*.pem
|
|
61
|
+
*.key
|
|
62
|
+
config.local.*
|
|
63
|
+
local_settings.py
|
|
64
|
+
|
|
65
|
+
_notes
|
|
66
|
+
.opencode
|
|
67
|
+
.pytest_cache/
|
|
68
|
+
|
|
69
|
+
# macOS AppleDouble sidecars (external drive)
|
|
70
|
+
._*
|
|
71
|
+
.DS_Store
|
|
72
|
+
|
|
73
|
+
# Real production env stays out of the public repo (use .env.production.example)
|
|
74
|
+
.env.production
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: feedforward-practice
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Private practice feedback on drafts against an instructor rubric (.ffrubric) — the FeedForward Desktop sidecar engine
|
|
5
|
+
Author-email: Michael Borck <michael@borck.dev>
|
|
6
|
+
License: MIT
|
|
7
|
+
Requires-Python: >=3.11
|
|
8
|
+
Requires-Dist: httpx>=0.27
|
|
9
|
+
Provides-Extra: dev
|
|
10
|
+
Requires-Dist: pytest>=7.0; extra == 'dev'
|
|
11
|
+
Requires-Dist: ruff>=0.1; extra == 'dev'
|
|
12
|
+
Provides-Extra: serve
|
|
13
|
+
Requires-Dist: fastapi>=0.110; extra == 'serve'
|
|
14
|
+
Requires-Dist: lens-contract>=0.3.1; extra == 'serve'
|
|
15
|
+
Requires-Dist: uvicorn>=0.29; extra == 'serve'
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
|
|
18
|
+
# feedforward-practice
|
|
19
|
+
|
|
20
|
+
Private practice feedback on drafts against an instructor rubric — the
|
|
21
|
+
engine and sidecar API behind **FeedForward Desktop**.
|
|
22
|
+
|
|
23
|
+
A student opens a `.ffrubric` file (exported by their instructor from
|
|
24
|
+
[FeedForward](https://github.com/michael-borck/feed-forward)), pastes or
|
|
25
|
+
opens a draft, points the app at a model endpoint they control, and gets
|
|
26
|
+
rubric-aligned formative feedback rendered with FeedForward's qualitative
|
|
27
|
+
levels (the dartboard: "Closing in", "On the board", …). Nothing is stored
|
|
28
|
+
on a server; there are no accounts.
|
|
29
|
+
|
|
30
|
+
## Model endpoints
|
|
31
|
+
|
|
32
|
+
One OpenAI-compatible client covers all supported setups:
|
|
33
|
+
|
|
34
|
+
| Setup | Base URL | Key |
|
|
35
|
+
|---|---|---|
|
|
36
|
+
| Local Ollama (default) | `http://localhost:11434/v1` | none |
|
|
37
|
+
| Remote Ollama behind a proxy | your server's URL | bearer token |
|
|
38
|
+
| BYOK (OpenAI, Anthropic-compat, OpenRouter, Groq…) | provider URL | your key |
|
|
39
|
+
|
|
40
|
+
Configure via arguments or `FEEDFORWARD_PRACTICE_BASE_URL`,
|
|
41
|
+
`FEEDFORWARD_PRACTICE_API_KEY`, `FEEDFORWARD_PRACTICE_MODEL`.
|
|
42
|
+
|
|
43
|
+
## Usage
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
pip install feedforward-practice # engine + CLI
|
|
47
|
+
pip install "feedforward-practice[serve]" # + the HTTP API for the desktop shell
|
|
48
|
+
|
|
49
|
+
feedforward-practice assess --rubric essay1.ffrubric --draft draft.md
|
|
50
|
+
feedforward-practice serve --port 8022
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
## Contracts
|
|
54
|
+
|
|
55
|
+
The rubric format, level words, and prompt/JSON contract are vendored at
|
|
56
|
+
build time from the FeedForward repo's `shared/` directory — the single
|
|
57
|
+
source of truth both the server and this package test against.
|
|
58
|
+
|
|
59
|
+
Part of the FeedForward project · MIT
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# feedforward-practice
|
|
2
|
+
|
|
3
|
+
Private practice feedback on drafts against an instructor rubric — the
|
|
4
|
+
engine and sidecar API behind **FeedForward Desktop**.
|
|
5
|
+
|
|
6
|
+
A student opens a `.ffrubric` file (exported by their instructor from
|
|
7
|
+
[FeedForward](https://github.com/michael-borck/feed-forward)), pastes or
|
|
8
|
+
opens a draft, points the app at a model endpoint they control, and gets
|
|
9
|
+
rubric-aligned formative feedback rendered with FeedForward's qualitative
|
|
10
|
+
levels (the dartboard: "Closing in", "On the board", …). Nothing is stored
|
|
11
|
+
on a server; there are no accounts.
|
|
12
|
+
|
|
13
|
+
## Model endpoints
|
|
14
|
+
|
|
15
|
+
One OpenAI-compatible client covers all supported setups:
|
|
16
|
+
|
|
17
|
+
| Setup | Base URL | Key |
|
|
18
|
+
|---|---|---|
|
|
19
|
+
| Local Ollama (default) | `http://localhost:11434/v1` | none |
|
|
20
|
+
| Remote Ollama behind a proxy | your server's URL | bearer token |
|
|
21
|
+
| BYOK (OpenAI, Anthropic-compat, OpenRouter, Groq…) | provider URL | your key |
|
|
22
|
+
|
|
23
|
+
Configure via arguments or `FEEDFORWARD_PRACTICE_BASE_URL`,
|
|
24
|
+
`FEEDFORWARD_PRACTICE_API_KEY`, `FEEDFORWARD_PRACTICE_MODEL`.
|
|
25
|
+
|
|
26
|
+
## Usage
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
pip install feedforward-practice # engine + CLI
|
|
30
|
+
pip install "feedforward-practice[serve]" # + the HTTP API for the desktop shell
|
|
31
|
+
|
|
32
|
+
feedforward-practice assess --rubric essay1.ffrubric --draft draft.md
|
|
33
|
+
feedforward-practice serve --port 8022
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
## Contracts
|
|
37
|
+
|
|
38
|
+
The rubric format, level words, and prompt/JSON contract are vendored at
|
|
39
|
+
build time from the FeedForward repo's `shared/` directory — the single
|
|
40
|
+
source of truth both the server and this package test against.
|
|
41
|
+
|
|
42
|
+
Part of the FeedForward project · MIT
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "feedforward-practice"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Private practice feedback on drafts against an instructor rubric (.ffrubric) — the FeedForward Desktop sidecar engine"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [{ name = "Michael Borck", email = "michael@borck.dev" }]
|
|
13
|
+
dependencies = [
|
|
14
|
+
"httpx>=0.27",
|
|
15
|
+
]
|
|
16
|
+
|
|
17
|
+
[project.optional-dependencies]
|
|
18
|
+
serve = [
|
|
19
|
+
"lens-contract>=0.3.1",
|
|
20
|
+
"fastapi>=0.110",
|
|
21
|
+
"uvicorn>=0.29",
|
|
22
|
+
]
|
|
23
|
+
dev = [
|
|
24
|
+
"pytest>=7.0",
|
|
25
|
+
"ruff>=0.1",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
[project.scripts]
|
|
29
|
+
feedforward-practice = "feedforward_practice.cli:main"
|
|
30
|
+
|
|
31
|
+
[tool.hatch.build.targets.wheel]
|
|
32
|
+
packages = ["src/feedforward_practice"]
|
|
33
|
+
|
|
34
|
+
[tool.hatch.build.targets.sdist]
|
|
35
|
+
include = ["src", "README.md"]
|
|
36
|
+
|
|
37
|
+
[tool.ruff]
|
|
38
|
+
line-length = 88
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""feedforward-practice — private practice feedback on drafts.
|
|
2
|
+
|
|
3
|
+
Students open an instructor-exported .ffrubric file, paste a draft, choose
|
|
4
|
+
their own model endpoint (local Ollama by default), and get rubric-aligned
|
|
5
|
+
formative feedback rendered with the same qualitative levels as FeedForward
|
|
6
|
+
itself. This package is the engine and sidecar API behind FeedForward
|
|
7
|
+
Desktop; it holds no accounts and stores nothing.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
__version__ = "0.1.0"
|
|
11
|
+
|
|
12
|
+
from feedforward_practice.providers import ProviderConfig # noqa: F401
|
|
13
|
+
from feedforward_practice.run import practice_feedback # noqa: F401
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""HTTP API for FeedForward Desktop (the Electron shell's sidecar).
|
|
2
|
+
|
|
3
|
+
lens-contract standard surface (/health, /manifest, bearer auth via
|
|
4
|
+
FEEDFORWARD_PRACTICE_AUTH_TOKEN) plus the practice routes. Feedback runs
|
|
5
|
+
are asynchronous jobs (the family's 202-and-poll pattern) because model
|
|
6
|
+
calls can take minutes on local hardware.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import threading
|
|
10
|
+
import uuid
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from fastapi import FastAPI, HTTPException
|
|
14
|
+
from lens_contract import add_auth, add_contract_routes, add_cors
|
|
15
|
+
from pydantic import BaseModel, Field
|
|
16
|
+
|
|
17
|
+
from feedforward_practice.manifest import MANIFEST
|
|
18
|
+
from feedforward_practice.providers import (
|
|
19
|
+
ProviderConfig,
|
|
20
|
+
ProviderError,
|
|
21
|
+
list_models,
|
|
22
|
+
)
|
|
23
|
+
from feedforward_practice.run import practice_feedback
|
|
24
|
+
|
|
25
|
+
app = FastAPI(title=MANIFEST["name"], version=MANIFEST["version"])
|
|
26
|
+
add_contract_routes(app, MANIFEST)
|
|
27
|
+
add_auth(app, env_prefix="FEEDFORWARD_PRACTICE") # before add_cors
|
|
28
|
+
add_cors(app, env_prefix="FEEDFORWARD_PRACTICE")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class ProviderIn(BaseModel):
|
|
32
|
+
base_url: str = ""
|
|
33
|
+
api_key: str = ""
|
|
34
|
+
model: str = ""
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class FeedbackRequest(BaseModel):
|
|
38
|
+
rubric: dict
|
|
39
|
+
draft_text: str
|
|
40
|
+
num_runs: int = Field(default=1, ge=1, le=5)
|
|
41
|
+
provider: ProviderIn = ProviderIn()
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
_jobs: dict[str, dict[str, Any]] = {}
|
|
45
|
+
_jobs_lock = threading.Lock()
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _run_job(job_id: str, req: FeedbackRequest) -> None:
|
|
49
|
+
provider = ProviderConfig.from_env().merged(req.provider.model_dump())
|
|
50
|
+
try:
|
|
51
|
+
result = practice_feedback(req.rubric, req.draft_text, provider, req.num_runs)
|
|
52
|
+
update = {"status": "done", "result": result}
|
|
53
|
+
except Exception as e: # surfaced to the UI as a friendly failure
|
|
54
|
+
update = {"status": "error", "error": str(e)}
|
|
55
|
+
with _jobs_lock:
|
|
56
|
+
_jobs[job_id].update(update)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@app.post("/practice/feedback", status_code=202)
|
|
60
|
+
def start_feedback(req: FeedbackRequest):
|
|
61
|
+
job_id = uuid.uuid4().hex
|
|
62
|
+
with _jobs_lock:
|
|
63
|
+
_jobs[job_id] = {"status": "running"}
|
|
64
|
+
threading.Thread(target=_run_job, args=(job_id, req), daemon=True).start()
|
|
65
|
+
return {"id": job_id, "status": "running"}
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@app.get("/practice/feedback/{job_id}")
|
|
69
|
+
def get_feedback(job_id: str):
|
|
70
|
+
with _jobs_lock:
|
|
71
|
+
job = _jobs.get(job_id)
|
|
72
|
+
if job is None:
|
|
73
|
+
raise HTTPException(status_code=404, detail="Unknown job")
|
|
74
|
+
return {"id": job_id, **job}
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@app.post("/practice/models")
|
|
78
|
+
def models(provider: ProviderIn):
|
|
79
|
+
cfg = ProviderConfig.from_env().merged(provider.model_dump())
|
|
80
|
+
try:
|
|
81
|
+
return {"models": list_models(cfg)}
|
|
82
|
+
except ProviderError as e:
|
|
83
|
+
raise HTTPException(status_code=502, detail=str(e)) from e
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""CLI: `feedforward-practice assess` (direct) and `serve` (desktop sidecar)."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
import pathlib
|
|
6
|
+
import sys
|
|
7
|
+
|
|
8
|
+
from feedforward_practice.providers import ProviderConfig
|
|
9
|
+
from feedforward_practice.run import practice_feedback
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def main(argv: list[str] | None = None) -> int:
|
|
13
|
+
parser = argparse.ArgumentParser(
|
|
14
|
+
prog="feedforward-practice",
|
|
15
|
+
description="Practice feedback on a draft against an instructor rubric",
|
|
16
|
+
)
|
|
17
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
18
|
+
|
|
19
|
+
p_assess = sub.add_parser("assess", help="Run practice feedback once")
|
|
20
|
+
p_assess.add_argument("--rubric", required=True, help=".ffrubric file")
|
|
21
|
+
p_assess.add_argument("--draft", required=True, help="draft text file (.txt/.md)")
|
|
22
|
+
p_assess.add_argument("--runs", type=int, default=1)
|
|
23
|
+
p_assess.add_argument("--base-url", default="")
|
|
24
|
+
p_assess.add_argument("--api-key", default="")
|
|
25
|
+
p_assess.add_argument("--model", default="")
|
|
26
|
+
p_assess.add_argument("--json", action="store_true", help="raw JSON output")
|
|
27
|
+
p_assess.set_defaults(func=_cmd_assess)
|
|
28
|
+
|
|
29
|
+
p_serve = sub.add_parser(
|
|
30
|
+
"serve",
|
|
31
|
+
help=(
|
|
32
|
+
"Run the HTTP API (needs the [serve] extra). Set "
|
|
33
|
+
"FEEDFORWARD_PRACTICE_AUTH_TOKEN to require a bearer token."
|
|
34
|
+
),
|
|
35
|
+
)
|
|
36
|
+
p_serve.add_argument("--host", default="127.0.0.1")
|
|
37
|
+
p_serve.add_argument("--port", type=int, default=8022)
|
|
38
|
+
p_serve.set_defaults(func=_cmd_serve)
|
|
39
|
+
|
|
40
|
+
args = parser.parse_args(argv)
|
|
41
|
+
return args.func(args)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _cmd_assess(args) -> int:
|
|
45
|
+
rubric = json.loads(pathlib.Path(args.rubric).read_text())
|
|
46
|
+
draft = pathlib.Path(args.draft).read_text()
|
|
47
|
+
provider = ProviderConfig.from_env().merged(
|
|
48
|
+
{"base_url": args.base_url, "api_key": args.api_key, "model": args.model}
|
|
49
|
+
)
|
|
50
|
+
result = practice_feedback(rubric, draft, provider, args.runs)
|
|
51
|
+
if args.json:
|
|
52
|
+
print(json.dumps(result, indent=2))
|
|
53
|
+
return 0
|
|
54
|
+
overall = result["overall"]
|
|
55
|
+
print(f"\nOverall: {overall['level']['label']}\n")
|
|
56
|
+
for cat in result["categories"]:
|
|
57
|
+
print(f"## {cat['name']} — {cat['level']['label']}")
|
|
58
|
+
if cat["feedback"]:
|
|
59
|
+
print(f" {cat['feedback']}")
|
|
60
|
+
for s in cat["strengths"]:
|
|
61
|
+
print(f" + {s}")
|
|
62
|
+
for i in cat["improvements"]:
|
|
63
|
+
print(f" > {i}")
|
|
64
|
+
print()
|
|
65
|
+
return 0
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _cmd_serve(args) -> int:
|
|
69
|
+
try:
|
|
70
|
+
import uvicorn
|
|
71
|
+
except ImportError:
|
|
72
|
+
print(
|
|
73
|
+
"serve needs the [serve] extra: pip install 'feedforward-practice[serve]'",
|
|
74
|
+
file=sys.stderr,
|
|
75
|
+
)
|
|
76
|
+
return 1
|
|
77
|
+
uvicorn.run("feedforward_practice.api:app", host=args.host, port=args.port)
|
|
78
|
+
return 0
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
if __name__ == "__main__":
|
|
82
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# Canonical feedback prompt (server ↔ desktop contract)
|
|
2
|
+
|
|
3
|
+
Placeholders in `{{double_braces}}`. The JSON response contract at the bottom
|
|
4
|
+
is the part both apps' parsers depend on — change it only with a
|
|
5
|
+
formatVersion bump and matching parser changes on both sides.
|
|
6
|
+
|
|
7
|
+
## System
|
|
8
|
+
|
|
9
|
+
You are an experienced educational assessor providing formative feedback on
|
|
10
|
+
student work. Your goal is to help the student improve their next draft. Be
|
|
11
|
+
specific, constructive, and encouraging; ground every point in the rubric.
|
|
12
|
+
Never invent rubric criteria. Address the student directly.
|
|
13
|
+
|
|
14
|
+
## User
|
|
15
|
+
|
|
16
|
+
## Assignment Context
|
|
17
|
+
Title: {{assignment_title}}
|
|
18
|
+
Description: {{assignment_description}}
|
|
19
|
+
|
|
20
|
+
## Rubric Criteria
|
|
21
|
+
{{rubric_categories}}
|
|
22
|
+
|
|
23
|
+
## Student Submission
|
|
24
|
+
{{submission_text}}
|
|
25
|
+
|
|
26
|
+
## Instructions
|
|
27
|
+
Assess the submission against each rubric criterion. For every criterion,
|
|
28
|
+
give a score from 0-100, concise feedback grounded in the submission, up to
|
|
29
|
+
three specific strengths, and up to three specific, actionable improvements.
|
|
30
|
+
|
|
31
|
+
## Response format (JSON contract)
|
|
32
|
+
Respond with ONLY valid JSON, no code fences, following exactly:
|
|
33
|
+
|
|
34
|
+
{
|
|
35
|
+
"overall_feedback": "string",
|
|
36
|
+
"categories": [
|
|
37
|
+
{
|
|
38
|
+
"name": "string (must match a rubric criterion name exactly)",
|
|
39
|
+
"score": 0,
|
|
40
|
+
"feedback": "string",
|
|
41
|
+
"strengths": ["string"],
|
|
42
|
+
"improvements": ["string"]
|
|
43
|
+
}
|
|
44
|
+
]
|
|
45
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "https://github.com/michael-borck/feed-forward/shared/ffrubric.schema.json",
|
|
4
|
+
"title": "FeedForward rubric export (.ffrubric)",
|
|
5
|
+
"type": "object",
|
|
6
|
+
"required": ["formatVersion", "kind", "assignment", "rubric"],
|
|
7
|
+
"properties": {
|
|
8
|
+
"formatVersion": { "type": "string", "pattern": "^1\\.[0-9]+$" },
|
|
9
|
+
"kind": { "const": "ffrubric" },
|
|
10
|
+
"exportedAt": { "type": "string" },
|
|
11
|
+
"source": {
|
|
12
|
+
"type": "object",
|
|
13
|
+
"properties": {
|
|
14
|
+
"app": { "type": "string" },
|
|
15
|
+
"courseCode": { "type": "string" },
|
|
16
|
+
"courseTitle": { "type": "string" }
|
|
17
|
+
}
|
|
18
|
+
},
|
|
19
|
+
"assignment": {
|
|
20
|
+
"type": "object",
|
|
21
|
+
"required": ["title"],
|
|
22
|
+
"properties": {
|
|
23
|
+
"title": { "type": "string" },
|
|
24
|
+
"description": { "type": "string" },
|
|
25
|
+
"dueDate": { "type": "string" },
|
|
26
|
+
"maxDrafts": { "type": "integer", "minimum": 1 }
|
|
27
|
+
}
|
|
28
|
+
},
|
|
29
|
+
"rubric": {
|
|
30
|
+
"type": "object",
|
|
31
|
+
"required": ["categories"],
|
|
32
|
+
"properties": {
|
|
33
|
+
"categories": {
|
|
34
|
+
"type": "array",
|
|
35
|
+
"minItems": 1,
|
|
36
|
+
"items": {
|
|
37
|
+
"type": "object",
|
|
38
|
+
"required": ["name", "weight"],
|
|
39
|
+
"properties": {
|
|
40
|
+
"name": { "type": "string", "minLength": 1 },
|
|
41
|
+
"description": { "type": "string" },
|
|
42
|
+
"weight": { "type": "number", "minimum": 0, "maximum": 100 }
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{
|
|
2
|
+
"description": "Score bands shown to students as qualitative levels (dartboard metaphor). min is inclusive; bands are evaluated top-down.",
|
|
3
|
+
"levels": [
|
|
4
|
+
{ "min": 90, "label": "On the bullseye", "color": "teal" },
|
|
5
|
+
{ "min": 80, "label": "Closing in", "color": "teal" },
|
|
6
|
+
{ "min": 70, "label": "On the board", "color": "amber" },
|
|
7
|
+
{ "min": 60, "label": "Finding the range", "color": "amber" },
|
|
8
|
+
{ "min": 0, "label": "Take another aim", "color": "red" }
|
|
9
|
+
],
|
|
10
|
+
"colorBands": [
|
|
11
|
+
{ "min": 80, "color": "teal" },
|
|
12
|
+
{ "min": 60, "color": "amber" },
|
|
13
|
+
{ "min": 0, "color": "red" }
|
|
14
|
+
]
|
|
15
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Access to the vendored server↔desktop contract files.
|
|
2
|
+
|
|
3
|
+
In an installed wheel these live in ``feedforward_practice/contracts/``
|
|
4
|
+
(vendored at build time from the repo's ``shared/``); in a repo checkout we
|
|
5
|
+
fall back to ``../../shared`` so editable installs and tests use the live
|
|
6
|
+
files.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import pathlib
|
|
11
|
+
from functools import cache
|
|
12
|
+
|
|
13
|
+
_VENDORED = pathlib.Path(__file__).parent / "contracts"
|
|
14
|
+
_REPO_SHARED = pathlib.Path(__file__).resolve().parents[3] / "shared"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _contract_path(name: str) -> pathlib.Path:
|
|
18
|
+
vendored = _VENDORED / name
|
|
19
|
+
if vendored.exists():
|
|
20
|
+
return vendored
|
|
21
|
+
return _REPO_SHARED / name
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@cache
|
|
25
|
+
def levels_spec() -> dict:
|
|
26
|
+
with open(_contract_path("levels.json")) as f:
|
|
27
|
+
return json.load(f)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@cache
|
|
31
|
+
def rubric_schema() -> dict:
|
|
32
|
+
with open(_contract_path("ffrubric.schema.json")) as f:
|
|
33
|
+
return json.load(f)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@cache
|
|
37
|
+
def prompt_contract() -> str:
|
|
38
|
+
return _contract_path("feedback-prompt.md").read_text()
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def level_for(score: float) -> dict:
|
|
42
|
+
"""Qualitative level (label + colour) for a 0-100 score."""
|
|
43
|
+
spec = levels_spec()
|
|
44
|
+
label = next(
|
|
45
|
+
(lv["label"] for lv in spec["levels"] if score >= lv["min"]),
|
|
46
|
+
spec["levels"][-1]["label"],
|
|
47
|
+
)
|
|
48
|
+
color = next(
|
|
49
|
+
(b["color"] for b in spec["colorBands"] if score >= b["min"]),
|
|
50
|
+
spec["colorBands"][-1]["color"],
|
|
51
|
+
)
|
|
52
|
+
return {"label": label, "color": color}
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
"""The practice feedback engine: rubric + draft -> prompt -> parsed feedback.
|
|
2
|
+
|
|
3
|
+
Pure functions over plain dicts — no database, no accounts. The prompt
|
|
4
|
+
skeleton and JSON response contract come from the vendored shared contract
|
|
5
|
+
(see contracts.py); providers.py performs the actual model call.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
import re
|
|
10
|
+
import statistics
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from feedforward_practice.contracts import level_for, prompt_contract
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class RubricError(ValueError):
|
|
17
|
+
"""The .ffrubric payload is not usable."""
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def validate_rubric(rubric: dict) -> dict:
|
|
21
|
+
"""Light structural validation of a parsed .ffrubric payload.
|
|
22
|
+
|
|
23
|
+
(Full JSON-schema validation happens app-side where jsonschema is
|
|
24
|
+
available; the engine only needs the fields it consumes.)
|
|
25
|
+
"""
|
|
26
|
+
if not isinstance(rubric, dict) or rubric.get("kind") != "ffrubric":
|
|
27
|
+
raise RubricError("Not a .ffrubric file")
|
|
28
|
+
cats = rubric.get("rubric", {}).get("categories") or []
|
|
29
|
+
if not cats:
|
|
30
|
+
raise RubricError("Rubric has no categories")
|
|
31
|
+
for cat in cats:
|
|
32
|
+
if not cat.get("name"):
|
|
33
|
+
raise RubricError("Rubric category missing a name")
|
|
34
|
+
return rubric
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def build_prompt(rubric: dict, draft_text: str) -> dict:
|
|
38
|
+
"""Render the shared prompt contract into {system, user} messages."""
|
|
39
|
+
contract = prompt_contract()
|
|
40
|
+
system = _section(contract, "System")
|
|
41
|
+
user_template = _section(contract, "User")
|
|
42
|
+
|
|
43
|
+
assignment = rubric.get("assignment", {})
|
|
44
|
+
categories = rubric["rubric"]["categories"]
|
|
45
|
+
rubric_lines = "\n".join(
|
|
46
|
+
f"- {c['name']} ({c.get('weight', 0):.0f}%): {c.get('description', '')}"
|
|
47
|
+
for c in categories
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
user = (
|
|
51
|
+
user_template.replace("{{assignment_title}}", assignment.get("title", ""))
|
|
52
|
+
.replace("{{assignment_description}}", assignment.get("description", ""))
|
|
53
|
+
.replace("{{rubric_categories}}", rubric_lines)
|
|
54
|
+
.replace("{{submission_text}}", draft_text)
|
|
55
|
+
)
|
|
56
|
+
return {"system": system.strip(), "user": user.strip()}
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _section(contract: str, heading: str) -> str:
|
|
60
|
+
"""Extract the System or User block from the prompt contract.
|
|
61
|
+
|
|
62
|
+
Only ``## System`` and ``## User`` are structural markers; any other
|
|
63
|
+
``##`` headings (Assignment Context, Instructions, …) are part of the
|
|
64
|
+
prompt text itself and must be preserved.
|
|
65
|
+
"""
|
|
66
|
+
if heading == "System":
|
|
67
|
+
m = re.search(r"^## System\n(.*?)(?=^## User)", contract, re.M | re.S)
|
|
68
|
+
elif heading == "User":
|
|
69
|
+
m = re.search(r"^## User\n(.*)", contract, re.M | re.S)
|
|
70
|
+
else:
|
|
71
|
+
m = None
|
|
72
|
+
if not m:
|
|
73
|
+
raise RuntimeError(f"prompt contract is missing the {heading} section")
|
|
74
|
+
return m.group(1)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def parse_response(raw: str, rubric: dict) -> dict:
|
|
78
|
+
"""Parse a model response against the JSON contract.
|
|
79
|
+
|
|
80
|
+
Tolerates code fences and stray prose around the JSON object. Categories
|
|
81
|
+
are matched to rubric names case-insensitively; unmatched model
|
|
82
|
+
categories are dropped, missing ones reported.
|
|
83
|
+
"""
|
|
84
|
+
text = raw.strip()
|
|
85
|
+
fence = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", text, re.S)
|
|
86
|
+
if fence:
|
|
87
|
+
text = fence.group(1)
|
|
88
|
+
else:
|
|
89
|
+
start, end = text.find("{"), text.rfind("}")
|
|
90
|
+
if start == -1 or end <= start:
|
|
91
|
+
raise ValueError("No JSON object found in model response")
|
|
92
|
+
text = text[start : end + 1]
|
|
93
|
+
|
|
94
|
+
data = json.loads(text)
|
|
95
|
+
wanted = {
|
|
96
|
+
c["name"].strip().lower(): c["name"] for c in rubric["rubric"]["categories"]
|
|
97
|
+
}
|
|
98
|
+
categories = []
|
|
99
|
+
for cat in data.get("categories", []):
|
|
100
|
+
key = str(cat.get("name", "")).strip().lower()
|
|
101
|
+
if key not in wanted:
|
|
102
|
+
continue
|
|
103
|
+
score = max(0.0, min(100.0, float(cat.get("score", 0))))
|
|
104
|
+
categories.append(
|
|
105
|
+
{
|
|
106
|
+
"name": wanted[key],
|
|
107
|
+
"score": score,
|
|
108
|
+
"feedback": str(cat.get("feedback", "")).strip(),
|
|
109
|
+
"strengths": [str(s) for s in cat.get("strengths", [])][:3],
|
|
110
|
+
"improvements": [str(s) for s in cat.get("improvements", [])][:3],
|
|
111
|
+
}
|
|
112
|
+
)
|
|
113
|
+
return {
|
|
114
|
+
"overall_feedback": str(data.get("overall_feedback", "")).strip(),
|
|
115
|
+
"categories": categories,
|
|
116
|
+
"missing_categories": [
|
|
117
|
+
name
|
|
118
|
+
for key, name in wanted.items()
|
|
119
|
+
if key not in {c["name"].strip().lower() for c in categories}
|
|
120
|
+
],
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def aggregate_runs(runs: list[dict], rubric: dict) -> dict:
|
|
125
|
+
"""Aggregate 1..N parsed runs into the final feedback payload.
|
|
126
|
+
|
|
127
|
+
Scores: mean per category. Text: taken from the run whose category score
|
|
128
|
+
is closest to the aggregated mean (a 'most representative run' rule —
|
|
129
|
+
deliberately simple, mirroring the server's mean aggregation default).
|
|
130
|
+
"""
|
|
131
|
+
if not runs:
|
|
132
|
+
raise ValueError("No successful runs to aggregate")
|
|
133
|
+
|
|
134
|
+
weights = {
|
|
135
|
+
c["name"]: float(c.get("weight", 0)) for c in rubric["rubric"]["categories"]
|
|
136
|
+
}
|
|
137
|
+
by_name: dict[str, list[dict]] = {}
|
|
138
|
+
for run in runs:
|
|
139
|
+
for cat in run["categories"]:
|
|
140
|
+
by_name.setdefault(cat["name"], []).append(cat)
|
|
141
|
+
|
|
142
|
+
categories: list[dict[str, Any]] = []
|
|
143
|
+
for name, cats in by_name.items():
|
|
144
|
+
mean = statistics.fmean(c["score"] for c in cats)
|
|
145
|
+
representative = min(cats, key=lambda c: abs(c["score"] - mean))
|
|
146
|
+
categories.append(
|
|
147
|
+
{
|
|
148
|
+
"name": name,
|
|
149
|
+
"weight": weights.get(name, 0),
|
|
150
|
+
"score": round(mean, 1),
|
|
151
|
+
"level": level_for(mean),
|
|
152
|
+
"feedback": representative["feedback"],
|
|
153
|
+
"strengths": representative["strengths"],
|
|
154
|
+
"improvements": representative["improvements"],
|
|
155
|
+
"runs": len(cats),
|
|
156
|
+
}
|
|
157
|
+
)
|
|
158
|
+
# Keep rubric order
|
|
159
|
+
order = [c["name"] for c in rubric["rubric"]["categories"]]
|
|
160
|
+
categories.sort(key=lambda c: order.index(c["name"]) if c["name"] in order else 99)
|
|
161
|
+
|
|
162
|
+
total_weight = sum(c["weight"] for c in categories)
|
|
163
|
+
if total_weight > 0:
|
|
164
|
+
overall = sum(c["score"] * c["weight"] for c in categories) / total_weight
|
|
165
|
+
else:
|
|
166
|
+
overall = statistics.fmean(c["score"] for c in categories)
|
|
167
|
+
|
|
168
|
+
return {
|
|
169
|
+
"overall": {"score": round(overall, 1), "level": level_for(overall)},
|
|
170
|
+
"overall_feedback": next(
|
|
171
|
+
(r["overall_feedback"] for r in runs if r["overall_feedback"]), ""
|
|
172
|
+
),
|
|
173
|
+
"categories": categories,
|
|
174
|
+
"runs": len(runs),
|
|
175
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""Family manifest — feedforward-practice is a lens, not an analyser."""
|
|
2
|
+
|
|
3
|
+
from lens_contract import make_manifest
|
|
4
|
+
|
|
5
|
+
MANIFEST = make_manifest(
|
|
6
|
+
name="feedforward-practice",
|
|
7
|
+
role="lens",
|
|
8
|
+
accepts=["text"],
|
|
9
|
+
produces="rubric-aligned formative feedback (practice)",
|
|
10
|
+
auto_routable=False,
|
|
11
|
+
)
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"""Model providers for practice feedback.
|
|
2
|
+
|
|
3
|
+
One OpenAI-compatible chat client covers all three supported setups:
|
|
4
|
+
|
|
5
|
+
- **Local Ollama** — base_url http://localhost:11434/v1, no key
|
|
6
|
+
- **Remote Ollama** (e.g. a department GPU box behind a proxy) — any base
|
|
7
|
+
URL plus a bearer key
|
|
8
|
+
- **BYOK** — any OpenAI-compatible endpoint (OpenAI, Anthropic's
|
|
9
|
+
compatibility endpoint, OpenRouter, Groq, ...) with the student's own key
|
|
10
|
+
|
|
11
|
+
Configuration via arguments or environment:
|
|
12
|
+
FEEDFORWARD_PRACTICE_BASE_URL, FEEDFORWARD_PRACTICE_API_KEY,
|
|
13
|
+
FEEDFORWARD_PRACTICE_MODEL.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
import os
|
|
17
|
+
from dataclasses import dataclass
|
|
18
|
+
|
|
19
|
+
import httpx
|
|
20
|
+
|
|
21
|
+
DEFAULT_BASE_URL = "http://localhost:11434/v1" # local Ollama
|
|
22
|
+
DEFAULT_TIMEOUT = 180.0
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class ProviderError(RuntimeError):
|
|
26
|
+
"""The model endpoint could not produce a completion."""
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass
|
|
30
|
+
class ProviderConfig:
|
|
31
|
+
base_url: str = DEFAULT_BASE_URL
|
|
32
|
+
api_key: str = ""
|
|
33
|
+
model: str = ""
|
|
34
|
+
|
|
35
|
+
@classmethod
|
|
36
|
+
def from_env(cls) -> "ProviderConfig":
|
|
37
|
+
return cls(
|
|
38
|
+
base_url=os.environ.get("FEEDFORWARD_PRACTICE_BASE_URL", DEFAULT_BASE_URL),
|
|
39
|
+
api_key=os.environ.get("FEEDFORWARD_PRACTICE_API_KEY", ""),
|
|
40
|
+
model=os.environ.get("FEEDFORWARD_PRACTICE_MODEL", ""),
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
def merged(self, overrides: dict | None) -> "ProviderConfig":
|
|
44
|
+
overrides = overrides or {}
|
|
45
|
+
return ProviderConfig(
|
|
46
|
+
base_url=(overrides.get("base_url") or self.base_url).rstrip("/"),
|
|
47
|
+
api_key=overrides.get("api_key") or self.api_key,
|
|
48
|
+
model=overrides.get("model") or self.model,
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _headers(config: ProviderConfig) -> dict:
|
|
53
|
+
headers = {"Content-Type": "application/json"}
|
|
54
|
+
if config.api_key:
|
|
55
|
+
headers["Authorization"] = f"Bearer {config.api_key}"
|
|
56
|
+
return headers
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def list_models(config: ProviderConfig) -> list[str]:
|
|
60
|
+
"""Model ids from the endpoint's /models (works for Ollama and OpenAI)."""
|
|
61
|
+
try:
|
|
62
|
+
resp = httpx.get(
|
|
63
|
+
f"{config.base_url}/models", headers=_headers(config), timeout=15.0
|
|
64
|
+
)
|
|
65
|
+
resp.raise_for_status()
|
|
66
|
+
return [m["id"] for m in resp.json().get("data", [])]
|
|
67
|
+
except httpx.HTTPError as e:
|
|
68
|
+
raise ProviderError(f"Could not list models at {config.base_url}: {e}") from e
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def chat(config: ProviderConfig, system: str, user: str) -> str:
|
|
72
|
+
"""One chat completion; returns the assistant text."""
|
|
73
|
+
if not config.model:
|
|
74
|
+
raise ProviderError("No model configured")
|
|
75
|
+
payload = {
|
|
76
|
+
"model": config.model,
|
|
77
|
+
"messages": [
|
|
78
|
+
{"role": "system", "content": system},
|
|
79
|
+
{"role": "user", "content": user},
|
|
80
|
+
],
|
|
81
|
+
"temperature": 0.4,
|
|
82
|
+
}
|
|
83
|
+
try:
|
|
84
|
+
resp = httpx.post(
|
|
85
|
+
f"{config.base_url}/chat/completions",
|
|
86
|
+
headers=_headers(config),
|
|
87
|
+
json=payload,
|
|
88
|
+
timeout=DEFAULT_TIMEOUT,
|
|
89
|
+
)
|
|
90
|
+
resp.raise_for_status()
|
|
91
|
+
data = resp.json()
|
|
92
|
+
return data["choices"][0]["message"]["content"]
|
|
93
|
+
except httpx.HTTPStatusError as e:
|
|
94
|
+
detail = e.response.text[:300]
|
|
95
|
+
raise ProviderError(
|
|
96
|
+
f"Model endpoint returned {e.response.status_code}: {detail}"
|
|
97
|
+
) from e
|
|
98
|
+
except (httpx.HTTPError, KeyError, IndexError, ValueError) as e:
|
|
99
|
+
raise ProviderError(f"Model call failed: {e}") from e
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""Top-level orchestration: rubric + draft + provider -> aggregated feedback."""
|
|
2
|
+
|
|
3
|
+
from feedforward_practice import engine
|
|
4
|
+
from feedforward_practice.providers import ProviderConfig, ProviderError, chat
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def practice_feedback(
|
|
8
|
+
rubric: dict,
|
|
9
|
+
draft_text: str,
|
|
10
|
+
provider: ProviderConfig,
|
|
11
|
+
num_runs: int = 1,
|
|
12
|
+
) -> dict:
|
|
13
|
+
"""Run the full practice loop.
|
|
14
|
+
|
|
15
|
+
Returns the aggregated feedback payload plus run diagnostics. Raises
|
|
16
|
+
RubricError / ProviderError / ValueError for unusable inputs; partial
|
|
17
|
+
run failures are tolerated as long as one run parses.
|
|
18
|
+
"""
|
|
19
|
+
rubric = engine.validate_rubric(rubric)
|
|
20
|
+
if not draft_text or not draft_text.strip():
|
|
21
|
+
raise ValueError("Draft is empty")
|
|
22
|
+
num_runs = max(1, min(int(num_runs), 5))
|
|
23
|
+
|
|
24
|
+
prompt = engine.build_prompt(rubric, draft_text)
|
|
25
|
+
|
|
26
|
+
parsed_runs: list[dict] = []
|
|
27
|
+
errors: list[str] = []
|
|
28
|
+
for _ in range(num_runs):
|
|
29
|
+
try:
|
|
30
|
+
raw = chat(provider, prompt["system"], prompt["user"])
|
|
31
|
+
parsed_runs.append(engine.parse_response(raw, rubric))
|
|
32
|
+
except (ProviderError, ValueError) as e: # noqa: PERF203
|
|
33
|
+
errors.append(str(e))
|
|
34
|
+
|
|
35
|
+
if not parsed_runs:
|
|
36
|
+
raise ProviderError(
|
|
37
|
+
"All model runs failed: " + (errors[0] if errors else "unknown error")
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
result = engine.aggregate_runs(parsed_runs, rubric)
|
|
41
|
+
result["failed_runs"] = len(errors)
|
|
42
|
+
result["word_count"] = len(draft_text.split())
|
|
43
|
+
return result
|