quizforge 0.1.0__tar.gz

quizforge-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,41 @@
+name: publish
+
+# Publishes to PyPI via Trusted Publishing (OIDC) — no API token stored.
+# Configure the publisher once at https://pypi.org/manage/account/publishing/
+# (project: quizforge, workflow: publish.yml, environment: pypi), then push a
+# version tag:  git tag v0.1.0 && git push origin v0.1.0
+
+on:
+  push:
+    tags: ["v*"]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Build sdist + wheel
+        run: |
+          python -m pip install --upgrade pip build
+          python -m build
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write  # required for Trusted Publishing
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

quizforge-0.1.0/.github/workflows/tests.yml

@@ -0,0 +1,24 @@
+name: tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+      - name: Run tests
+        run: pytest -q

quizforge-0.1.0/.gitignore

@@ -0,0 +1,11 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+.eggs/
+build/
+dist/
+.pytest_cache/
+.venv/
+venv/
+.env
+*.xlsx

quizforge-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Vinay Vobbilichetty
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

quizforge-0.1.0/PKG-INFO

@@ -0,0 +1,104 @@
+Metadata-Version: 2.4
+Name: quizforge
+Version: 0.1.0
+Summary: Generate a deep, mixed-format question bank from source material and grade it — deterministic where it can, LLM where it must. Bring your own chat model.
+Project-URL: Homepage, https://github.com/vinayvobbili/quizforge
+Project-URL: Source, https://github.com/vinayvobbili/quizforge
+Author: Vinay Vobbilichetty
+License: MIT
+License-File: LICENSE
+Keywords: assessment,education,grading,llm,question-bank,quiz,training
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Education :: Testing
+Requires-Python: >=3.10
+Requires-Dist: pydantic>=2
+Requires-Dist: pyyaml>=6
+Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == 'dev'
+Provides-Extra: openai
+Requires-Dist: langchain-openai>=0.1; extra == 'openai'
+Description-Content-Type: text/markdown
+
+# quizforge
+
+Generate a deep, **mixed-format** question bank from any source material, then grade it — **deterministic where it can, LLM where it must**. Bring your own chat model.
+
+quizforge is the engine behind a training/readiness feature: it drafts far more questions than any single test shows (multiple choice, fill-in-the-blank, match-the-following, short answer, and free-response scenarios), samples a fresh shuffled test on each attempt — so two learners rarely see the same one — and grades every format. MC/fill/match are graded instantly with no model call; open-ended answers are scored 0–1 with coaching feedback by an LLM you provide.
+
+- **Model-agnostic** — pass any LangChain-style chat model (`with_structured_output`). No SDK is bundled.
+- **Deep bank, anti-sharing sampling** — unseen-first, difficulty-spread draws per a configurable blueprint.
+- **Cheap grading** — only open-ended answers cost a model call; everything else is local and free.
+- **Plain dicts in, plain dicts out** — YAML/JSON-friendly, easy to store and template.
+
+## Install
+
+```bash
+pip install quizforge
+```
+
+Bring a chat model from whichever provider you use, e.g.:
+
+```bash
+pip install langchain-openai   # or langchain-anthropic, etc.
+```
+
+## Quickstart
+
+### Generate a bank
+
+```python
+from quizforge import generate_bank
+from langchain_openai import ChatOpenAI
+
+llm = ChatOpenAI(model="gpt-4.1", temperature=0.4)
+
+material = open("citrix_lesson.md").read()
+new_questions = generate_bank(
+    material, llm,
+    targets={"mc": 40, "fill_blank": 20, "match": 12, "short": 16, "freetext": 12},
+    existing=[],                       # pass your current bank to top it up
+    coverage="At least half should be applied incident-response scenarios.",
+)
+# -> list of dicts with id/type/difficulty + per-format fields. Store as you like.
+```
+
+`generate_bank` only produces the *shortfall* to reach `targets`, validates each
+question, and never duplicates an existing prompt — safe to re-run to grow a bank.
+
+### Sample a test
+
+```python
+from quizforge import sample_test, DEFAULT_BLUEPRINT
+
+test = sample_test(bank, blueprint=DEFAULT_BLUEPRINT, seen_ids=already_seen)
+# DEFAULT_BLUEPRINT draws mc8 / fill4 / match2 / short4 / freetext2 = 20, shuffled.
+```
+
+### Grade
+
+```python
+from quizforge import grade_fill_blank, grade_match, grade_open_answer
+
+grade_fill_blank(q, "ICA")                 # {"score": 1.0, "correct": True, ...}
+grade_match(q, {"0": "RDP", "1": "ICA"})   # per-pair partial credit
+grade_open_answer(q, learner_text, llm)    # QuizGrade(score, verdict, feedback, ...) or None
+```
+
+`grade_open_answer` returns `None` if the model was unavailable — exclude that
+question from the attempt's max score rather than penalizing the learner.
+
+## Question shapes
+
+Each question is a dict with `id`, `type`, `difficulty`, `prompt`, plus:
+
+- `mc` — `choices: [str]`, `answer_idx: int`, `explanation: str`
+- `fill_blank` — `accepted_answers: [str]`, `explanation: str`
+- `match` — `pairs: [{left, right}]`, `explanation: str`
+- `short` / `freetext` — `model_answer: str`, `rubric: [str]`
+
+## License
+
+MIT

quizforge-0.1.0/README.md

@@ -0,0 +1,80 @@
+# quizforge
+
+Generate a deep, **mixed-format** question bank from any source material, then grade it — **deterministic where it can, LLM where it must**. Bring your own chat model.
+
+quizforge is the engine behind a training/readiness feature: it drafts far more questions than any single test shows (multiple choice, fill-in-the-blank, match-the-following, short answer, and free-response scenarios), samples a fresh shuffled test on each attempt — so two learners rarely see the same one — and grades every format. MC/fill/match are graded instantly with no model call; open-ended answers are scored 0–1 with coaching feedback by an LLM you provide.
+
+- **Model-agnostic** — pass any LangChain-style chat model (`with_structured_output`). No SDK is bundled.
+- **Deep bank, anti-sharing sampling** — unseen-first, difficulty-spread draws per a configurable blueprint.
+- **Cheap grading** — only open-ended answers cost a model call; everything else is local and free.
+- **Plain dicts in, plain dicts out** — YAML/JSON-friendly, easy to store and template.
+
+## Install
+
+```bash
+pip install quizforge
+```
+
+Bring a chat model from whichever provider you use, e.g.:
+
+```bash
+pip install langchain-openai   # or langchain-anthropic, etc.
+```
+
+## Quickstart
+
+### Generate a bank
+
+```python
+from quizforge import generate_bank
+from langchain_openai import ChatOpenAI
+
+llm = ChatOpenAI(model="gpt-4.1", temperature=0.4)
+
+material = open("citrix_lesson.md").read()
+new_questions = generate_bank(
+    material, llm,
+    targets={"mc": 40, "fill_blank": 20, "match": 12, "short": 16, "freetext": 12},
+    existing=[],                       # pass your current bank to top it up
+    coverage="At least half should be applied incident-response scenarios.",
+)
+# -> list of dicts with id/type/difficulty + per-format fields. Store as you like.
+```
+
+`generate_bank` only produces the *shortfall* to reach `targets`, validates each
+question, and never duplicates an existing prompt — safe to re-run to grow a bank.
+
+### Sample a test
+
+```python
+from quizforge import sample_test, DEFAULT_BLUEPRINT
+
+test = sample_test(bank, blueprint=DEFAULT_BLUEPRINT, seen_ids=already_seen)
+# DEFAULT_BLUEPRINT draws mc8 / fill4 / match2 / short4 / freetext2 = 20, shuffled.
+```
+
+### Grade
+
+```python
+from quizforge import grade_fill_blank, grade_match, grade_open_answer
+
+grade_fill_blank(q, "ICA")                 # {"score": 1.0, "correct": True, ...}
+grade_match(q, {"0": "RDP", "1": "ICA"})   # per-pair partial credit
+grade_open_answer(q, learner_text, llm)    # QuizGrade(score, verdict, feedback, ...) or None
+```
+
+`grade_open_answer` returns `None` if the model was unavailable — exclude that
+question from the attempt's max score rather than penalizing the learner.
+
+## Question shapes
+
+Each question is a dict with `id`, `type`, `difficulty`, `prompt`, plus:
+
+- `mc` — `choices: [str]`, `answer_idx: int`, `explanation: str`
+- `fill_blank` — `accepted_answers: [str]`, `explanation: str`
+- `match` — `pairs: [{left, right}]`, `explanation: str`
+- `short` / `freetext` — `model_answer: str`, `rubric: [str]`
+
+## License
+
+MIT

quizforge-0.1.0/pyproject.toml

@@ -0,0 +1,42 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "quizforge"
+version = "0.1.0"
+description = "Generate a deep, mixed-format question bank from source material and grade it — deterministic where it can, LLM where it must. Bring your own chat model."
+readme = "README.md"
+requires-python = ">=3.10"
+license = { text = "MIT" }
+authors = [{ name = "Vinay Vobbilichetty" }]
+keywords = ["quiz", "assessment", "question-bank", "training", "llm", "grading", "education"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Topic :: Education :: Testing",
+]
+dependencies = [
+    "pydantic>=2",
+    "pyyaml>=6",
+]
+
+[project.optional-dependencies]
+openai = ["langchain-openai>=0.1"]
+dev = ["pytest>=7"]
+
+[project.scripts]
+quizforge = "quizforge.cli:main"
+
+[project.urls]
+Homepage = "https://github.com/vinayvobbili/quizforge"
+Source = "https://github.com/vinayvobbili/quizforge"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/quizforge"]
+
+[tool.pytest.ini_options]
+pythonpath = ["src"]
+testpaths = ["tests"]

quizforge-0.1.0/src/quizforge/__init__.py

@@ -0,0 +1,99 @@
+"""quizforge — generate a deep, mixed-format question bank from source material
+and grade it. Deterministic where it can, LLM where it must. Bring your own
+chat model.
+
+Public API
+----------
+Generate:
+    generate_bank(material, llm, *, targets=..., existing=...) -> list[dict]
+Sample:
+    sample_test(questions, blueprint=..., seen_ids=...) -> list[dict]
+    pick_spread(pool, want, seen_ids) -> list[dict]
+    DEFAULT_BLUEPRINT
+Grade:
+    grade_fill_blank(question, user_answer) -> dict
+    grade_match(question, selections) -> dict
+    grade_open_answer(question, user_answer, llm) -> QuizGrade | None
+    QuizGrade
+Integrity:
+    assess_speed(*, elapsed_seconds, question_types, passed) -> IntegrityFlag
+    expected_min_seconds(question_types) -> float
+    IntegrityFlag
+Certificate:
+    make_certificate(*, learner_id, ..., score_pct, awarded_on) -> Certificate
+    verification_code(...) -> str   |   verify(certificate) -> bool
+    Certificate
+Utilities:
+    normalize(text) -> str
+    structured_output(llm, schema) -> chain
+"""
+
+from .bank import Bank
+from .certificate import (
+    Certificate,
+    is_eligible,
+    level_for,
+    make_certificate,
+    verification_code,
+    verify,
+)
+from .generate import (
+    DEFAULT_COVERAGE,
+    DEFAULT_DIFF_SPLIT,
+    DEFAULT_SYSTEM,
+    DEFAULT_TARGETS,
+    generate_bank,
+)
+from .grade import (
+    DEFAULT_GRADE_SYSTEM,
+    QuizGrade,
+    grade_fill_blank,
+    grade_match,
+    grade_open_answer,
+)
+from .integrity import (
+    DEFAULT_MIN_SECONDS_PER_TYPE,
+    IntegrityFlag,
+    assess_speed,
+    expected_min_seconds,
+)
+from .llm import structured_output
+from .sample import DEFAULT_BLUEPRINT, DIFFICULTY_ORDER, pick_spread, sample_test
+from .schemas import DIFFICULTIES, FORMAT_KEYS, FORMATS
+from .text import normalize
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "Bank",
+    "generate_bank",
+    "sample_test",
+    "pick_spread",
+    "grade_fill_blank",
+    "grade_match",
+    "grade_open_answer",
+    "QuizGrade",
+    "assess_speed",
+    "expected_min_seconds",
+    "IntegrityFlag",
+    "DEFAULT_MIN_SECONDS_PER_TYPE",
+    "Certificate",
+    "make_certificate",
+    "verification_code",
+    "verify",
+    "is_eligible",
+    "level_for",
+    "normalize",
+    "structured_output",
+    "DEFAULT_BLUEPRINT",
+    "DEFAULT_TARGETS",
+    "DEFAULT_DIFF_SPLIT",
+    "DEFAULT_SYSTEM",
+    "DEFAULT_COVERAGE",
+    "DEFAULT_GRADE_SYSTEM",
+    "DIFFICULTY_ORDER",
+    "DIFFICULTIES",
+    "FORMAT_KEYS",
+    "FORMATS",
+    "__version__",
+]

quizforge-0.1.0/src/quizforge/bank.py

@@ -0,0 +1,89 @@
+"""Bank — a YAML-backed question bank with grow/sample/save convenience.
+
+A bank file is YAML with (at least) a ``questions:`` list; any other top-level
+keys are metadata, carried through untouched on save. An optional ``material:``
+key holds the source text used to ground generation (or pass it explicitly).
+
+Comments are NOT preserved on save (clean round-trip via PyYAML). If you keep a
+heavily-commented bank file, generate into memory and write where you control
+the formatting instead of calling ``save``.
+"""
+
+from collections import Counter
+from pathlib import Path
+from typing import List, Optional, Union
+
+import yaml
+
+from .generate import generate_bank
+from .sample import sample_test
+
+PathLike = Union[str, Path]
+
+
+class Bank:
+    """Load, grow, sample, and save a question bank."""
+
+    def __init__(self, questions: Optional[List[dict]] = None, meta: Optional[dict] = None):
+        self.questions: List[dict] = list(questions or [])
+        self.meta: dict = dict(meta or {})
+
+    # ----- construction -----------------------------------------------------
+    @classmethod
+    def load(cls, path: PathLike) -> "Bank":
+        data = yaml.safe_load(Path(path).read_text()) or {}
+        if not isinstance(data, dict):
+            raise ValueError(f"{path}: expected a YAML mapping, got {type(data).__name__}")
+        questions = data.pop("questions", []) or []
+        return cls(questions=questions, meta=data)
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "Bank":
+        data = dict(data or {})
+        return cls(questions=data.pop("questions", []) or [], meta=data)
+
+    # ----- inspection -------------------------------------------------------
+    @property
+    def material(self) -> str:
+        return self.meta.get("material", "") or ""
+
+    def counts_by_type(self) -> dict:
+        return dict(Counter(q.get("type", "mc") for q in self.questions))
+
+    def counts_by_difficulty(self) -> dict:
+        return dict(Counter(q.get("difficulty", "medium") for q in self.questions))
+
+    def __len__(self) -> int:
+        return len(self.questions)
+
+    # ----- mutation ---------------------------------------------------------
+    def add(self, new_questions: List[dict]) -> "Bank":
+        self.questions.extend(new_questions)
+        return self
+
+    def grow(self, llm, *, targets: Optional[dict] = None,
+             material: Optional[str] = None, **kwargs) -> List[dict]:
+        """Generate the shortfall to reach ``targets`` and append it in place.
+
+        Grounds on ``material`` if given, else the bank's ``material`` metadata.
+        Returns just the newly added questions.
+        """
+        mat = material if material is not None else self.material
+        if not (mat or "").strip():
+            raise ValueError("no material to ground generation — pass material=... "
+                             "or set a 'material:' key in the bank file")
+        new = generate_bank(mat, llm, targets=targets, existing=self.questions, **kwargs)
+        self.add(new)
+        return new
+
+    def sample(self, blueprint: Optional[dict] = None, seen_ids=(), rng=None) -> List[dict]:
+        return sample_test(self.questions, blueprint=blueprint, seen_ids=seen_ids, rng=rng)
+
+    # ----- persistence ------------------------------------------------------
+    def to_dict(self) -> dict:
+        return {**self.meta, "questions": self.questions}
+
+    def save(self, path: PathLike) -> None:
+        with open(path, "w") as f:
+            yaml.safe_dump(self.to_dict(), f, sort_keys=False, allow_unicode=True,
+                           width=100, default_flow_style=False)

quizforge-0.1.0/src/quizforge/certificate.py

@@ -0,0 +1,84 @@
+"""Completion certificates — the data + a tamper-evident verification code.
+
+quizforge owns the *facts* of a certificate (who, which topic, what level, when)
+and a deterministic verification code derived from them, so any consumer can
+re-derive the code to confirm a certificate wasn't altered. Rendering — PDF,
+HTML, image — is deliberately left to the consumer, where the branding lives, so
+this module stays dependency-light (pydantic only) and clockless (the caller
+supplies ``awarded_on``, keeping generation reproducible).
+"""
+
+import hashlib
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+PASS = "passed"
+DISTINCTION = "distinction"
+_SEP = "\x1f"  # unit separator — unlikely to appear in any field
+
+
+class Certificate(BaseModel):
+    """An earned completion certificate's facts + its verification code."""
+
+    learner_id: str = Field(description="Stable learner identity (e.g. email).")
+    learner_name: str = Field(description="Display name to print on the certificate.")
+    topic_id: str = Field(description="Topic/lesson identifier.")
+    topic_title: str = Field(description="Human-readable topic title.")
+    score_pct: int = Field(description="Best score as a 0-100 percentage.")
+    level: str = Field(description="'passed' or 'distinction'.")
+    awarded_on: str = Field(description="Award date as the caller's display string.")
+    verification_code: str = Field(description="Deterministic, tamper-evident code.")
+
+
+def verification_code(*, learner_id: str, topic_id: str, awarded_on: str, level: str,
+                      score_pct: int, secret: str = "", prefix: str = "QF") -> str:
+    """Derive a short, stable verification code from a certificate's fields.
+
+    Same fields (+ same ``secret``) always yield the same code; changing any
+    field changes the code, so a printed certificate can be checked against the
+    record. With a non-empty ``secret`` the code is unforgeable without it.
+    """
+    raw = _SEP.join([learner_id.lower().strip(), topic_id, awarded_on, level,
+                     str(score_pct), secret])
+    digest = hashlib.sha256(raw.encode("utf-8")).hexdigest().upper()
+    return f"{prefix}-{digest[:4]}-{digest[4:8]}"
+
+
+def is_eligible(score_pct: int, pass_threshold: float = 0.6) -> bool:
+    """Whether a score earns a certificate at all."""
+    return score_pct / 100.0 >= pass_threshold
+
+
+def level_for(score_pct: int, distinction_threshold: float = 0.8) -> str:
+    """'distinction' at/above the threshold, else 'passed'."""
+    return DISTINCTION if score_pct / 100.0 >= distinction_threshold else PASS
+
+
+def make_certificate(*, learner_id: str, learner_name: str, topic_id: str, topic_title: str,
+                     score_pct: int, awarded_on: str, pass_threshold: float = 0.6,
+                     distinction_threshold: float = 0.8, secret: str = "",
+                     prefix: str = "QF") -> Certificate:
+    """Build a :class:`Certificate` for an eligible score.
+
+    Raises ``ValueError`` if ``score_pct`` is below ``pass_threshold`` — there is
+    no certificate for a non-pass.
+    """
+    if not is_eligible(score_pct, pass_threshold):
+        raise ValueError(f"score {score_pct}% is below the pass threshold "
+                         f"{round(pass_threshold * 100)}% — no certificate")
+    level = level_for(score_pct, distinction_threshold)
+    code = verification_code(learner_id=learner_id, topic_id=topic_id, awarded_on=awarded_on,
+                             level=level, score_pct=score_pct, secret=secret, prefix=prefix)
+    return Certificate(learner_id=learner_id, learner_name=learner_name, topic_id=topic_id,
+                       topic_title=topic_title, score_pct=score_pct, level=level,
+                       awarded_on=awarded_on, verification_code=code)
+
+
+def verify(certificate: Certificate, secret: str = "", prefix: str = "QF") -> bool:
+    """Re-derive the code from the certificate's fields and confirm it matches."""
+    expected = verification_code(
+        learner_id=certificate.learner_id, topic_id=certificate.topic_id,
+        awarded_on=certificate.awarded_on, level=certificate.level,
+        score_pct=certificate.score_pct, secret=secret, prefix=prefix)
+    return expected == certificate.verification_code