f1-temporal-bench 0.1.0__tar.gz

f1_temporal_bench-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,19 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - run: pip install -e ".[dev]"
+      - run: pytest
+      - run: f1-temporal-bench validate

f1_temporal_bench-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,27 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - "v*"
+
+jobs:
+  build-and-publish:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write   # needed for PyPI trusted publishing
+      contents: write    # needed to create the GitHub release
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - run: pip install build
+      - run: python -m build
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v2
+        with:
+          files: dist/*
+          generate_release_notes: true

f1_temporal_bench-0.1.0/.gitignore

@@ -0,0 +1,7 @@
+.venv/
+__pycache__/
+*.pyc
+dist/
+build/
+*.egg-info/
+.pytest_cache/

f1_temporal_bench-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 YOUR_NAME
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

f1_temporal_bench-0.1.0/PKG-INFO

@@ -0,0 +1,68 @@
+Metadata-Version: 2.5
+Name: f1-temporal-bench
+Version: 0.1.0
+Summary: A benchmark for evaluating temporal knowledge and hallucination in language models, using Formula 1 as a fast-changing ground truth domain.
+Project-URL: Homepage, https://github.com/YOUR_USERNAME/f1-temporal-bench
+Project-URL: Issues, https://github.com/YOUR_USERNAME/f1-temporal-bench/issues
+Author: Your Name
+License: MIT
+License-File: LICENSE
+Keywords: benchmark,formula1,huggingface,llm-evaluation,nlp
+Requires-Python: >=3.10
+Requires-Dist: click>=8.1
+Requires-Dist: huggingface-hub>=0.24
+Requires-Dist: requests>=2.31
+Requires-Dist: rich>=13.7
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.5; extra == 'dev'
+Provides-Extra: local-models
+Requires-Dist: torch>=2.2; extra == 'local-models'
+Requires-Dist: transformers>=4.40; extra == 'local-models'
+Description-Content-Type: text/markdown
+
+# F1 Temporal Knowledge Benchmark
+
+A benchmark for measuring how well language models track fast-changing,
+verifiable facts — using Formula 1 standings, results, and records as
+ground truth. F1 facts change every ~2 weeks during the season, making
+this domain a natural, continuously-refreshed test of temporal knowledge
+and hallucination in LLMs.
+
+## Why F1?
+
+Most knowledge benchmarks go stale the moment a model is trained. F1
+gives us a domain where "true as of last week" and "true as of last year"
+are meaningfully different — a clean way to measure whether a model
+knows what it knows, or confidently guesses.
+
+## Install
+
+\`\`\`bash
+pip install f1-temporal-bench
+\`\`\`
+
+## Usage
+
+\`\`\`bash
+f1-temporal-bench validate
+f1-temporal-bench run --model meta-llama/Llama-3.2-1B-Instruct
+\`\`\`
+
+## Metrics
+
+- **Accuracy** — exact/alias match against ground truth
+- **Confidently wrong rate** — model gives a specific, wrong answer
+  instead of hedging
+- **Refusal rate** — model declines to answer / says it doesn't know
+
+## Contributing questions
+
+Add new rows to \`data/questions.jsonl\` after each race weekend. Run
+\`f1-temporal-bench validate\` before committing.
+
+## Roadmap
+
+- [ ] Automated dataset updates via GitHub Actions after each race
+- [ ] Multi-model leaderboard published to GitHub Pages
+- [ ] Dataset published on the Hugging Face Hub with versioned season snapshots

f1_temporal_bench-0.1.0/README.md

@@ -0,0 +1,45 @@
+# F1 Temporal Knowledge Benchmark
+
+A benchmark for measuring how well language models track fast-changing,
+verifiable facts — using Formula 1 standings, results, and records as
+ground truth. F1 facts change every ~2 weeks during the season, making
+this domain a natural, continuously-refreshed test of temporal knowledge
+and hallucination in LLMs.
+
+## Why F1?
+
+Most knowledge benchmarks go stale the moment a model is trained. F1
+gives us a domain where "true as of last week" and "true as of last year"
+are meaningfully different — a clean way to measure whether a model
+knows what it knows, or confidently guesses.
+
+## Install
+
+\`\`\`bash
+pip install f1-temporal-bench
+\`\`\`
+
+## Usage
+
+\`\`\`bash
+f1-temporal-bench validate
+f1-temporal-bench run --model meta-llama/Llama-3.2-1B-Instruct
+\`\`\`
+
+## Metrics
+
+- **Accuracy** — exact/alias match against ground truth
+- **Confidently wrong rate** — model gives a specific, wrong answer
+  instead of hedging
+- **Refusal rate** — model declines to answer / says it doesn't know
+
+## Contributing questions
+
+Add new rows to \`data/questions.jsonl\` after each race weekend. Run
+\`f1-temporal-bench validate\` before committing.
+
+## Roadmap
+
+- [ ] Automated dataset updates via GitHub Actions after each race
+- [ ] Multi-model leaderboard published to GitHub Pages
+- [ ] Dataset published on the Hugging Face Hub with versioned season snapshots

f1_temporal_bench-0.1.0/data/questions.jsonl

@@ -0,0 +1,3 @@
+{"id": "2024-r08-wdc-leader", "date": "2024-05-27", "question": "Who was leading the F1 drivers' championship after round 8 of the 2024 season?", "answer": "Max Verstappen", "aliases": ["Verstappen"], "category": "wdc_standings", "season": 2024, "round": 8}
+{"id": "2023-constructors-champion", "date": "2023-11-26", "question": "Which team won the F1 constructors' championship in 2023?", "answer": "Red Bull Racing", "aliases": ["Red Bull"], "category": "constructors_champion", "season": 2023, "round": null}
+{"id": "2021-drivers-champion", "date": "2021-12-12", "question": "Who won the F1 drivers' championship in 2021?", "answer": "Max Verstappen", "aliases": ["Verstappen"], "category": "wdc_champion", "season": 2021, "round": null}

f1_temporal_bench-0.1.0/pyproject.toml

@@ -0,0 +1,33 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "f1-temporal-bench"
+version = "0.1.0"
+description = "A benchmark for evaluating temporal knowledge and hallucination in language models, using Formula 1 as a fast-changing ground truth domain."
+readme = "README.md"
+requires-python = ">=3.10"
+license = { text = "MIT" }
+authors = [{ name = "Your Name" }]
+keywords = ["nlp", "llm-evaluation", "benchmark", "formula1", "huggingface"]
+dependencies = [
+    "click>=8.1",
+    "huggingface_hub>=0.24",
+    "requests>=2.31",
+    "rich>=13.7",
+]
+
+[project.optional-dependencies]
+local-models = ["transformers>=4.40", "torch>=2.2"]
+dev = ["pytest>=8.0", "ruff>=0.5"]
+
+[project.scripts]
+f1-temporal-bench = "f1_temporal_bench.cli:main"
+
+[project.urls]
+Homepage = "https://github.com/YOUR_USERNAME/f1-temporal-bench"
+Issues = "https://github.com/YOUR_USERNAME/f1-temporal-bench/issues"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/f1_temporal_bench"]

f1_temporal_bench-0.1.0/src/f1_temporal_bench/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

f1_temporal_bench-0.1.0/src/f1_temporal_bench/cli.py

@@ -0,0 +1,77 @@
+"""Command-line interface: f1-temporal-bench"""
+
+import json
+import sys
+
+import click
+from rich.console import Console
+from rich.table import Table
+
+from .dataset import load_dataset, validate_dataset
+from .eval import run_eval
+from .models import query_hf_inference, query_local_model
+
+console = Console()
+
+
+@click.group()
+def main():
+    """F1 Temporal Knowledge Benchmark CLI."""
+    pass
+
+
+@main.command()
+@click.option("--path", default="data/questions.jsonl", help="Path to dataset.")
+def validate(path):
+    """Validate the dataset file for schema errors."""
+    ok, errors = validate_dataset(path)
+    if ok:
+        console.print(f"[green]Dataset valid: {path}[/green]")
+    else:
+        console.print(f"[red]{len(errors)} error(s) found:[/red]")
+        for e in errors:
+            console.print(f"  - {e}")
+        sys.exit(1)
+
+
+@main.command()
+@click.option("--model", required=True, help="HF model id, e.g. meta-llama/Llama-3.2-1B-Instruct")
+@click.option("--dataset", default="data/questions.jsonl", help="Path to dataset.")
+@click.option("--local", is_flag=True, help="Run the model locally instead of via HF Inference API.")
+@click.option("--output", default=None, help="Optional path to write JSON results.")
+def run(model, dataset, local, output):
+    """Evaluate a model against the dataset."""
+    questions = load_dataset(dataset)
+    console.print(f"Loaded {len(questions)} questions. Evaluating [bold]{model}[/bold]...")
+
+    query_fn = (
+        (lambda q: query_local_model(model, q))
+        if local
+        else (lambda q: query_hf_inference(model, q))
+    )
+
+    result, per_question = run_eval(query_fn, questions)
+
+    table = Table(title=f"Results: {model}")
+    table.add_column("Metric")
+    table.add_column("Value")
+    table.add_row("Total questions", str(result.total))
+    table.add_row("Accuracy", f"{result.accuracy:.1%}")
+    table.add_row("Confidently wrong", f"{result.confidently_wrong_rate:.1%}")
+    table.add_row("Refusal rate", f"{result.refusal_rate:.1%}")
+    console.print(table)
+
+    if output:
+        with open(output, "w", encoding="utf-8") as f:
+            json.dump({
+                "model": model,
+                "accuracy": result.accuracy,
+                "confidently_wrong_rate": result.confidently_wrong_rate,
+                "refusal_rate": result.refusal_rate,
+                "details": per_question,
+            }, f, indent=2)
+        console.print(f"Saved results to {output}")
+
+
+if __name__ == "__main__":
+    main()

f1_temporal_bench-0.1.0/src/f1_temporal_bench/dataset.py

@@ -0,0 +1,60 @@
+"""Load and validate the F1 temporal knowledge dataset."""
+
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+
+
+@dataclass
+class Question:
+    id: str
+    date: str            # ISO date the fact became true (e.g. race date)
+    question: str
+    answer: str
+    aliases: list[str] = field(default_factory=list)
+    category: str = "general"
+    season: int | None = None
+    round: int | None = None
+
+    def matches(self, model_answer: str) -> bool:
+        """Case-insensitive containment match against answer + aliases."""
+        candidates = [self.answer] + self.aliases
+        model_answer_lower = model_answer.lower()
+        return any(c.lower() in model_answer_lower for c in candidates)
+
+
+def load_dataset(path: str | Path) -> list[Question]:
+    path = Path(path)
+    questions = []
+    with path.open(encoding="utf-8") as f:
+        for line_num, line in enumerate(f, start=1):
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                row = json.loads(line)
+            except json.JSONDecodeError as e:
+                raise ValueError(f"Invalid JSON on line {line_num}: {e}") from e
+            questions.append(Question(**row))
+    return questions
+
+
+def validate_dataset(path: str | Path) -> tuple[bool, list[str]]:
+    """Returns (is_valid, list_of_errors)."""
+    errors = []
+    try:
+        questions = load_dataset(path)
+    except Exception as e:
+        return False, [str(e)]
+
+    seen_ids = set()
+    for q in questions:
+        if q.id in seen_ids:
+            errors.append(f"Duplicate id: {q.id}")
+        seen_ids.add(q.id)
+        if not q.question.strip():
+            errors.append(f"{q.id}: empty question")
+        if not q.answer.strip():
+            errors.append(f"{q.id}: empty answer")
+
+    return len(errors) == 0, errors

f1_temporal_bench-0.1.0/src/f1_temporal_bench/eval.py

@@ -0,0 +1,71 @@
+"""Run a model over the dataset and score it."""
+
+from dataclasses import dataclass
+
+from .dataset import Question
+
+
+REFUSAL_PHRASES = [
+    "i don't know", "i'm not sure", "as of my knowledge",
+    "i cannot provide", "i am unable", "no information",
+]
+
+
+@dataclass
+class EvalResult:
+    total: int
+    correct: int
+    confidently_wrong: int
+    refused: int
+
+    @property
+    def accuracy(self) -> float:
+        return self.correct / self.total if self.total else 0.0
+
+    @property
+    def confidently_wrong_rate(self) -> float:
+        return self.confidently_wrong / self.total if self.total else 0.0
+
+    @property
+    def refusal_rate(self) -> float:
+        return self.refused / self.total if self.total else 0.0
+
+
+def is_refusal(model_answer: str) -> bool:
+    lower = model_answer.lower()
+    return any(phrase in lower for phrase in REFUSAL_PHRASES)
+
+
+def run_eval(query_fn, questions: list[Question]) -> tuple[EvalResult, list[dict]]:
+    """query_fn: callable(question_text: str) -> str"""
+    correct = confidently_wrong = refused = 0
+    per_question = []
+
+    for q in questions:
+        model_answer = query_fn(q.question)
+        refusal = is_refusal(model_answer)
+        correct_flag = q.matches(model_answer)
+
+        if correct_flag:
+            correct += 1
+        elif refusal:
+            refused += 1
+        else:
+            confidently_wrong += 1
+
+        per_question.append({
+            "id": q.id,
+            "question": q.question,
+            "expected": q.answer,
+            "model_answer": model_answer,
+            "correct": correct_flag,
+            "refused": refusal,
+        })
+
+    result = EvalResult(
+        total=len(questions),
+        correct=correct,
+        confidently_wrong=confidently_wrong,
+        refused=refused,
+    )
+    return result, per_question

f1_temporal_bench-0.1.0/src/f1_temporal_bench/models.py

@@ -0,0 +1,30 @@
+"""Thin wrapper around HF Inference API / local transformers pipeline."""
+
+import os
+from huggingface_hub import InferenceClient
+
+
+def query_hf_inference(model_id: str, question: str, token: str | None = None) -> str:
+    """Query a model via the Hugging Face Inference API."""
+    token = token or os.environ.get("HF_TOKEN")
+    client = InferenceClient(model=model_id, token=token)
+    prompt = (
+        f"Answer the following question in one short phrase, "
+        f"with no explanation.\nQuestion: {question}\nAnswer:"
+    )
+    response = client.text_generation(prompt, max_new_tokens=30)
+    return response.strip()
+
+
+def query_local_model(model_id: str, question: str) -> str:
+    """Query a model loaded locally via transformers (requires the
+    'local-models' extra: pip install f1-temporal-bench[local-models])."""
+    from transformers import pipeline
+
+    generator = pipeline("text-generation", model=model_id)
+    prompt = (
+        f"Answer the following question in one short phrase, "
+        f"with no explanation.\nQuestion: {question}\nAnswer:"
+    )
+    result = generator(prompt, max_new_tokens=30, do_sample=False)
+    return result[0]["generated_text"][len(prompt):].strip()

f1_temporal_bench-0.1.0/tests/test_eval.py

@@ -0,0 +1,29 @@
+from f1_temporal_bench.dataset import Question
+from f1_temporal_bench.eval import is_refusal, run_eval
+
+
+def test_question_matches():
+    q = Question(id="t1", date="2024-01-01", question="Who won?", answer="Max Verstappen", aliases=["Verstappen"])
+    assert q.matches("The winner was Max Verstappen.")
+    assert q.matches("It was Verstappen.")
+    assert not q.matches("Lewis Hamilton won.")
+
+
+def test_is_refusal():
+    assert is_refusal("I don't know the answer to that.")
+    assert not is_refusal("Max Verstappen")
+
+
+def test_run_eval_scores_correctly():
+    questions = [
+        Question(id="t1", date="2024-01-01", question="Q1", answer="Alpha"),
+        Question(id="t2", date="2024-01-01", question="Q2", answer="Beta"),
+    ]
+
+    fake_answers = {"Q1": "Alpha", "Q2": "I'm not sure"}
+    result, _ = run_eval(lambda q: fake_answers[q], questions)
+
+    assert result.total == 2
+    assert result.correct == 1
+    assert result.refused == 1
+    assert result.confidently_wrong == 0