truthbench 0.1.0__py3-none-any.whl

truthbench/__init__.py

@@ -0,0 +1,4 @@
+from .truth_pipeline import truth_pipeline
+from .pipeline import Pipeline, Step, Reader, LLM
+
+__all__ = ["truth_pipeline", "Pipeline", "Step", "Reader", "LLM"]

truthbench/cli.py

@@ -0,0 +1,46 @@
+import argparse
+import pathlib
+
+import truthbench
+from truthbench.models import Report, Tracker, Sample
+from truthbench.readers.json_reader import JsonReader
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Run truthbench pipeline")
+    parser.add_argument(
+        "--output-dir", "-o", required=True, type=pathlib.Path,
+        help="Directory where to place the output dataset and the execution report"
+    )
+    parser.add_argument(
+        "--input-file", "-i", required=True, type=pathlib.Path,
+        help="Input json dataset containing questions and ground truths"
+    )
+    parser.add_argument(
+        "--keep", "-k", default=.8, type=float,
+        help="Percentage of factual data to preserve"
+    )
+    parser.add_argument(
+        "--num-levels", "-l", default=5, type=int,
+        help="Number of perturbation levels to produce A0-AX"
+    )
+
+    args = parser.parse_args()
+
+    pipeline = truthbench.truth_pipeline(keep=args.keep, num_levels=args.num_levels)
+    samples, tracker = pipeline.run(JsonReader(args.input_file))
+
+    report = Report(report=Tracker(**tracker), questions=[Sample(**s) for s in samples])
+    dataset = report.to_dataset()
+
+    args.output_dir.mkdir(parents=True, exist_ok=True)
+
+    with open(args.output_dir / "report.json", "w", encoding="utf-8") as f:
+        f.write(report.model_dump_json(indent=4))
+
+    with open(args.output_dir / "dataset.json", "w", encoding="utf-8") as f:
+        f.write(dataset.model_dump_json(indent=4))
+
+
+if __name__ == "__main__":
+    main()

truthbench/llms/openai.py

@@ -0,0 +1,16 @@
+from typing import Dict, List
+
+from openai import OpenAI
+
+from truthbench.pipeline import LLM
+
+
+class GPT(LLM):
+
+    def __init__(self, client: OpenAI, model: str = "gpt-4o"):
+        self._client = client
+        self._model = model
+
+    def query(self, messages: List[Dict[str, str]]) -> str:
+        completion = self._client.chat.completions.create(model=self._model, messages=messages)
+        return completion.choices[0].message.content.strip()

truthbench/models.py

@@ -0,0 +1,51 @@
+from typing import Optional, List, Dict
+
+import pydantic
+
+
+class Tracker(pydantic.BaseModel):
+    input_samples: int = 0
+    find_factual_data_error: int = 0
+    json_parse_ranking_error: int = 0
+    index_ranking_error: int = 0
+    ranking_factual_data_error: int = 0
+    output_samples: int = 0
+
+
+class Sample(pydantic.BaseModel):
+    question: Optional[str] = None
+    ground_truth: Optional[str] = None
+    raw_factual_data: Optional[List[str]] = None
+    with_brackets: Optional[Dict[str, str]] = None
+    thinking: Optional[Dict[str, str]] = None
+    blacklisted: Optional[List[str]] = None
+    factual_data: Optional[List[str]] = None
+    ranked_factual_data: Optional[List[str]] = None
+    answers: Optional[Dict[str, str]] = None
+
+    def is_valid(self) -> bool:
+        return len(self.answers.keys()) > 1
+
+
+class Item(pydantic.BaseModel):
+    id: int
+    question: str
+    ground_truth: str
+    answers: Dict[str, str]
+
+    @classmethod
+    def from_sample(cls, id_: int, sample: Sample) -> 'Item':
+        return Item(id=id_, question=sample.question, ground_truth=sample.ground_truth, answers=sample.answers)
+
+
+class Dataset(pydantic.BaseModel):
+    questions: List[Item]
+
+
+class Report(pydantic.BaseModel):
+    report: Tracker
+    questions: List[Sample]
+
+    def to_dataset(self) -> Dataset:
+        items = [Item.from_sample(id_=i, sample=s) for i, s in enumerate(self.questions) if s.is_valid()]
+        return Dataset(questions=items)

truthbench/pipeline.py

@@ -0,0 +1,174 @@
+import abc
+from typing import List, Dict, Tuple, Any, Set
+
+from tqdm import tqdm
+
+
+class StrictTracker(dict):
+    """
+    A dictionary subclass that enforces allowed keys and initializes counters.
+
+    This tracker only allows keys declared in `allowed_keys`. Accessing or setting
+    a key not in `allowed_keys` raises a KeyError with a helpful message.
+    """
+
+    def __init__(self, allowed_keys: set[str]):
+        super().__init__()
+        self._allowed_keys = allowed_keys
+        self.update({k: 0 for k in self._allowed_keys})
+
+    def __getitem__(self, key):
+        if key not in self._allowed_keys:
+            raise KeyError(
+                f"Tracker counter '{key}' is being set but was not declared. "
+                f"Declare it at the Step constructor with: "
+                f"    super().__init__(..., counters=frozenset({{{repr(key)}}}))"
+            )
+        return super().__getitem__(key)
+
+    def __setitem__(self, key, value):
+        if key not in self._allowed_keys:
+            raise KeyError(
+                f"Tracker counter '{key}' is being set but was not declared. "
+                f"Declare it at the Step constructor with: "
+                f"    super().__init__(..., counters=frozenset({{{repr(key)}}}))"
+            )
+        return super().__setitem__(key, value)
+
+
+class LLM(abc.ABC):
+    """
+    Abstract base class for Language Models.
+
+    Subclasses must implement the `query` method to send messages to the LLM
+    and return the response as a string.
+    """
+
+    @abc.abstractmethod
+    def query(self, messages: List[Dict[str, str]]) -> str:
+        """
+        Query the language model with a list of messages and get the output string.
+
+        Args:
+            messages (List[Dict[str, str]]): A list of message dicts with keys like 'role' and 'content'.
+
+        Returns:
+            str: The LLM's response as a string.
+        """
+        ...
+
+
+class Step(abc.ABC):
+    """
+    Abstract base class representing a single processing step in the pipeline.
+
+    Args:
+        required_fields (Set[str]): Set of keys that must be present in each sample before running this step.
+        counters (Set[str]): Set of counter names that this step may increment in the tracker.
+    """
+
+    def __init__(self, required_fields: Set[str] = frozenset(), counters: Set[str] = frozenset()):
+        self.required_fields = required_fields
+        self.counters = counters
+
+    def validate(self, sample: Dict[str, Any]) -> None:
+        """
+        Validate that the sample contains all required fields for this step.
+
+        Raises:
+            ValueError: If any required field is missing in the sample.
+        """
+
+        current_fields = set(sample.keys())
+        if not self.required_fields.issubset(current_fields):
+            missing = self.required_fields.difference(current_fields)
+            raise ValueError(
+                f"{type(self).__name__} requires {sorted(self.required_fields)}, but some are missing from the sample: "
+                f"{sorted(missing)}. Check pipeline dependencies before proceeding."
+            )
+
+    @abc.abstractmethod
+    def step(self, sample: Dict[str, Any], tracker: Dict[str, int]) -> None:
+        """
+        Execute the step logic on the sample, possibly updating the tracker.
+
+        Args:
+            sample (Dict[str, Any]): The data sample to process.
+            tracker (Dict[str, int]): A dictionary tracking counters/errors during processing.
+        """
+        ...
+
+
+class Reader(abc.ABC):
+    """
+    Abstract base class for data readers that provide samples to the pipeline.
+
+    Subclasses must implement the `samples` method that returns a list of validated samples.
+    """
+
+    @abc.abstractmethod
+    def samples(self) -> List[Dict[str, Any]]:
+        """
+        Load and return validated samples.
+
+        Returns:
+            List[Dict[str, Any]]: A list of dictionaries containing 'question' and 'ground_truth' keys.
+
+        Raises:
+            ValueError: If the source could not be read or has an invalid format.
+        """
+        ...
+
+
+class Pipeline:
+    """
+    Orchestrates a sequence of Steps to process data samples.
+
+    Args:
+        with_progress (bool): Whether to display a progress bar during execution (tqdm).
+    """
+
+    def __init__(self, with_progress: bool = True):
+        self._steps: List[Step] = []
+        self._with_progress = with_progress
+
+    def with_step(self, step: Step) -> 'Pipeline':
+        """
+        Add a processing step to the pipeline.
+
+        Args:
+            step (Step): A step instance to add.
+
+        Returns:
+            Pipeline: Self, to allow method chaining.
+        """
+        self._steps.append(step)
+        return self
+
+    def run(self, reader: Reader) -> Tuple[List[Dict[str, Any]], Dict[str, int]]:
+        """
+        Execute all steps in sequence on each sample provided by the reader.
+
+        Args:
+            reader (Reader): Data reader yielding samples.
+
+        Returns:
+            Tuple[List[Dict[str, Any]], Dict[str, int]]:
+                - List of processed samples.
+                - Tracker dictionary with counters collected during processing.
+        """
+        allowed_keys = {"input_samples"} | frozenset.union(*(step.counters for step in self._steps))
+
+        tracker = StrictTracker(allowed_keys)
+
+        samples = reader.samples()
+
+        collected = []
+        for sample in tqdm(samples, desc="Samples:", disable=not self._with_progress):
+            tracker["input_samples"] += 1
+            for step in self._steps:
+                step.validate(sample)
+                step.step(sample, tracker)
+            collected.append(sample)
+
+        return collected, tracker

truthbench/readers/json_reader.py

@@ -0,0 +1,52 @@
+import json
+import pathlib
+from typing import List, Dict, Any
+
+from truthbench.pipeline import Reader
+
+
+class JsonReader(Reader):
+    """
+    A reader that loads question-answer samples from a JSON file.
+
+    The JSON file must contain a list of dictionaries. Each dictionary must include
+    the following keys:
+        - "question" (str)
+        - "ground_truth" (str)
+
+    Example input JSON:
+        [
+            {"question": "What is Python?", "ground_truth": "A programming language."},
+            {"question": "What is 2+2?", "ground_truth": "4"}
+        ]
+
+    Parameters:
+        input_file (pathlib.Path): Path to the input JSON file.
+
+    Raises:
+        ValueError: If the JSON is invalid, not a list of objects, or missing required keys.
+    """
+
+    def __init__(self, input_file: pathlib.Path):
+        self._input_file = input_file
+
+    def samples(self) -> List[Dict[str, Any]]:
+        with open(self._input_file, "r") as f:
+            content = f.read()
+
+        gold_dataset = json.loads(content)
+
+        if not isinstance(gold_dataset, list):
+            raise ValueError("Expected top-level JSON array (list of samples)")
+
+        samples = []
+        for d in gold_dataset:
+            if not isinstance(d, dict):
+                raise ValueError(f"Samples must be JSON objects")
+            if "question" not in d or "ground_truth" not in d:
+                raise ValueError(
+                    f"Missing required keys: 'question' and 'ground_truth'"
+                )
+            samples.append({"question": d["question"], "ground_truth": d["ground_truth"]})
+
+        return samples

truthbench/steps/blacklist.py

@@ -0,0 +1,60 @@
+import re
+from typing import Set, Dict, Any
+
+from truthbench.pipeline import Step
+
+
+class BlacklistItemsFromQuestionStep(Step):
+    """
+    Pipeline step that filters out factual items which contain words already present in the question.
+
+    For each item in `raw_factual_data`, this step checks whether any token in the item
+    (split by whitespace) overlaps with the question tokens. If so, the item is added to
+    the `blacklisted` list in the sample. Tokens are lowercased and stripped of punctuation.
+
+    Stop words can be provided to exclude common words from the question during matching.
+
+    Attributes:
+        - stop_words (Set[str]): A set of words to ignore when matching against the question.
+
+    Expected Sample Fields:
+        - question (str): The question text.
+        - raw_factual_data (List[str]): List of candidate factual items.
+
+    Modifies:
+        - sample["blacklisted"] (List[str] | None): List of lowercased blacklisted items,
+          or None if the question or factual data is empty.
+
+    Counter:
+        - Increments no counters
+
+    Notes:
+        - This step does not increment or modify the tracker.
+        - Matching is case-insensitive and ignores punctuation.
+        - If either `question` or `raw_factual_data` is empty, `blacklisted` is set to None.
+
+    Example:
+        question: "What is climate change?"
+        raw_factual_data: ["Climate models", "Carbon emissions", "Solar activity"]
+        -> blacklisted: ["climate models", "carbon emissions"]
+    """
+
+    def __init__(self, stop_words: Set[str]):
+        self._stop_words = stop_words
+        super().__init__(
+            required_fields=frozenset({"question", "raw_factual_data"})
+        )
+
+    def step(self, sample: Dict[str, Any], tracker: Dict[str, int]) -> None:
+        if not sample["question"] or not sample["raw_factual_data"]:
+            sample["blacklisted"] = None
+            return
+
+        # Simple tokenization of the question
+        # (strip punctuation, lowercase, then split on whitespace)
+        question_words = set(re.findall(r"\w+", sample["question"].lower()))
+        question_words = question_words - self._stop_words
+        sample["blacklisted"] = [
+            term.lower() for term in sample["raw_factual_data"]
+            if any(word.lower() in question_words for word in term.split())
+        ]

truthbench/steps/counter.py

@@ -0,0 +1,46 @@
+from typing import Dict, Any
+
+from truthbench.pipeline import Step
+
+
+class CounterStep(Step):
+    """
+    Pipeline step that counts samples with the expected number of answers.
+
+    This step checks whether the `answers` field exists and contains exactly the expected number
+    of answer levels. If so, it increments the `output_samples` counter in the tracker.
+
+    Attributes:
+        - expected_levels (int): The required number of answer levels for a valid sample.
+
+    Expected Sample Fields:
+        - answers (List[Any]): A list of answers to be validated.
+
+    Modifies:
+        - It does not modify any field.
+
+    Counter:
+        - tracker["output_samples"] (int): Incremented by 1 if the sample passes the check.
+
+    Notes:
+        - This step does not modify the sample.
+        - The sample is only counted if `answers` is present and its length matches the expected level.
+        - Does nothing if `answers` is None or its length does not match `expected_levels`.
+
+    Example:
+        expected_levels: 3
+        sample["answers"]: ["yes", "no", "maybe"]
+        -> tracker["output_samples"] += 1
+    """
+
+    def __init__(self, expected_levels: int):
+        self._expected_levels = expected_levels
+        super().__init__(
+            required_fields=frozenset({"answers"}),
+            counters=frozenset({"output_samples"})
+        )
+
+    def step(self, sample: Dict[str, Any], tracker: Dict[str, int]) -> None:
+        if sample["answers"] and len(sample["answers"]) == self._expected_levels:
+            tracker["output_samples"] += 1
+            return

truthbench/steps/factual.py

@@ -0,0 +1,228 @@
+import abc
+import re
+from typing import Union, Iterator, List, Tuple, Dict, Any
+
+from spacy import Language, Errors
+from spacy.symbols import NOUN, PROPN, ADV, ADJ, amod, NUM
+from spacy.tokens import Doc, Span
+
+from truthbench.pipeline import Step
+
+
+class FactualChunker(abc.ABC):
+    """
+    Extract factual components of a sentence.
+    """
+
+    @abc.abstractmethod
+    def tag(self, sentence: str) -> str:
+        ...
+
+
+class NounAdverbFactualChunker(FactualChunker):
+    """
+     Implements a rule-based factual chunker that identifies textual spans likely to carry factual content
+     within a sentence, focusing on noun phrases and adverbial modifiers.
+
+     This chunker uses syntactic dependency parsing to detect spans such as:
+       - Direct objects, attributes, and complements of the main verb.
+       - Adverbial modifiers and prepositional phrases expressing circumstantial details.
+       - Appositional phrases and descriptive noun modifiers.
+       - Numerical expressions excluding those in the subject.
+
+     To avoid altering the core meaning of sentences, it excludes spans that belong to grammatical subjects.
+     For noun phrase subjects containing embedded relative clauses, only the head noun is excluded,
+     allowing modifiers in relative clauses to be eligible.
+
+     Coordination is handled by propagating eligibility from a conjunct to its siblings if the head
+     satisfies the criteria. The chunker also suppresses nested or overlapping spans to produce a clean,
+     non-redundant set of factual candidates.
+
+     The output is a bracketed string marking identified factual spans, forming an intermediate
+     representation for downstream filtering, ranking, or perturbation.
+
+     Example:
+         Input:  "The government announced the new policy in 2021 with confidence."
+         Output: "The government announced [the new policy] in [2021] with [confidence]."
+
+     Methods:
+         - tag(sentence: str) -> str:
+             Returns the input sentence with factual spans bracketed.
+
+     Notes:
+         - Requires a syntactic dependency parse (e.g., from spaCy).
+         - Focuses on spans relevant for factual content modification.
+         - Does not modify spans related to sentence subjects to prevent meaning distortion.
+     """
+
+    def __init__(self, nlp: Language):
+        self._nlp = nlp
+
+    def span_boxes(self, doclike: Union[Doc, Span]) -> Iterator[Span]:
+        """
+        Detect base noun phrases in the object and adverbs from a dependency parse.
+        """
+        labels = [
+            "oprd",
+            "dobj",
+            "advmod",
+            "amod",
+            "npadvmod",
+            "pcomp",
+            "pobj",
+            "dative",
+            "appos",
+            "attr",
+            "ROOT",
+        ]
+
+        doc = doclike.doc  # Ensure works on both Doc and Span.
+
+        if not doc.has_annotation("DEP"):
+            raise ValueError(Errors.E029)
+
+        np_deps = [doc.vocab.strings.add(label) for label in labels]
+        conj = doc.vocab.strings.add("conj")
+        prev_end = -1
+
+        # Collect subject heads within the doclike (Span or Doc)
+        subject_heads = [token for token in doclike if token.dep_ in {'nsubj', 'nsubjpass'}]
+
+        # Collect indices of all tokens in their subtrees
+        subject_indices = set()
+        for head in subject_heads:
+            # Add all tokens in the subject head's subtree
+            head_subtree = {t.i for t in head.subtree}
+            subject_indices.update(head_subtree)
+
+            # Subtract tokens in relative clauses (relcl) attached to the subject head
+            for child in head.children:
+                if child.dep_ == "relcl":
+                    relcl_subtree = {t.i for t in child.subtree}
+                    subject_indices.difference_update(relcl_subtree)
+
+        for i, word in enumerate(doclike):
+            if word.pos not in (NOUN, PROPN, ADV, ADJ, NUM):
+                continue
+
+            # Skip if part of the subject
+            if word.i in subject_indices:
+                continue
+
+            if word.pos == ADJ and word.dep == amod and word.head.pos in (NOUN, PROPN):
+                continue
+
+            # Prevent nested chunks from being produced
+            if word.left_edge.i <= prev_end:
+                continue
+
+            if word.dep in np_deps or (word.pos == NUM and word.dep_ in ("nummod", "appos", "attr")):
+                prev_end = word.i
+                yield doc[word.left_edge.i:word.i + 1]
+            elif word.dep == conj:
+                head = word.head
+
+                while head.dep == conj and head.head.i < head.i:
+                    head = head.head
+
+                # If the head is an NP, and we're coordinated to it, we're an NP
+                if head.dep in np_deps:
+                    prev_end = word.i
+                    yield doc[word.left_edge.i:word.i + 1]
+
+    def overlaps(self, idx: List[Tuple[int, int]]) -> bool:
+        sorted_intervals = sorted(idx)
+        return any(
+            current_end > next_start
+            for (_, current_end), (next_start, _) in zip(sorted_intervals, sorted_intervals[1:])
+        )
+
+    def tag(self, sentence: str) -> str:
+        doc = self._nlp(sentence)
+
+        idx = []
+        for box in self.span_boxes(doc):
+            idx.append((min(b.idx for b in box), max(b.idx + len(b) for b in box)))
+
+        idx.sort(reverse=True)
+
+        assert not self.overlaps(idx), \
+            f"Something went wrong... Overlapping indexes for `{sentence}`"
+
+        boxed_sentence = sentence
+        for start, end in idx:
+            boxed_sentence = boxed_sentence[:start] + "[" + boxed_sentence[start:end] + "]" + boxed_sentence[end:]
+
+        return boxed_sentence
+
+
+class FactualDataStep(Step):
+    """
+    Step that identifies factual data spans within an answer text by leveraging a
+    provided FactualChunker implementation. It marks spans in the answer likely to
+    contain factual content, brackets them, and extracts these spans for downstream processing.
+
+    Attributes:
+        - chunker (FactualChunker): An instance responsible for tagging factual spans in sentences.
+
+    Expected Sample Fields:
+        - answers (Dict[str, str]): A dictionary containing answer texts keyed by identifiers
+          (e.g., "A0"). This step processes the text under the "A0" key.
+
+    Modifies:
+        - sample["with_brackets"] (Dict[str, str] or None): Adds a dictionary mapping answer keys
+          to bracketed strings marking factual spans. Set to None if input is missing or invalid.
+        - sample["raw_factual_data"] (List[str] or None): Extracted factual spans as a list of strings.
+          Set to None if no factual spans are found.
+
+    Counter:
+        - find_factual_data_error: Incremented when no factual spans are detected in the answer text.
+
+    Notes:
+        - Relies on the injected FactualChunker to perform the actual span identification and tagging.
+        - If "answers" is missing or does not contain the key "A0", no processing occurs and relevant
+          fields are set to None.
+        - Extracted spans are obtained by regex matching bracketed sections in the tagged text.
+
+    Example:
+        sample = {
+            "answers": {"A0": "The government announced the new policy in 2021 with confidence."}
+        }
+        tracker = {"find_factual_data_error": 0}
+
+        chunker = NounAdverbFactualChunker(nlp)
+        step = FactualDataStep(chunker)
+        step.step(sample, tracker)
+
+        # After processing:
+        # sample["with_brackets"]["A0"] will be:
+        # "The government announced [the new policy] in [2021] with [confidence]."
+        # sample["raw_factual_data"] will be:
+        # ["the new policy", "2021", "confidence"]
+        # tracker["find_factual_data_error"] remains 0 because factual spans were found.
+    """
+
+    def __init__(self, chunker: FactualChunker):
+        self._chunker = chunker
+        super().__init__(
+            required_fields=frozenset({"answers"}),
+            counters=frozenset({"find_factual_data_error"})
+        )
+
+    def step(self, sample: Dict[str, Any], tracker: Dict[str, int]) -> None:
+        if not sample["answers"] or "A0" not in sample["answers"].keys():
+            sample["with_brackets"] = None
+            sample["raw_factual_data"] = None
+            return
+
+        sample["with_brackets"] = {}
+        response_text = self._chunker.tag(sample["answers"]["A0"])
+        sample["with_brackets"]["A0"] = response_text
+
+        matches = re.findall(r"\[(.*?)]", response_text)
+        if not matches:
+            tracker["find_factual_data_error"] += 1
+            sample["raw_factual_data"] = None
+            return
+
+        sample["raw_factual_data"] = matches