contextattr 0.1.0__py3-none-any.whl

contextattr/wrapper/toy.py

@@ -0,0 +1,129 @@
+"""ToyModel: a compatible wrapper with known answers and no real model behind it, for tests and a first try."""
+
+from __future__ import annotations
+
+import hashlib
+import math
+import random
+import re
+from collections.abc import Mapping, Sequence
+
+from .contract import Rendered, Sampling, Scored
+
+_TOKEN = re.compile(r"\s*\S+|\s+$")   # each word with the spaces before it; trailing spaces as a last token
+_EDGE = re.compile(r"^\W+|\W+$")
+
+
+class ToyModel:
+    """A stand-in model whose answers are known exactly.
+
+    A response is cut into words, each with the spaces before it. Word w gets the log-probability
+    log σ(bias + Σ rules[cue][w] + Σ pairs[(cue1, cue2)][w]), summed over the cues present in the prompt, and
+    over the pairs whose two cues are both present. A cue is present when its text occurs in the prompt, case
+    ignored (a chat prompt's message contents are joined with newlines); a word is matched lower-cased, without
+    punctuation around it.
+
+    `generate` at temperature 0 gives the reply of the first cue in `replies` that is present, else `default_reply`,
+    sampling.n times. Above temperature 0 each reply is drawn uniformly from the replies of all the cues present (the
+    default when none is), so its rate is known exactly; the draw is seeded by the seed and the prompt's text, and
+    seed None draws afresh."""
+
+    def __init__(self, rules: Mapping[str, Mapping[str, float]] | None = None, *,
+                 pairs: Mapping[tuple[str, str], Mapping[str, float]] | None = None, bias: float = 0.0,
+                 replies: Mapping[str, str] | None = None, default_reply: str = "I do not know.",
+                 name: str = "toy"):
+        self.rules = {_cue(c): _weights(w, c) for c, w in (rules or {}).items()}
+        self.pairs = {}
+        for key, w in (pairs or {}).items():
+            if not (isinstance(key, tuple) and len(key) == 2):
+                raise TypeError(f"ToyModel: a pair is a tuple of two cues, not {key!r}")
+            self.pairs[(_cue(key[0]), _cue(key[1]))] = _weights(w, key)
+        if isinstance(bias, bool) or not isinstance(bias, (int, float)) or not math.isfinite(bias):
+            raise TypeError("ToyModel: bias must be a finite number")
+        self.bias = float(bias)
+        self.replies = {_cue(c): r for c, r in (replies or {}).items()}
+        if not all(isinstance(r, str) for r in self.replies.values()) or not isinstance(default_reply, str):
+            raise TypeError("ToyModel: replies must be strings")
+        self.default_reply = default_reply
+        self.name = name
+
+    @classmethod
+    def demo(cls) -> ToyModel:
+        """A toy model of the interface's Rio example: Brazil's winter makes "cold" likely, its summer "hot" and
+        "humid", more so when the prompt also says Carnival falls in February or March; it replies "cold" when it
+        reads only of the winter."""
+        return cls(rules={"winter in brazil": {"cold": 2.0, "winter": 2.0},
+                          "summer in brazil": {"hot": 2.0, "humid": 1.0, "summer": 2.0}},
+                   pairs={("february or march", "summer in brazil"): {"hot": 1.5, "summer": 1.5}},
+                   bias=-1.0,
+                   replies={"summer in brazil": "It is hot and humid: it is summer in Brazil.",
+                            "winter in brazil": "It is cold, because Carnival is in winter."},
+                   name="toy-demo")
+
+    def score(self, prompts: Sequence[Rendered], responses: Sequence[str]) -> list[Scored]:
+        if len(prompts) != len(responses):
+            raise ValueError(f"ToyModel.score: {len(prompts)} prompts but {len(responses)} responses")
+        out = []
+        for prompt, response in zip(prompts, responses):
+            if not isinstance(response, str):
+                raise TypeError(f"ToyModel.score: a response is a string, not {type(response).__name__}")
+            text = _text(prompt)
+            tokens = _TOKEN.findall(response)
+            out.append(Scored(tuple(self._logprob(text, t) for t in tokens), tuple(tokens)))
+        return out
+
+    def generate(self, prompts: Sequence[Rendered], sampling: Sampling) -> list[list[str]]:
+        out = []
+        for prompt in prompts:
+            text = _text(prompt)
+            present = [r for c, r in self.replies.items() if c in text] or [self.default_reply]
+            if sampling.temperature == 0 or len(present) == 1:
+                out.append([present[0]] * sampling.n)
+                continue
+            rng = random.Random(None if sampling.seed is None else _seed(text, sampling.seed))
+            out.append([rng.choice(present) for _ in range(sampling.n)])
+        return out
+
+    def _logprob(self, text: str, token: str) -> float:
+        word = _EDGE.sub("", token.strip()).lower()
+        x = self.bias
+        x += sum(w.get(word, 0.0) for c, w in self.rules.items() if c in text)
+        x += sum(w.get(word, 0.0) for (c1, c2), w in self.pairs.items() if c1 in text and c2 in text)
+        return log_sigmoid(x)
+
+
+def _seed(text: str, seed: int) -> int:
+    return int.from_bytes(hashlib.sha256(f"{seed}\0{text}".encode()).digest()[:8], "big")
+
+
+def log_sigmoid(x: float) -> float:
+    """log σ(x) = log(1 / (1 + e^-x)), without overflow."""
+    return min(x, 0.0) - math.log1p(math.exp(-abs(x)))
+
+
+def _text(prompt: Rendered) -> str:
+    """The prompt's text, lower-cased; a chat prompt's contents joined with newlines. Checks the rendered shape."""
+    if isinstance(prompt, str):
+        return prompt.lower()
+    if not isinstance(prompt, list) or not all(isinstance(m, dict) and set(m) == {"role", "content"} and
+                                               isinstance(m["role"], str) and isinstance(m["content"], str)
+                                               for m in prompt):
+        raise TypeError("ToyModel: a prompt is a string or a list of {'role', 'content'} string dicts")
+    return "\n".join(m["content"] for m in prompt).lower()
+
+
+def _cue(cue) -> str:
+    if not isinstance(cue, str) or not cue:
+        raise TypeError(f"ToyModel: a cue is a non-empty string, not {cue!r}")
+    return cue.lower()
+
+
+def _weights(w, cue) -> dict[str, float]:
+    if not isinstance(w, Mapping):
+        raise TypeError(f"ToyModel: the weights of {cue!r} are a mapping word → number")
+    out = {}
+    for word, x in w.items():
+        if not isinstance(word, str) or isinstance(x, bool) or not isinstance(x, (int, float)) or not math.isfinite(x):
+            raise TypeError(f"ToyModel: weight {word!r}: {x!r} of {cue!r} must be a word and a finite number")
+        out[word.lower()] = float(x)
+    return out

contextattr-0.1.0.dist-info/METADATA

@@ -0,0 +1,129 @@
+Metadata-Version: 2.4
+Name: contextattr
+Version: 0.1.0
+Summary: Context attribution: which parts of a prompt make a model give an output, for a model hosted anywhere
+Author: Surya Rayala
+License-Expression: MIT
+Project-URL: Repository, https://github.com/Surya-Rayala/contextattr
+Project-URL: Documentation, https://github.com/Surya-Rayala/contextattr/tree/main/docs
+Project-URL: Issues, https://github.com/Surya-Rayala/contextattr/issues
+Keywords: context attribution,explainability,interpretability,LLM,feature interactions,Shapley values,SPEX
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.26
+Requires-Dist: scikit-learn>=1.9
+Provides-Extra: interactions
+Requires-Dist: shapiq[sparse]<1.9,>=1.8; extra == "interactions"
+Requires-Dist: sparse-transform<0.3,>=0.2.1; extra == "interactions"
+Requires-Dist: lightgbm>=4; extra == "interactions"
+Provides-Extra: test
+Requires-Dist: pytest>=8; extra == "test"
+Requires-Dist: hypothesis>=6.100; extra == "test"
+Dynamic: license-file
+
+# ContextAttr
+
+Find out which parts of a prompt make a language model give an answer.
+
+You split a prompt into parts: sentences, documents, words, or any pieces you choose. ContextAttr asks your model about many copies of the prompt with some parts removed. It then tells you how much each part, and each small group of parts working together, pushes the model towards an answer, and how far to trust that result. It works with any model, wherever it runs: you connect yours with a short wrapper.
+
+Version 0.1.0 is an early release: the interface may still change.
+
+## Install
+
+```
+pip install contextattr
+```
+
+Add `[interactions]` (`pip install "contextattr[interactions]"`) for SPEX and the other methods that find parts which matter only together. With uv: `uv add contextattr`. Needs Python 3.12 or later.
+
+## Try it
+
+This runs anywhere in seconds: `ToyModel` stands in for a real model.
+
+```python
+import contextattr as ca
+
+prompt = ca.Prompt("rio", messages=[{"role": "user", "content": [
+    ca.Unit("d1", "Rio Carnival takes place in February or March.", sep="\n"),
+    ca.Unit("d2", "Summer in Brazil runs from December to March.", sep="\n"),
+    ca.Unit("d3", "Winter in Brazil runs from June to September.", sep="\n"),
+    "Question: What is the weather like during Rio Carnival?"]}],
+    responses=[ca.Response("bad", "It is cold, because Carnival is in winter.", label="negative"),
+               ca.Response("good", "It is hot and humid: it is summer in Brazil.", label="positive")])
+
+model = ca.ToyModel.demo()        # a stand-in: put your own model's wrapper here
+e = ca.explain(prompt, model, ca.Contrast(), method="kernelshap", budget=8)
+print(e)
+```
+
+It prints:
+
+```
+rio · KernelSHAP(paired=True) · 8 masks · R² 0.991 on 128 test masks · noise 0 · 22 requests
+  d3              +0.2500  Winter in Brazil runs from June to September.
+  d1              -0.0234  Rio Carnival takes place in February or March.
+  d2              -0.2854  Summer in Brazil runs from December to March.
+```
+
+**What the example says:**
+
+| In the code | Means |
+|---|---|
+| `ca` | the package, under the short name all the docs use |
+| `ca.Prompt("rio", messages=...)` | one prompt, named `rio`, as chat messages |
+| `ca.Unit("d1", "...", sep="\n")` | a part that may be removed, named `d1`. `sep` is text that goes with it, here the line break. Everything else, like the question, always stays |
+| `ca.Response("bad", "...", label="negative")` | an answer the model could give. `negative` marks the answer you do not want, `positive` the one you do |
+| `model` | the wrapper around your model |
+| `ca.Contrast()` | what to explain: how much more likely the model finds the unwanted answers than the wanted ones |
+| `method="kernelshap"`, `budget=8` | how the scores are worked out, and how many copies of the prompt, each with some parts removed, the model is asked about |
+| `e` | the result |
+
+**How to read the output:**
+
+- The first line names the prompt and the method and says how far to trust the result. R² close to 1 means the scores describe the model well; close to 0, do not use them.
+- Then each part with its score, highest first. A positive score pushes the model towards the unwanted answer: `d3`, the sentence about winter, is what makes this model say "cold". A negative score pushes it away: `d2` protects against it. A score near zero means the part hardly matters.
+- `e.scores` gives the scores as a dictionary, and `e.to_html("report.html")` writes a page with each part shaded by its score.
+
+[Reading a result](https://github.com/Surya-Rayala/contextattr/blob/main/docs/results.md) shows the output of every method on this example and what its scores mean.
+
+## Use it with your own model
+
+ContextAttr reaches your model through a small class that you write, a *wrapper*. [Writing a wrapper](https://github.com/Surya-Rayala/contextattr/blob/main/docs/wrapper.md) is the guide, with two complete wrappers to copy from: one for an API and one for a model you run yourself.
+
+1. **Get the file to fill in.** `contextattr template my_wrapper.py` writes it. It needs one or both of:
+   - `generate`: writes replies to a prompt. Every model can do this, including the OpenAI, Gemini and Anthropic APIs.
+   - `score`: gives the log-probability of each token of a given answer. Possible when you run the model yourself, in your own code or on a vLLM or SGLang server.
+2. **Check it.** `python my_wrapper.py` runs `ca.check_model` on it and says what, if anything, is wrong.
+3. **Use it.** Put your wrapper where the example has `model`.
+
+## Choose what to explain, and how
+
+- **Parts:** mark them with `ca.Unit`, as above, or cut a text automatically: `ca.split(text, "sentence", id="s")` (or `"word"`, `"line"`, `"paragraph"`).
+- **Answer:** `ca.LogProb()` explains one answer. `ca.Contrast()` explains what moves the model from the answers you want to the ones you don't. `ca.Sampled(check)` works when your model can only generate text.
+- **Method:** `contextcite`, the default, takes a quick and cheap look, even at long prompts. `spex` finds parts that matter only together. `budget` sets how many copies of the prompt the model is asked about: more is steadier and costs more.
+- **Many prompts:** `contextattr run run.toml` explains a whole file of prompts and writes a report.
+
+## Learn more
+
+| Document | Covers |
+|---|---|
+| [docs/wrapper.md](https://github.com/Surya-Rayala/contextattr/blob/main/docs/wrapper.md) | connecting your model: what to write, two complete wrappers, the rules, and what to do when the check fails |
+| [docs/results.md](https://github.com/Surya-Rayala/contextattr/blob/main/docs/results.md) | reading a result: the output of every method on one example, what its scores mean, and how far to trust them |
+| [docs/methods.md](https://github.com/Surya-Rayala/contextattr/blob/main/docs/methods.md) | which method to use, how many masks to give it, and every setting with its default |
+| [docs/interface.md](https://github.com/Surya-Rayala/contextattr/blob/main/docs/interface.md) | the reference: prompts and the prompt file, answers and labels, what can be explained, costs, runs and the command line |
+
+## Licence
+
+ContextAttr is released under the MIT licence: see [LICENSE](https://github.com/Surya-Rayala/contextattr/blob/main/LICENSE).

contextattr-0.1.0.dist-info/RECORD

@@ -0,0 +1,38 @@
+contextattr/__init__.py,sha256=4nyU5K4kR14X4qybjSQYifex2DykprwCseegxdC7xGs,1247
+contextattr/__main__.py,sha256=csk-ut-m00mlCsYR1xRESKPb4qfCxaWwn87vxJzrhvU,109
+contextattr/cache.py,sha256=MGoAyvplajOjZMepLZesGc9Uig8G7An2LyV7W4jp0V8,2752
+contextattr/cli.py,sha256=MDV-L3zJjnR4piZazuJ5CrnKaDo71TbxoL6t-3Wcn4k,8590
+contextattr/evaluate.py,sha256=iPIkWkc45ZnnTpuIGA49GrRKCKkjp302SZSTwmSZd30,7103
+contextattr/report.py,sha256=pUDAlxLjP1omdHCagh4PkeLTUTQpQN_JKXqdatPO-rA,5682
+contextattr/result.py,sha256=ZgxpYoeYzRxL8Nxrqhz5slJKHYoILpR8BfPNBxreYuo,5947
+contextattr/run.py,sha256=6iSypAPfC0n4Bo_QQQXuOg8ZiVZLIkt-keOffKUAEi4,17517
+contextattr/value.py,sha256=K11gSSACqUTfboanHo4ZhImUwtfVkbfqxvoBr-x_2uc,11068
+contextattr/methods/__init__.py,sha256=QEeQvEbAdLZrnzQfQpU1vRxY0zxoDn0Cev2BIkiZLJY,1336
+contextattr/methods/base.py,sha256=xvhk-rR7_-S6HGvW9cLhZuu4B1GVQQ8HAeAbbelocBM,3587
+contextattr/methods/indices.py,sha256=-KaXDhqb5Io1f0g-w5SDzDXYvv8kR8n8zIZIR-7EUdQ,1986
+contextattr/methods/interactions.py,sha256=4wUDWCpVw-LHAN3t7Bl7exq03htATEWceM_xHuDH5tc,11284
+contextattr/methods/kernelshap.py,sha256=56MUsb14hAIPnsGMdxBt2Ta-9yrv4iSZ4d_qPT5Cmdo,5830
+contextattr/methods/linear.py,sha256=5iQnHESFy2yeLy8IA_68Llz11PZdShshAbsNhifw5TY,5195
+contextattr/methods/loo.py,sha256=R3blTynnnImRq_OtIFSvwQhL3EsFZi58ydveTQnrRQo,1344
+contextattr/prompt/__init__.py,sha256=P7j6cR5CEpRpVHBrwD0IYT0gNZjZMf5koLAZ3VGP8MI,517
+contextattr/prompt/io.py,sha256=1-VcVpzyTp_rAzv3M81aT1maynEFdiY5GxWhUxpXgfc,8875
+contextattr/prompt/render.py,sha256=yzeodc5-_d_A55DDoo6woCTmO-uLaBBF9B22OxI30rg,2964
+contextattr/prompt/split.py,sha256=wEZFORXpzJgHsL-gp0JrGOqbb0yRbFjW7ZI5TwWu8SM,6892
+contextattr/prompt/units.py,sha256=jvBOt7AXnSSieC7GpnluGSmPSCihE7fHvcIYeTwB6DM,11209
+contextattr/responses/__init__.py,sha256=wr1dp5MiGSEXzSnjIF6FwXwFo1sOupSQVMjFfDoiEHo,313
+contextattr/responses/checks.py,sha256=oXHbYZ82M_bdlHXYmEWCmqAUqc-hLibG-yeOFEPb_9k,7024
+contextattr/responses/sample.py,sha256=ARNb0jzmfW4ZIx8darHNhYfQKSL4Wd0hsugjP7EeS94,1886
+contextattr/targets/__init__.py,sha256=i4JxFaHGQqi9mrsgBR6ceYuYdgAsRP5yUL9rarqjdis,362
+contextattr/targets/sampled.py,sha256=3u60m_NCHdOxUWWz1dXmsHwn7upZ8MitDV8rsmBklhA,2813
+contextattr/targets/scored.py,sha256=n7p6iaNdn051bqHz8ZI17K2vxDGsgxgfsadVY_U84Og,6371
+contextattr/wrapper/__init__.py,sha256=jFzFVUwRW-ECkuQ8bxZT6ySuvuHC_bWqy8aJuuosrek,552
+contextattr/wrapper/check.py,sha256=nxkVF1dUEdvW1YnK0XV6l_CKg793J7TsucsNbydItKY,8190
+contextattr/wrapper/contract.py,sha256=d3Qjml5rRNRKvu6HIRX24rn8OCKHhlVs7ZdLU_0Sf7I,7206
+contextattr/wrapper/template.py,sha256=lskvy8hwqSd7ZxhwEunZpN14QWgxky_pOGiqVP27oLM,3047
+contextattr/wrapper/toy.py,sha256=oyQX-Lu3IlJv1z9N9tlH1ALjC27Vi43fIjPW79lvPvk,6610
+contextattr-0.1.0.dist-info/licenses/LICENSE,sha256=lsxcqFqfXFICgMR2fHHX8MrOD8p2cH-GPv5AffW1gDg,1069
+contextattr-0.1.0.dist-info/METADATA,sha256=YpkRmuSWa8z12CVgBLc93GG5tzP-wlx-7_rAQpiHz4c,7846
+contextattr-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+contextattr-0.1.0.dist-info/entry_points.txt,sha256=IyucNr2RurEFZpf0-NqwjKxCV2DCiks3p7VkC-I-Y3w,53
+contextattr-0.1.0.dist-info/top_level.txt,sha256=tqF9BghHlsVfEpUVelZMxQWkSxQ6ZMJMktUqSRjuqG8,12
+contextattr-0.1.0.dist-info/RECORD,,

contextattr-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

contextattr-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+contextattr = contextattr.cli:main

contextattr-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Surya Rayala
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

contextattr-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+contextattr