evalvitals 0.1.0__tar.gz

evalvitals-0.1.0/LICENSE

@@ -0,0 +1,121 @@
+Creative Commons Legal Code
+
+CC0 1.0 Universal
+
+    CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE
+    LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN
+    ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS
+    INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES
+    REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS
+    PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM
+    THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED
+    HEREUNDER.
+
+Statement of Purpose
+
+The laws of most jurisdictions throughout the world automatically confer
+exclusive Copyright and Related Rights (defined below) upon the creator
+and subsequent owner(s) (each and all, an "owner") of an original work of
+authorship and/or a database (each, a "Work").
+
+Certain owners wish to permanently relinquish those rights to a Work for
+the purpose of contributing to a commons of creative, cultural and
+scientific works ("Commons") that the public can reliably and without fear
+of later claims of infringement build upon, modify, incorporate in other
+works, reuse and redistribute as freely as possible in any form whatsoever
+and for any purposes, including without limitation commercial purposes.
+These owners may contribute to the Commons to promote the ideal of a free
+culture and the further production of creative, cultural and scientific
+works, or to gain reputation or greater distribution for their Work in
+part through the use and efforts of others.
+
+For these and/or other purposes and motivations, and without any
+expectation of additional consideration or compensation, the person
+associating CC0 with a Work (the "Affirmer"), to the extent that he or she
+is an owner of Copyright and Related Rights in the Work, voluntarily
+elects to apply CC0 to the Work and publicly distribute the Work under its
+terms, with knowledge of his or her Copyright and Related Rights in the
+Work and the meaning and intended legal effect of CC0 on those rights.
+
+1. Copyright and Related Rights. A Work made available under CC0 may be
+protected by copyright and related or neighboring rights ("Copyright and
+Related Rights"). Copyright and Related Rights include, but are not
+limited to, the following:
+
+  i. the right to reproduce, adapt, distribute, perform, display,
+     communicate, and translate a Work;
+ ii. moral rights retained by the original author(s) and/or performer(s);
+iii. publicity and privacy rights pertaining to a person's image or
+     likeness depicted in a Work;
+ iv. rights protecting against unfair competition in regards to a Work,
+     subject to the limitations in paragraph 4(a), below;
+  v. rights protecting the extraction, dissemination, use and reuse of data
+     in a Work;
+ vi. database rights (such as those arising under Directive 96/9/EC of the
+     European Parliament and of the Council of 11 March 1996 on the legal
+     protection of databases, and under any national implementation
+     thereof, including any amended or successor version of such
+     directive); and
+vii. other similar, equivalent or corresponding rights throughout the
+     world based on applicable law or treaty, and any national
+     implementations thereof.
+
+2. Waiver. To the greatest extent permitted by, but not in contravention
+of, applicable law, Affirmer hereby overtly, fully, permanently,
+irrevocably and unconditionally waives, abandons, and surrenders all of
+Affirmer's Copyright and Related Rights and associated claims and causes
+of action, whether now known or unknown (including existing as well as
+future claims and causes of action), in the Work (i) in all territories
+worldwide, (ii) for the maximum duration provided by applicable law or
+treaty (including future time extensions), (iii) in any current or future
+medium and for any number of copies, and (iv) for any purpose whatsoever,
+including without limitation commercial, advertising or promotional
+purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each
+member of the public at large and to the detriment of Affirmer's heirs and
+successors, fully intending that such Waiver shall not be subject to
+revocation, rescission, cancellation, termination, or any other legal or
+equitable action to disrupt the quiet enjoyment of the Work by the public
+as contemplated by Affirmer's express Statement of Purpose.
+
+3. Public License Fallback. Should any part of the Waiver for any reason
+be judged legally invalid or ineffective under applicable law, then the
+Waiver shall be preserved to the maximum extent permitted taking into
+account Affirmer's express Statement of Purpose. In addition, to the
+extent the Waiver is so judged Affirmer hereby grants to each affected
+person a royalty-free, non transferable, non sublicensable, non exclusive,
+irrevocable and unconditional license to exercise Affirmer's Copyright and
+Related Rights in the Work (i) in all territories worldwide, (ii) for the
+maximum duration provided by applicable law or treaty (including future
+time extensions), (iii) in any current or future medium and for any number
+of copies, and (iv) for any purpose whatsoever, including without
+limitation commercial, advertising or promotional purposes (the
+"License"). The License shall be deemed effective as of the date CC0 was
+applied by Affirmer to the Work. Should any part of the License for any
+reason be judged legally invalid or ineffective under applicable law, such
+partial invalidity or ineffectiveness shall not invalidate the remainder
+of the License, and in such case Affirmer hereby affirms that he or she
+will not (i) exercise any of his or her remaining Copyright and Related
+Rights in the Work or (ii) assert any associated claims and causes of
+action with respect to the Work, in either case contrary to Affirmer's
+express Statement of Purpose.
+
+4. Limitations and Disclaimers.
+
+ a. No trademark or patent rights held by Affirmer are waived, abandoned,
+    surrendered, licensed or otherwise affected by this document.
+ b. Affirmer offers the Work as-is and makes no representations or
+    warranties of any kind concerning the Work, express, implied,
+    statutory or otherwise, including without limitation warranties of
+    title, merchantability, fitness for a particular purpose, non
+    infringement, or the absence of latent or other defects, accuracy, or
+    the present or absence of errors, whether or not discoverable, all to
+    the greatest extent permissible under applicable law.
+ c. Affirmer disclaims responsibility for clearing rights of other persons
+    that may apply to the Work or any use thereof, including without
+    limitation any person's Copyright and Related Rights in the Work.
+    Further, Affirmer disclaims responsibility for obtaining any necessary
+    consents, permissions or other rights required for any use of the
+    Work.
+ d. Affirmer understands and acknowledges that Creative Commons is not a
+    party to this document and has no duty or obligation with respect to
+    this CC0 or use of the Work.

evalvitals-0.1.0/PKG-INFO

@@ -0,0 +1,300 @@
+Metadata-Version: 2.4
+Name: evalvitals
+Version: 0.1.0
+Summary: Failure case analysis in the era of LLM.
+License: Creative Commons Legal Code
+        
+        CC0 1.0 Universal
+        
+            CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE
+            LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN
+            ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS
+            INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES
+            REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS
+            PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM
+            THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED
+            HEREUNDER.
+        
+        Statement of Purpose
+        
+        The laws of most jurisdictions throughout the world automatically confer
+        exclusive Copyright and Related Rights (defined below) upon the creator
+        and subsequent owner(s) (each and all, an "owner") of an original work of
+        authorship and/or a database (each, a "Work").
+        
+        Certain owners wish to permanently relinquish those rights to a Work for
+        the purpose of contributing to a commons of creative, cultural and
+        scientific works ("Commons") that the public can reliably and without fear
+        of later claims of infringement build upon, modify, incorporate in other
+        works, reuse and redistribute as freely as possible in any form whatsoever
+        and for any purposes, including without limitation commercial purposes.
+        These owners may contribute to the Commons to promote the ideal of a free
+        culture and the further production of creative, cultural and scientific
+        works, or to gain reputation or greater distribution for their Work in
+        part through the use and efforts of others.
+        
+        For these and/or other purposes and motivations, and without any
+        expectation of additional consideration or compensation, the person
+        associating CC0 with a Work (the "Affirmer"), to the extent that he or she
+        is an owner of Copyright and Related Rights in the Work, voluntarily
+        elects to apply CC0 to the Work and publicly distribute the Work under its
+        terms, with knowledge of his or her Copyright and Related Rights in the
+        Work and the meaning and intended legal effect of CC0 on those rights.
+        
+        1. Copyright and Related Rights. A Work made available under CC0 may be
+        protected by copyright and related or neighboring rights ("Copyright and
+        Related Rights"). Copyright and Related Rights include, but are not
+        limited to, the following:
+        
+          i. the right to reproduce, adapt, distribute, perform, display,
+             communicate, and translate a Work;
+         ii. moral rights retained by the original author(s) and/or performer(s);
+        iii. publicity and privacy rights pertaining to a person's image or
+             likeness depicted in a Work;
+         iv. rights protecting against unfair competition in regards to a Work,
+             subject to the limitations in paragraph 4(a), below;
+          v. rights protecting the extraction, dissemination, use and reuse of data
+             in a Work;
+         vi. database rights (such as those arising under Directive 96/9/EC of the
+             European Parliament and of the Council of 11 March 1996 on the legal
+             protection of databases, and under any national implementation
+             thereof, including any amended or successor version of such
+             directive); and
+        vii. other similar, equivalent or corresponding rights throughout the
+             world based on applicable law or treaty, and any national
+             implementations thereof.
+        
+        2. Waiver. To the greatest extent permitted by, but not in contravention
+        of, applicable law, Affirmer hereby overtly, fully, permanently,
+        irrevocably and unconditionally waives, abandons, and surrenders all of
+        Affirmer's Copyright and Related Rights and associated claims and causes
+        of action, whether now known or unknown (including existing as well as
+        future claims and causes of action), in the Work (i) in all territories
+        worldwide, (ii) for the maximum duration provided by applicable law or
+        treaty (including future time extensions), (iii) in any current or future
+        medium and for any number of copies, and (iv) for any purpose whatsoever,
+        including without limitation commercial, advertising or promotional
+        purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each
+        member of the public at large and to the detriment of Affirmer's heirs and
+        successors, fully intending that such Waiver shall not be subject to
+        revocation, rescission, cancellation, termination, or any other legal or
+        equitable action to disrupt the quiet enjoyment of the Work by the public
+        as contemplated by Affirmer's express Statement of Purpose.
+        
+        3. Public License Fallback. Should any part of the Waiver for any reason
+        be judged legally invalid or ineffective under applicable law, then the
+        Waiver shall be preserved to the maximum extent permitted taking into
+        account Affirmer's express Statement of Purpose. In addition, to the
+        extent the Waiver is so judged Affirmer hereby grants to each affected
+        person a royalty-free, non transferable, non sublicensable, non exclusive,
+        irrevocable and unconditional license to exercise Affirmer's Copyright and
+        Related Rights in the Work (i) in all territories worldwide, (ii) for the
+        maximum duration provided by applicable law or treaty (including future
+        time extensions), (iii) in any current or future medium and for any number
+        of copies, and (iv) for any purpose whatsoever, including without
+        limitation commercial, advertising or promotional purposes (the
+        "License"). The License shall be deemed effective as of the date CC0 was
+        applied by Affirmer to the Work. Should any part of the License for any
+        reason be judged legally invalid or ineffective under applicable law, such
+        partial invalidity or ineffectiveness shall not invalidate the remainder
+        of the License, and in such case Affirmer hereby affirms that he or she
+        will not (i) exercise any of his or her remaining Copyright and Related
+        Rights in the Work or (ii) assert any associated claims and causes of
+        action with respect to the Work, in either case contrary to Affirmer's
+        express Statement of Purpose.
+        
+        4. Limitations and Disclaimers.
+        
+         a. No trademark or patent rights held by Affirmer are waived, abandoned,
+            surrendered, licensed or otherwise affected by this document.
+         b. Affirmer offers the Work as-is and makes no representations or
+            warranties of any kind concerning the Work, express, implied,
+            statutory or otherwise, including without limitation warranties of
+            title, merchantability, fitness for a particular purpose, non
+            infringement, or the absence of latent or other defects, accuracy, or
+            the present or absence of errors, whether or not discoverable, all to
+            the greatest extent permissible under applicable law.
+         c. Affirmer disclaims responsibility for clearing rights of other persons
+            that may apply to the Work or any use thereof, including without
+            limitation any person's Copyright and Related Rights in the Work.
+            Further, Affirmer disclaims responsibility for obtaining any necessary
+            consents, permissions or other rights required for any use of the
+            Work.
+         d. Affirmer understands and acknowledges that Creative Commons is not a
+            party to this document and has no duty or obligation with respect to
+            this CC0 or use of the Work.
+        
+Project-URL: Homepage, https://github.com/evalvitals/evalvitals
+Project-URL: Documentation, https://evalvitals.github.io/evalvitals/
+Project-URL: Repository, https://github.com/evalvitals/evalvitals
+Project-URL: Issues, https://github.com/evalvitals/evalvitals/issues
+Project-URL: Changelog, https://github.com/evalvitals/evalvitals/blob/main/CHANGELOG.md
+Keywords: llm,vlm,interpretability,failure-analysis,explainability
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: CC0 1.0 Universal (CC0 1.0) Public Domain Dedication
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: numpy>=1.24
+Provides-Extra: local
+Requires-Dist: torch>=2.0.0; extra == "local"
+Requires-Dist: transformers<5.0.0,>=4.51.0; extra == "local"
+Requires-Dist: accelerate>=0.26.0; extra == "local"
+Provides-Extra: api
+Requires-Dist: openai>=1.0; extra == "api"
+Provides-Extra: interp
+Requires-Dist: captum>=0.7; extra == "interp"
+Requires-Dist: inseq>=0.6; extra == "interp"
+Requires-Dist: nnsight>=0.3; extra == "interp"
+Provides-Extra: data
+Requires-Dist: datasets>=2.0; extra == "data"
+Requires-Dist: pillow>=9.0; extra == "data"
+Provides-Extra: viz
+Requires-Dist: matplotlib>=3.7; extra == "viz"
+Provides-Extra: stats
+Requires-Dist: statsmodels>=0.14; extra == "stats"
+Requires-Dist: scikit-learn>=1.3; extra == "stats"
+Provides-Extra: dashboard
+Requires-Dist: streamlit>=1.30; extra == "dashboard"
+Requires-Dist: pandas>=2.0; extra == "dashboard"
+Provides-Extra: gemini
+Requires-Dist: google-genai>=1.0; extra == "gemini"
+Provides-Extra: all
+Requires-Dist: torch>=2.0.0; extra == "all"
+Requires-Dist: transformers<5.0.0,>=4.51.0; extra == "all"
+Requires-Dist: accelerate>=0.26.0; extra == "all"
+Requires-Dist: openai>=1.0; extra == "all"
+Requires-Dist: captum>=0.7; extra == "all"
+Requires-Dist: inseq>=0.6; extra == "all"
+Requires-Dist: nnsight>=0.3; extra == "all"
+Requires-Dist: matplotlib>=3.7; extra == "all"
+Requires-Dist: statsmodels>=0.14; extra == "all"
+Requires-Dist: scikit-learn>=1.3; extra == "all"
+Requires-Dist: streamlit>=1.30; extra == "all"
+Requires-Dist: pandas>=2.0; extra == "all"
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0; extra == "dev"
+Requires-Dist: ruff>=0.4; extra == "dev"
+Requires-Dist: mypy>=1.8; extra == "dev"
+Requires-Dist: jsonschema>=4.0; extra == "dev"
+Dynamic: license-file
+
+# EvalVitals
+
+[![CI](https://github.com/evalvitals/evalvitals/actions/workflows/ci.yml/badge.svg)](https://github.com/evalvitals/evalvitals/actions/workflows/ci.yml)
+[![PyPI](https://img.shields.io/pypi/v/evalvitals)](https://pypi.org/project/evalvitals/)
+[![Python](https://img.shields.io/pypi/pyversions/evalvitals)](https://pypi.org/project/evalvitals/)
+[![Docs](https://img.shields.io/badge/docs-mkdocs-blue)](https://evalvitals.github.io/evalvitals/)
+[![License: CC0-1.0](https://img.shields.io/badge/license-CC0--1.0-green)](LICENSE)
+
+EvalVitals is an sklearn-like toolkit for failure-case analysis in the era of
+LLMs, VLMs, omni (text+image+audio+video) models, and agentic systems. Model
+identity (`ModelSpec`) is separate from runtime (`Backend`); analyzers
+(`Analyzer`) connect to models by capability matching (`Capability`), so the
+same spec runs through a black-box API or a white-box local backend — only
+the capability set changes.
+
+| Contract | Role |
+|---|---|
+| `ModelSpec` | Model identity: family, HF repo, architecture traits, modalities. |
+| `Backend` | Runtime: local HF internals, black-box API, or offline batch engines. |
+| `Model` | A runnable object with `generate()` and `forward(capture=...) -> Trace`. |
+| `Analyzer` | An sklearn-style estimator: `Analyzer(**params).run(model, data) -> Result`. |
+| `Capability` | Vocabulary matching analyzers to compatible model runtimes. |
+| `FailureCase` | Central data unit for prompts, labels, provenance, agent trajectories. |
+| `Result` | Uniform output: human-readable summary + structured findings. |
+
+## What's Inside
+
+EvalVitals covers three layers of the analysis workflow, usable independently
+or chained into one automated loop:
+
+1. **Analyzer toolkit** — 26 registered analyzers (attention, uncertainty,
+   hallucination, Shapley attribution, logit lens, representation geometry,
+   agent-trajectory analysis) run against any model/backend pair through the
+   same `Analyzer(**params).run(model, data) -> Result` call shape. See the
+   [Demo](#demo) below and the [Analyzer Zoo](docs/analyzers.md).
+2. **Data analysis agent (M2/M3)** — `evalvitals explore` points a coding
+   agent at a raw results directory (any JSON/JSONL shape, no host-side
+   parsing) and gets back descriptive takeaways, charts, and 1-3 falsifiable
+   hypotheses — no code required. See
+   [Exploratory Analysis (M2/M3)](docs/m2_analysis.md).
+3. **Intervention (M4/M5)** — `HypothesisTester` verifies a hypothesis
+   statistically and against the stated experiment protocol; `FixAgent` then
+   proposes and validates candidate repairs (prompt → scaffold → internals →
+   parameter space) against the unmodified baseline with paired McNemar +
+   e-value. See [Intervention & Verification (M4/M5)](docs/intervention.md).
+
+`VLDiagnoseLoop` chains all three (M1→M2→M3→M5, M4 post-loop) into one
+automated failure-attribution pipeline — see [Quickstart](docs/quickstart.md).
+
+## Demo
+
+Relative attention on a VLM — ["MLLMs Know Where to Look"](https://arxiv.org/abs/2502.17422) ([code](https://github.com/saccharomycetes/mllms_know)):
+
+```python
+from PIL import Image
+from evalvitals import compose, Capability
+from evalvitals.analyzers.attention import RelativeAttentionAnalyzer
+from evalvitals.core.case import Inputs
+
+# Load Qwen2.5-VL with white-box attention capture
+model = compose("qwen2.5-vl-7b-instruct", "hf_local", want={Capability.ATTENTION})
+
+# Run relative attention: ratio of task-specific vs generic image attention
+result = RelativeAttentionAnalyzer(layer=22, top_k=5).run(
+    model,
+    Inputs(prompt="What color is the car?", image=Image.open("scene.jpg")),
+)
+
+print(result.summary())   # agent-readable findings
+result.plot()             # (H, W) heatmap — requires evalvitals[viz]
+```
+
+The same call shape works for any registered model/analyzer pair — a plain
+text LLM, a config-driven YAML run, or an explicit backend — see
+[Quickstart](docs/quickstart.md) for those and for the automated
+failure-attribution loop (`VLDiagnoseLoop`, M1→M2→M3→M5) and no-code
+exploratory analysis (`evalvitals explore`).
+
+## Install
+
+```bash
+pip install -e .
+pip install -e ".[viz]"
+pip install -e ".[dev]"
+```
+
+## Documentation
+
+- [Docs overview](docs/index.md)
+- [Quickstart](docs/quickstart.md) — runnable examples for every entry point, the diagnosis loop, and submitting a run
+- [Exploratory Analysis (M2/M3)](docs/m2_analysis.md) — standalone `evalvitals explore`
+- [Intervention & Verification (M4/M5)](docs/intervention.md) — `HypothesisTester` verification, `FixAgent` tiered repair
+- [Analyzer Zoo](docs/analyzers.md) — reference tables of analyzers and registered models
+- [Architecture](docs/architecture.md) — package structure and design contracts
+- [Extending EvalVitals](docs/extending.md) — add analyzers, specs, backends
+- [Roadmap](docs/roadmap.md)
+
+## Examples
+
+Each directory under `examples/` is a self-contained, runnable demo:
+
+```bash
+cd examples/analyzer_demos/qwen_attention  && docker compose up   # attention analysis on a text LLM
+cd examples/m2_statistics/deco_hallu_explore && docker compose up # M2/M3 explore, real M1 data
+cd examples/m2_statistics/deco_hallu_explore && bash run_attn.sh  # ... attention-enriched: FAIL/PASS distributions + cross-checkpoint geometry (no GPU; data ships with the repo)
+cd examples/m2_statistics/deco_hallu_explore && bash run_attn_pipeline.sh  # ... FULL held-out pipeline: propose → held-out test + LLM judge → L1..L3b fix → one 4-tab report (SKIP_FIX=1 for the no-GPU half)
+cd examples/diagnosis_loops/qwen_loop_agy  && docker compose up   # VLDiagnoseLoop M1→M5 (VLM)
+```
+
+See [`examples/README.md`](examples/README.md) for the full list, grouped by
+layer (single analyzers, standalone M2/M3, full diagnosis loop).

evalvitals-0.1.0/README.md

@@ -0,0 +1,111 @@
+# EvalVitals
+
+[![CI](https://github.com/evalvitals/evalvitals/actions/workflows/ci.yml/badge.svg)](https://github.com/evalvitals/evalvitals/actions/workflows/ci.yml)
+[![PyPI](https://img.shields.io/pypi/v/evalvitals)](https://pypi.org/project/evalvitals/)
+[![Python](https://img.shields.io/pypi/pyversions/evalvitals)](https://pypi.org/project/evalvitals/)
+[![Docs](https://img.shields.io/badge/docs-mkdocs-blue)](https://evalvitals.github.io/evalvitals/)
+[![License: CC0-1.0](https://img.shields.io/badge/license-CC0--1.0-green)](LICENSE)
+
+EvalVitals is an sklearn-like toolkit for failure-case analysis in the era of
+LLMs, VLMs, omni (text+image+audio+video) models, and agentic systems. Model
+identity (`ModelSpec`) is separate from runtime (`Backend`); analyzers
+(`Analyzer`) connect to models by capability matching (`Capability`), so the
+same spec runs through a black-box API or a white-box local backend — only
+the capability set changes.
+
+| Contract | Role |
+|---|---|
+| `ModelSpec` | Model identity: family, HF repo, architecture traits, modalities. |
+| `Backend` | Runtime: local HF internals, black-box API, or offline batch engines. |
+| `Model` | A runnable object with `generate()` and `forward(capture=...) -> Trace`. |
+| `Analyzer` | An sklearn-style estimator: `Analyzer(**params).run(model, data) -> Result`. |
+| `Capability` | Vocabulary matching analyzers to compatible model runtimes. |
+| `FailureCase` | Central data unit for prompts, labels, provenance, agent trajectories. |
+| `Result` | Uniform output: human-readable summary + structured findings. |
+
+## What's Inside
+
+EvalVitals covers three layers of the analysis workflow, usable independently
+or chained into one automated loop:
+
+1. **Analyzer toolkit** — 26 registered analyzers (attention, uncertainty,
+   hallucination, Shapley attribution, logit lens, representation geometry,
+   agent-trajectory analysis) run against any model/backend pair through the
+   same `Analyzer(**params).run(model, data) -> Result` call shape. See the
+   [Demo](#demo) below and the [Analyzer Zoo](docs/analyzers.md).
+2. **Data analysis agent (M2/M3)** — `evalvitals explore` points a coding
+   agent at a raw results directory (any JSON/JSONL shape, no host-side
+   parsing) and gets back descriptive takeaways, charts, and 1-3 falsifiable
+   hypotheses — no code required. See
+   [Exploratory Analysis (M2/M3)](docs/m2_analysis.md).
+3. **Intervention (M4/M5)** — `HypothesisTester` verifies a hypothesis
+   statistically and against the stated experiment protocol; `FixAgent` then
+   proposes and validates candidate repairs (prompt → scaffold → internals →
+   parameter space) against the unmodified baseline with paired McNemar +
+   e-value. See [Intervention & Verification (M4/M5)](docs/intervention.md).
+
+`VLDiagnoseLoop` chains all three (M1→M2→M3→M5, M4 post-loop) into one
+automated failure-attribution pipeline — see [Quickstart](docs/quickstart.md).
+
+## Demo
+
+Relative attention on a VLM — ["MLLMs Know Where to Look"](https://arxiv.org/abs/2502.17422) ([code](https://github.com/saccharomycetes/mllms_know)):
+
+```python
+from PIL import Image
+from evalvitals import compose, Capability
+from evalvitals.analyzers.attention import RelativeAttentionAnalyzer
+from evalvitals.core.case import Inputs
+
+# Load Qwen2.5-VL with white-box attention capture
+model = compose("qwen2.5-vl-7b-instruct", "hf_local", want={Capability.ATTENTION})
+
+# Run relative attention: ratio of task-specific vs generic image attention
+result = RelativeAttentionAnalyzer(layer=22, top_k=5).run(
+    model,
+    Inputs(prompt="What color is the car?", image=Image.open("scene.jpg")),
+)
+
+print(result.summary())   # agent-readable findings
+result.plot()             # (H, W) heatmap — requires evalvitals[viz]
+```
+
+The same call shape works for any registered model/analyzer pair — a plain
+text LLM, a config-driven YAML run, or an explicit backend — see
+[Quickstart](docs/quickstart.md) for those and for the automated
+failure-attribution loop (`VLDiagnoseLoop`, M1→M2→M3→M5) and no-code
+exploratory analysis (`evalvitals explore`).
+
+## Install
+
+```bash
+pip install -e .
+pip install -e ".[viz]"
+pip install -e ".[dev]"
+```
+
+## Documentation
+
+- [Docs overview](docs/index.md)
+- [Quickstart](docs/quickstart.md) — runnable examples for every entry point, the diagnosis loop, and submitting a run
+- [Exploratory Analysis (M2/M3)](docs/m2_analysis.md) — standalone `evalvitals explore`
+- [Intervention & Verification (M4/M5)](docs/intervention.md) — `HypothesisTester` verification, `FixAgent` tiered repair
+- [Analyzer Zoo](docs/analyzers.md) — reference tables of analyzers and registered models
+- [Architecture](docs/architecture.md) — package structure and design contracts
+- [Extending EvalVitals](docs/extending.md) — add analyzers, specs, backends
+- [Roadmap](docs/roadmap.md)
+
+## Examples
+
+Each directory under `examples/` is a self-contained, runnable demo:
+
+```bash
+cd examples/analyzer_demos/qwen_attention  && docker compose up   # attention analysis on a text LLM
+cd examples/m2_statistics/deco_hallu_explore && docker compose up # M2/M3 explore, real M1 data
+cd examples/m2_statistics/deco_hallu_explore && bash run_attn.sh  # ... attention-enriched: FAIL/PASS distributions + cross-checkpoint geometry (no GPU; data ships with the repo)
+cd examples/m2_statistics/deco_hallu_explore && bash run_attn_pipeline.sh  # ... FULL held-out pipeline: propose → held-out test + LLM judge → L1..L3b fix → one 4-tab report (SKIP_FIX=1 for the no-GPU half)
+cd examples/diagnosis_loops/qwen_loop_agy  && docker compose up   # VLDiagnoseLoop M1→M5 (VLM)
+```
+
+See [`examples/README.md`](examples/README.md) for the full list, grouped by
+layer (single analyzers, standalone M2/M3, full diagnosis loop).

evalvitals-0.1.0/evalvitals/__init__.py

@@ -0,0 +1,112 @@
+"""EvalVitals — failure case analysis for LLMs and VLMs.
+
+A model is built once from a **spec** (identity, in :mod:`evalvitals.specs`) and a
+**backend** (runtime: ``hf_local`` / ``api`` / ``vllm_offline``); the backend
+determines the capability set.  Analyzers are sklearn-style estimators matched to
+models by capability.
+
+Equivalent ways to run an analysis:
+
+0. Bring your own model — wrap an already-loaded HF model + tokenizer::
+
+    import evalvitals
+    from transformers import AutoModelForCausalLM, AutoTokenizer
+    from evalvitals.analyzers.lens.logit_lens import LogitLensAnalyzer
+
+    m = AutoModelForCausalLM.from_pretrained("my-org/my-llama")
+    tok = AutoTokenizer.from_pretrained("my-org/my-llama")
+    model = evalvitals.wrap(m, tok)                         # captum-style on-ramp
+    result = LogitLensAnalyzer().run(model, "The capital of France is")
+
+1. Curated checkpoints — build a model from the registry by key::
+
+    import evalvitals
+    from evalvitals.analyzers.attention.summary import AttentionAnalyzer
+
+    model = evalvitals.load("qwen2.5-7b-instruct")          # spec key
+    result = AttentionAnalyzer(top_k=5).run(model, "The capital of France is")
+    print(result.summary())
+
+2. Config-driven — declare model + analysis in YAML::
+
+    from evalvitals import load_config, run
+
+    config = load_config("configs/qwen_attention.yaml")
+    result = run(config, "The capital of France is")
+
+3. Hybrid convenience shim (auto-derived from capabilities)::
+
+    result = model.call_attention("The capital of France is")
+
+4. Explicit engine — pick the backend yourself::
+
+    from evalvitals.models import compose
+    model = compose("qwen2.5-7b-instruct", "hf_local", want={evalvitals.Capability.ATTENTION})
+"""
+
+# Importing these populates the registry (models + analyzers self-register).
+import evalvitals.analyzers as _analyzers  # noqa: E402,F401
+from evalvitals.config import AnalysisConfig, load_config
+from evalvitals.core import (
+    Analyzer,
+    Capability,
+    CaseBatch,
+    FailureCase,
+    Model,
+    Result,
+    registry,
+)
+from evalvitals.core.tool import Tool, ToolCall
+from evalvitals.models import Agent, RuntimeConfig, compose, load, load_model, wrap
+from evalvitals.specs import get_spec, list_specs
+
+__version__ = "0.1.0"
+__all__ = [
+    "load",
+    "load_config",
+    "load_model",
+    "wrap",
+    "compose",
+    "RuntimeConfig",
+    "Agent",
+    "Tool",
+    "ToolCall",
+    "get_spec",
+    "list_specs",
+    "run",
+    "AnalysisConfig",
+    "Model",
+    "Analyzer",
+    "Result",
+    "FailureCase",
+    "CaseBatch",
+    "Capability",
+    "registry",
+]
+
+
+def run(config: AnalysisConfig, data, **kwargs):
+    """Run the analysis declared by *config* on *data*.
+
+    Equivalent to::
+
+        analyzer = registry.analyzers.get(config.analysis)(**config.analysis_kwargs)
+        analyzer.run(load_model(config.model), data)
+
+    Args:
+        config: Loaded :class:`AnalysisConfig` (see :func:`load_config`).
+        data:   ``str | FailureCase | CaseBatch | iterable`` to analyse.
+        **kwargs: Override or extend ``config.analysis_kwargs`` at call time.
+
+    Returns:
+        A :class:`~evalvitals.core.result.Result` subclass.
+    """
+    # Back-compat: tolerate a leading "call_" in the config (old style).
+    name = config.analysis
+    if name.startswith("call_"):
+        name = name[len("call_"):]
+
+    analyzer_cls = registry.analyzers.get(name)
+    analyzer = analyzer_cls(**{**config.analysis_kwargs, **kwargs})
+    model = load_model(config.model)
+    return analyzer.run(model, data)

evalvitals-0.1.0/evalvitals/agent_assets/__init__.py

@@ -0,0 +1,2 @@
+"""Bundled runtime assets for EvalVitals agents."""
+