evalvitals 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- evalvitals/__init__.py +112 -0
- evalvitals/agent_assets/__init__.py +2 -0
- evalvitals/agent_assets/skills/README.md +28 -0
- evalvitals/agent_assets/skills/eval-chart-style/SKILL.md +113 -0
- evalvitals/agent_assets/skills/evalvitals-report-ui/SKILL.md +116 -0
- evalvitals/agent_assets/skills/nature-figure/LICENSE +201 -0
- evalvitals/agent_assets/skills/nature-figure/README.md +400 -0
- evalvitals/agent_assets/skills/nature-figure/SKILL.md +60 -0
- evalvitals/agent_assets/skills/nature-figure/manifest.yaml +59 -0
- evalvitals/agent_assets/skills/nature-figure/references/api.md +428 -0
- evalvitals/agent_assets/skills/nature-figure/references/backend-selection.md +100 -0
- evalvitals/agent_assets/skills/nature-figure/references/chart-types.md +281 -0
- evalvitals/agent_assets/skills/nature-figure/references/common-patterns.md +350 -0
- evalvitals/agent_assets/skills/nature-figure/references/demos.md +65 -0
- evalvitals/agent_assets/skills/nature-figure/references/design-theory.md +436 -0
- evalvitals/agent_assets/skills/nature-figure/references/figure-contract.md +93 -0
- evalvitals/agent_assets/skills/nature-figure/references/figure-legend-conventions.md +71 -0
- evalvitals/agent_assets/skills/nature-figure/references/nature-2026-observations.md +112 -0
- evalvitals/agent_assets/skills/nature-figure/references/qa-contract.md +119 -0
- evalvitals/agent_assets/skills/nature-figure/references/r-template-index.md +66 -0
- evalvitals/agent_assets/skills/nature-figure/references/r-workflow.md +161 -0
- evalvitals/agent_assets/skills/nature-figure/references/tutorials.md +251 -0
- evalvitals/agent_assets/skills/nature-figure/static/core/contract.md +29 -0
- evalvitals/agent_assets/skills/nature-figure/static/core/stance.md +37 -0
- evalvitals/agent_assets/skills/nature-figure/static/fragments/backend/python.md +37 -0
- evalvitals/agent_assets/skills/nature-figure/static/fragments/backend/r.md +44 -0
- evalvitals/agent_assets/skills/outcome-driver-analysis/SKILL.md +202 -0
- evalvitals/agent_assets/skills/outcome-driver-analysis/assets/analysis_report_template.md +53 -0
- evalvitals/agent_assets/skills/outcome-driver-analysis/references/model_selection.md +72 -0
- evalvitals/agent_assets/skills/outcome-driver-analysis/scripts/explanatory_var_eda.R +130 -0
- evalvitals/agent_assets/skills/outcome-driver-analysis/scripts/explanatory_var_eda.py +150 -0
- evalvitals/agent_assets/skills/outcome-driver-analysis/scripts/fit_outcome_model.R +181 -0
- evalvitals/agent_assets/skills/outcome-driver-analysis/scripts/fit_outcome_model.py +186 -0
- evalvitals/agent_assets/skills/outcome-driver-analysis/scripts/univariate_eda.R +149 -0
- evalvitals/agent_assets/skills/outcome-driver-analysis/scripts/univariate_eda.py +177 -0
- evalvitals/agent_assets/skills.py +27 -0
- evalvitals/analysis/__init__.py +107 -0
- evalvitals/analysis/adjudicate.py +154 -0
- evalvitals/analysis/cli.py +114 -0
- evalvitals/analysis/dashboard.py +206 -0
- evalvitals/analysis/dashboard_app.py +3283 -0
- evalvitals/analysis/eval_case_matrix.py +118 -0
- evalvitals/analysis/eval_viz_theme.py +785 -0
- evalvitals/analysis/explore_run.py +223 -0
- evalvitals/analysis/explorer.py +1101 -0
- evalvitals/analysis/fused_pipeline.py +489 -0
- evalvitals/analysis/hypothesis_agent.py +187 -0
- evalvitals/analysis/operationalize.py +442 -0
- evalvitals/analysis/planner.py +165 -0
- evalvitals/analysis/profile.py +268 -0
- evalvitals/analysis/stats_agent.py +8 -0
- evalvitals/analysis/stats_tools.py +25 -0
- evalvitals/analyzers/__init__.py +39 -0
- evalvitals/analyzers/agent/__init__.py +13 -0
- evalvitals/analyzers/agent/counterfactual.py +81 -0
- evalvitals/analyzers/agent/first_error_judge.py +92 -0
- evalvitals/analyzers/agent/ignored_obs.py +77 -0
- evalvitals/analyzers/agent/loop_detect.py +75 -0
- evalvitals/analyzers/attention/__init__.py +19 -0
- evalvitals/analyzers/attention/relative_attn.py +607 -0
- evalvitals/analyzers/attention/rollout.py +73 -0
- evalvitals/analyzers/attention/sink.py +56 -0
- evalvitals/analyzers/attention/summary.py +190 -0
- evalvitals/analyzers/attribution/__init__.py +6 -0
- evalvitals/analyzers/attribution/generic_attn.py +31 -0
- evalvitals/analyzers/attribution/gradcam.py +30 -0
- evalvitals/analyzers/base.py +12 -0
- evalvitals/analyzers/geometry/__init__.py +6 -0
- evalvitals/analyzers/geometry/cka.py +70 -0
- evalvitals/analyzers/geometry/linear_probe.py +157 -0
- evalvitals/analyzers/hallucination/__init__.py +8 -0
- evalvitals/analyzers/hallucination/chair.py +75 -0
- evalvitals/analyzers/hallucination/opera.py +26 -0
- evalvitals/analyzers/hallucination/pope.py +107 -0
- evalvitals/analyzers/hallucination/vcd.py +26 -0
- evalvitals/analyzers/lens/__init__.py +6 -0
- evalvitals/analyzers/lens/logit_lens.py +130 -0
- evalvitals/analyzers/lens/tuned_lens.py +30 -0
- evalvitals/analyzers/patching/__init__.py +5 -0
- evalvitals/analyzers/patching/causal_trace.py +30 -0
- evalvitals/analyzers/perturbation/__init__.py +8 -0
- evalvitals/analyzers/perturbation/_shapley.py +54 -0
- evalvitals/analyzers/perturbation/mm_shap.py +97 -0
- evalvitals/analyzers/perturbation/prompt_contrast.py +244 -0
- evalvitals/analyzers/perturbation/rise.py +94 -0
- evalvitals/analyzers/perturbation/vl_shap.py +99 -0
- evalvitals/analyzers/uncertainty/__init__.py +14 -0
- evalvitals/analyzers/uncertainty/entropy.py +90 -0
- evalvitals/analyzers/uncertainty/logprob_entropy.py +69 -0
- evalvitals/analyzers/uncertainty/self_consistency.py +64 -0
- evalvitals/analyzers/uncertainty/verbalized_conf.py +64 -0
- evalvitals/cli.py +123 -0
- evalvitals/config.py +77 -0
- evalvitals/core/__init__.py +85 -0
- evalvitals/core/analyzer.py +129 -0
- evalvitals/core/capability.py +54 -0
- evalvitals/core/case.py +320 -0
- evalvitals/core/experiment.py +106 -0
- evalvitals/core/model.py +198 -0
- evalvitals/core/pipeline.py +42 -0
- evalvitals/core/registry.py +142 -0
- evalvitals/core/result.py +64 -0
- evalvitals/core/spec.py +155 -0
- evalvitals/core/tokentype.py +165 -0
- evalvitals/core/tool.py +66 -0
- evalvitals/datasets/__init__.py +41 -0
- evalvitals/datasets/base.py +68 -0
- evalvitals/datasets/gui_os.py +52 -0
- evalvitals/datasets/llm_qa.py +57 -0
- evalvitals/datasets/pure_qa.py +12 -0
- evalvitals/datasets/vlm_qa.py +695 -0
- evalvitals/datasets/web_search_qa.py +52 -0
- evalvitals/eval_agent/__init__.py +315 -0
- evalvitals/eval_agent/_docker_runner.py +89 -0
- evalvitals/eval_agent/_tools.py +81 -0
- evalvitals/eval_agent/ab_runner.py +50 -0
- evalvitals/eval_agent/cli_agent.py +56 -0
- evalvitals/eval_agent/cli_runtime.py +97 -0
- evalvitals/eval_agent/cli_skills.py +5 -0
- evalvitals/eval_agent/cli_transcript.py +138 -0
- evalvitals/eval_agent/cli_types.py +63 -0
- evalvitals/eval_agent/codegen/__init__.py +5 -0
- evalvitals/eval_agent/codegen/runner.py +88 -0
- evalvitals/eval_agent/evolution.py +396 -0
- evalvitals/eval_agent/experiment_harness.py +117 -0
- evalvitals/eval_agent/factory.py +102 -0
- evalvitals/eval_agent/git_manager.py +215 -0
- evalvitals/eval_agent/hypothesis.py +128 -0
- evalvitals/eval_agent/log_schema.py +332 -0
- evalvitals/eval_agent/loop.py +1741 -0
- evalvitals/eval_agent/models/__init__.py +6 -0
- evalvitals/eval_agent/models/agy.py +144 -0
- evalvitals/eval_agent/models/claude.py +114 -0
- evalvitals/eval_agent/nl_runner.py +460 -0
- evalvitals/eval_agent/orchestrator.py +61 -0
- evalvitals/eval_agent/preregister.py +93 -0
- evalvitals/eval_agent/prompts/__init__.py +1 -0
- evalvitals/eval_agent/prompts/case_discovery.py +25 -0
- evalvitals/eval_agent/prompts/diagnosis.py +48 -0
- evalvitals/eval_agent/prompts/experiment_writer.py +253 -0
- evalvitals/eval_agent/prompts/explorer.py +319 -0
- evalvitals/eval_agent/prompts/fix_agent.py +124 -0
- evalvitals/eval_agent/prompts/hypothesis_tester.py +15 -0
- evalvitals/eval_agent/prompts/nl_runner.py +35 -0
- evalvitals/eval_agent/prompts/probe_agent.py +25 -0
- evalvitals/eval_agent/prompts/probe_generator.py +35 -0
- evalvitals/eval_agent/prompts/stats_agent.py +44 -0
- evalvitals/eval_agent/prompts/stats_tool_generator.py +43 -0
- evalvitals/eval_agent/prompts/whitebox_probe_generator.py +38 -0
- evalvitals/eval_agent/providers/__init__.py +21 -0
- evalvitals/eval_agent/providers/antigravity.py +27 -0
- evalvitals/eval_agent/providers/base.py +97 -0
- evalvitals/eval_agent/providers/claude_code.py +39 -0
- evalvitals/eval_agent/providers/codex.py +33 -0
- evalvitals/eval_agent/providers/gemini_cli.py +26 -0
- evalvitals/eval_agent/providers/kimi_cli.py +27 -0
- evalvitals/eval_agent/providers/opencode.py +27 -0
- evalvitals/eval_agent/providers/registry.py +58 -0
- evalvitals/eval_agent/report.py +58 -0
- evalvitals/eval_agent/run_context.py +460 -0
- evalvitals/eval_agent/run_log.schema.json +717 -0
- evalvitals/eval_agent/run_logger.py +1454 -0
- evalvitals/eval_agent/sandbox.py +517 -0
- evalvitals/eval_agent/skills/__init__.py +19 -0
- evalvitals/eval_agent/skills/installer.py +68 -0
- evalvitals/eval_agent/skills/prompt_policy.py +85 -0
- evalvitals/eval_agent/skills/resolver.py +19 -0
- evalvitals/eval_agent/stages/__init__.py +49 -0
- evalvitals/eval_agent/stages/analysis.py +316 -0
- evalvitals/eval_agent/stages/case_discovery.py +222 -0
- evalvitals/eval_agent/stages/diagnosis.py +571 -0
- evalvitals/eval_agent/stages/experiment_writer.py +1488 -0
- evalvitals/eval_agent/stages/fix_agent.py +1341 -0
- evalvitals/eval_agent/stages/fix_internals.py +209 -0
- evalvitals/eval_agent/stages/fix_pipeline.py +284 -0
- evalvitals/eval_agent/stages/fix_tiers.py +177 -0
- evalvitals/eval_agent/stages/fix_tools.py +629 -0
- evalvitals/eval_agent/stages/hypothesis_tester.py +695 -0
- evalvitals/eval_agent/stages/probe.py +173 -0
- evalvitals/eval_agent/stages/probe_agent.py +859 -0
- evalvitals/eval_agent/stages/probe_generator.py +302 -0
- evalvitals/eval_agent/stages/protocol.py +96 -0
- evalvitals/eval_agent/stages/stats_agent.py +856 -0
- evalvitals/eval_agent/stages/stats_tool_agent.py +261 -0
- evalvitals/eval_agent/stages/stats_tool_generator.py +407 -0
- evalvitals/eval_agent/stages/stats_tools.py +1114 -0
- evalvitals/eval_agent/stages/surgery.py +491 -0
- evalvitals/eval_agent/stages/whitebox_probe_generator.py +327 -0
- evalvitals/eval_agent/store.py +231 -0
- evalvitals/models/__init__.py +161 -0
- evalvitals/models/_discover.py +101 -0
- evalvitals/models/agent.py +198 -0
- evalvitals/models/backends/__init__.py +38 -0
- evalvitals/models/backends/api.py +150 -0
- evalvitals/models/backends/base.py +50 -0
- evalvitals/models/backends/hf_local.py +452 -0
- evalvitals/models/backends/vllm_offline.py +116 -0
- evalvitals/models/base.py +24 -0
- evalvitals/models/blackbox/__init__.py +4 -0
- evalvitals/models/blackbox/agent.py +31 -0
- evalvitals/models/blackbox/base.py +29 -0
- evalvitals/models/blackbox/gemini.py +79 -0
- evalvitals/models/blackbox/llm_api.py +17 -0
- evalvitals/models/blackbox/vlm_api.py +17 -0
- evalvitals/models/compose.py +66 -0
- evalvitals/models/inference.py +88 -0
- evalvitals/models/toolcodec.py +140 -0
- evalvitals/models/whitebox/__init__.py +16 -0
- evalvitals/models/whitebox/agent.py +31 -0
- evalvitals/models/whitebox/base.py +24 -0
- evalvitals/models/whitebox/qwen.py +61 -0
- evalvitals/models/whitebox/qwen_omni.py +53 -0
- evalvitals/models/whitebox/qwen_vl.py +62 -0
- evalvitals/reporting/__init__.py +16 -0
- evalvitals/reporting/compiler.py +436 -0
- evalvitals/reporting/model.py +105 -0
- evalvitals/reporting/stages.py +59 -0
- evalvitals/specs.py +331 -0
- evalvitals/stats/__init__.py +38 -0
- evalvitals/stats/api.py +142 -0
- evalvitals/stats/bootstrap.py +86 -0
- evalvitals/stats/ebh.py +27 -0
- evalvitals/stats/evalue.py +40 -0
- evalvitals/stats/friedman.py +138 -0
- evalvitals/stats/mcnemar.py +40 -0
- evalvitals/stats/multiplicity.py +159 -0
- evalvitals/stats/subset_sampling.py +55 -0
- evalvitals/viz/__init__.py +7 -0
- evalvitals/viz/labels.py +77 -0
- evalvitals/viz/prompts.py +39 -0
- evalvitals/viz/renderer.py +263 -0
- evalvitals/viz/schema.py +36 -0
- evalvitals/viz/style.py +106 -0
- evalvitals-0.1.0.dist-info/METADATA +300 -0
- evalvitals-0.1.0.dist-info/RECORD +239 -0
- evalvitals-0.1.0.dist-info/WHEEL +5 -0
- evalvitals-0.1.0.dist-info/entry_points.txt +3 -0
- evalvitals-0.1.0.dist-info/licenses/LICENSE +121 -0
- evalvitals-0.1.0.dist-info/top_level.txt +1 -0
evalvitals/__init__.py
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"""EvalVitals — failure case analysis for LLMs and VLMs.
|
|
2
|
+
|
|
3
|
+
A model is built once from a **spec** (identity, in :mod:`evalvitals.specs`) and a
|
|
4
|
+
**backend** (runtime: ``hf_local`` / ``api`` / ``vllm_offline``); the backend
|
|
5
|
+
determines the capability set. Analyzers are sklearn-style estimators matched to
|
|
6
|
+
models by capability.
|
|
7
|
+
|
|
8
|
+
Equivalent ways to run an analysis:
|
|
9
|
+
|
|
10
|
+
0. Bring your own model — wrap an already-loaded HF model + tokenizer::
|
|
11
|
+
|
|
12
|
+
import evalvitals
|
|
13
|
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
|
14
|
+
from evalvitals.analyzers.lens.logit_lens import LogitLensAnalyzer
|
|
15
|
+
|
|
16
|
+
m = AutoModelForCausalLM.from_pretrained("my-org/my-llama")
|
|
17
|
+
tok = AutoTokenizer.from_pretrained("my-org/my-llama")
|
|
18
|
+
model = evalvitals.wrap(m, tok) # captum-style on-ramp
|
|
19
|
+
result = LogitLensAnalyzer().run(model, "The capital of France is")
|
|
20
|
+
|
|
21
|
+
1. Curated checkpoints — build a model from the registry by key::
|
|
22
|
+
|
|
23
|
+
import evalvitals
|
|
24
|
+
from evalvitals.analyzers.attention.summary import AttentionAnalyzer
|
|
25
|
+
|
|
26
|
+
model = evalvitals.load("qwen2.5-7b-instruct") # spec key
|
|
27
|
+
result = AttentionAnalyzer(top_k=5).run(model, "The capital of France is")
|
|
28
|
+
print(result.summary())
|
|
29
|
+
|
|
30
|
+
2. Config-driven — declare model + analysis in YAML::
|
|
31
|
+
|
|
32
|
+
from evalvitals import load_config, run
|
|
33
|
+
|
|
34
|
+
config = load_config("configs/qwen_attention.yaml")
|
|
35
|
+
result = run(config, "The capital of France is")
|
|
36
|
+
|
|
37
|
+
3. Hybrid convenience shim (auto-derived from capabilities)::
|
|
38
|
+
|
|
39
|
+
result = model.call_attention("The capital of France is")
|
|
40
|
+
|
|
41
|
+
4. Explicit engine — pick the backend yourself::
|
|
42
|
+
|
|
43
|
+
from evalvitals.models import compose
|
|
44
|
+
model = compose("qwen2.5-7b-instruct", "hf_local", want={evalvitals.Capability.ATTENTION})
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
# Importing these populates the registry (models + analyzers self-register).
|
|
48
|
+
import evalvitals.analyzers as _analyzers # noqa: E402,F401
|
|
49
|
+
from evalvitals.config import AnalysisConfig, load_config
|
|
50
|
+
from evalvitals.core import (
|
|
51
|
+
Analyzer,
|
|
52
|
+
Capability,
|
|
53
|
+
CaseBatch,
|
|
54
|
+
FailureCase,
|
|
55
|
+
Model,
|
|
56
|
+
Result,
|
|
57
|
+
registry,
|
|
58
|
+
)
|
|
59
|
+
from evalvitals.core.tool import Tool, ToolCall
|
|
60
|
+
from evalvitals.models import Agent, RuntimeConfig, compose, load, load_model, wrap
|
|
61
|
+
from evalvitals.specs import get_spec, list_specs
|
|
62
|
+
|
|
63
|
+
__version__ = "0.1.0"
|
|
64
|
+
__all__ = [
|
|
65
|
+
"load",
|
|
66
|
+
"load_config",
|
|
67
|
+
"load_model",
|
|
68
|
+
"wrap",
|
|
69
|
+
"compose",
|
|
70
|
+
"RuntimeConfig",
|
|
71
|
+
"Agent",
|
|
72
|
+
"Tool",
|
|
73
|
+
"ToolCall",
|
|
74
|
+
"get_spec",
|
|
75
|
+
"list_specs",
|
|
76
|
+
"run",
|
|
77
|
+
"AnalysisConfig",
|
|
78
|
+
"Model",
|
|
79
|
+
"Analyzer",
|
|
80
|
+
"Result",
|
|
81
|
+
"FailureCase",
|
|
82
|
+
"CaseBatch",
|
|
83
|
+
"Capability",
|
|
84
|
+
"registry",
|
|
85
|
+
]
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def run(config: AnalysisConfig, data, **kwargs):
|
|
89
|
+
"""Run the analysis declared by *config* on *data*.
|
|
90
|
+
|
|
91
|
+
Equivalent to::
|
|
92
|
+
|
|
93
|
+
analyzer = registry.analyzers.get(config.analysis)(**config.analysis_kwargs)
|
|
94
|
+
analyzer.run(load_model(config.model), data)
|
|
95
|
+
|
|
96
|
+
Args:
|
|
97
|
+
config: Loaded :class:`AnalysisConfig` (see :func:`load_config`).
|
|
98
|
+
data: ``str | FailureCase | CaseBatch | iterable`` to analyse.
|
|
99
|
+
**kwargs: Override or extend ``config.analysis_kwargs`` at call time.
|
|
100
|
+
|
|
101
|
+
Returns:
|
|
102
|
+
A :class:`~evalvitals.core.result.Result` subclass.
|
|
103
|
+
"""
|
|
104
|
+
# Back-compat: tolerate a leading "call_" in the config (old style).
|
|
105
|
+
name = config.analysis
|
|
106
|
+
if name.startswith("call_"):
|
|
107
|
+
name = name[len("call_"):]
|
|
108
|
+
|
|
109
|
+
analyzer_cls = registry.analyzers.get(name)
|
|
110
|
+
analyzer = analyzer_cls(**{**config.analysis_kwargs, **kwargs})
|
|
111
|
+
model = load_model(config.model)
|
|
112
|
+
return analyzer.run(model, data)
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# Bundled Agent Skills
|
|
2
|
+
|
|
3
|
+
Agent Skills vendored into the package so they travel with the repo (both
|
|
4
|
+
`git clone` and `pip install`) and are auto-discovered by `evalvitals explore`
|
|
5
|
+
on the **claude** / **agy** backends. They style the figures the exploratory
|
|
6
|
+
coding agent writes under `figures/` — they do **not** affect the host-rendered,
|
|
7
|
+
deterministic chart specs (`render_chart_specs`).
|
|
8
|
+
|
|
9
|
+
Each `explore` run vendors these into its sandbox at
|
|
10
|
+
`<workdir>/.claude/skills/<name>/`, which an `--add-dir <workdir>` Claude Code /
|
|
11
|
+
agy run discovers automatically.
|
|
12
|
+
|
|
13
|
+
## nature-figure
|
|
14
|
+
|
|
15
|
+
Submission-grade Nature/high-impact-journal matplotlib/ggplot figure workflow.
|
|
16
|
+
|
|
17
|
+
- **Source:** https://github.com/Yuan1z0825/nature-skills (`skills/nature-figure`)
|
|
18
|
+
- **License:** Apache-2.0 — see [`nature-figure/LICENSE`](nature-figure/LICENSE).
|
|
19
|
+
- **Trimmed:** only the functional core is vendored (SKILL.md, manifest.yaml,
|
|
20
|
+
`static/`, `references/`, ~176 KB). The upstream `assets/` demo galleries
|
|
21
|
+
(~30 MB of example PNGs and figures4papers scripts) are **omitted** to keep
|
|
22
|
+
the repo lean; fetch them from the source repo if you want the demo galleries.
|
|
23
|
+
|
|
24
|
+
## Disabling / overriding
|
|
25
|
+
|
|
26
|
+
- `evalvitals explore … --no-skills` skips bundled skills for a run.
|
|
27
|
+
- `evalvitals explore … --skill /path/to/other-skill` adds more skill dirs.
|
|
28
|
+
- `--allow-skills` also lets globally-installed `~/.claude/skills` be used.
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: eval-chart-style
|
|
3
|
+
version: 0.2.0
|
|
4
|
+
description: >
|
|
5
|
+
Chart-type policy + house style for FAIL-vs-PASS LLM/VLM eval analysis
|
|
6
|
+
figures. Use whenever you plot eval results inside an EvalVitals analysis
|
|
7
|
+
sandbox — matplotlib PNGs under figures/ or deterministic chart specs — and
|
|
8
|
+
the point is to NOT default to a bar chart. This skill decides WHAT chart to
|
|
9
|
+
draw and the semantic palette; for journal-grade polish (fonts, export, QA)
|
|
10
|
+
additionally apply the nature-figure skill on the figures you draw.
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
# Eval Chart Style
|
|
14
|
+
|
|
15
|
+
House chart-type policy for FAIL-vs-PASS eval analysis. It exists to stop three
|
|
16
|
+
recurring failures: plotting a **mean as a bar** when the claim depends on the
|
|
17
|
+
*distribution*; inconsistent FAIL/PASS colors between charts; and raw machine
|
|
18
|
+
output (column ids, `pandas.cut` edges) leaking into axes and titles.
|
|
19
|
+
|
|
20
|
+
It is the agent-side counterpart of the host module
|
|
21
|
+
`evalvitals.analysis.eval_viz_theme` (plotly). If plotly is available you may
|
|
22
|
+
`from evalvitals.analysis import eval_viz_theme as viz; viz.apply()` and use its
|
|
23
|
+
builders; otherwise apply the rules below directly in matplotlib.
|
|
24
|
+
|
|
25
|
+
## 0. Chart-type policy (the most important rule)
|
|
26
|
+
|
|
27
|
+
**A bar's filled area means "amount accumulated from zero." Use bars only for
|
|
28
|
+
counts.** For a mean, a proportion, a measurement, or an effect size, a bar
|
|
29
|
+
hides the distribution and fakes certainty. Pick by *what the reader must see*:
|
|
30
|
+
|
|
31
|
+
| What you're showing | DON'T | DO |
|
|
32
|
+
|---|---|---|
|
|
33
|
+
| Discrete class counts (FAIL/PASS n) | — | grouped bars, n annotated |
|
|
34
|
+
| A continuous signal across outcomes | two-bar "mean by outcome" | violin/box + jittered points, one panel |
|
|
35
|
+
| Several signals' effects ranked | green/grey bars | horizontal **dot + CI** (forest) |
|
|
36
|
+
| Fail rate vs a continuous signal | bare binned bars | binned rate **line** (or logistic curve) + n per bin |
|
|
37
|
+
| Two continuous signals jointly | scatter squeezed small | full-width scatter colored by outcome (+ marginals if easy) |
|
|
38
|
+
| Rates across two categoricals | grouped bars ×k | **heatmap** with annotated cells |
|
|
39
|
+
| Distribution shape / tail claims | histogram only | ECDF or KDE overlay by outcome |
|
|
40
|
+
| Paired/intervention outcomes | two bars | paired slope or discordant counts |
|
|
41
|
+
|
|
42
|
+
Why distributions: with small FAIL n a mean is outlier-driven, and any
|
|
43
|
+
"two sub-populations" story is a *bimodality claim* — a bar can show neither.
|
|
44
|
+
|
|
45
|
+
**Suppress degenerate charts.** A binary/constant signal's fail-rate curve
|
|
46
|
+
collapses to one or two dots — don't render it; note it in caveats instead.
|
|
47
|
+
Aim for *diversity that matches the data*: across the figure set prefer
|
|
48
|
+
violin + ECDF + heatmap + forest + scatter over eight bar charts.
|
|
49
|
+
|
|
50
|
+
## 1. Semantic palette (color encodes role, not decoration)
|
|
51
|
+
|
|
52
|
+
Lock these and reuse in every figure — the reader must never re-learn who is who:
|
|
53
|
+
|
|
54
|
+
- FAIL → warning red `#C0413B`; PASS → neutral slate `#5B7A99` (same side, same
|
|
55
|
+
hue in every chart).
|
|
56
|
+
- Supported / survived-adjudication → green `#2E8B6F`; inconclusive → grey
|
|
57
|
+
`#B8BCC2`. Unknown significance defaults to grey, never green.
|
|
58
|
+
- Leakage/sanity signals → muted grey `#9AA0A6`, never a "winner" color.
|
|
59
|
+
|
|
60
|
+
Non-outcome dimensions get their own color logic — do NOT squeeze every panel
|
|
61
|
+
into the FAIL/PASS pair plus grey:
|
|
62
|
+
|
|
63
|
+
- **Categorical series** (checkpoint/model lines, object classes, several
|
|
64
|
+
signals overlaid on one axis): use this series order, in order —
|
|
65
|
+
`#0F4D92` deep blue, `#8BCF8B` green, `#B64342` red\*, `#42949E` teal,
|
|
66
|
+
`#9A4D8E` violet, `#CFCECE` neutral grey. (\*Skip the red slot whenever
|
|
67
|
+
FAIL-red already appears on the same panel — the outcome hue must stay
|
|
68
|
+
unambiguous.)
|
|
69
|
+
- **Ordered dimensions** (model size 2B→4B→8B, ordered bins, dose/strength
|
|
70
|
+
ladders): ONE hue with a luminance/alpha ramp (light → dark = small →
|
|
71
|
+
large), not distinct hues — the ordering should be readable from the ramp
|
|
72
|
+
alone. E.g. three blues `#B4C0E4 → #7884B4 → #0F4D92`.
|
|
73
|
+
- Precedence: semantic role colors always win — wherever the outcome appears,
|
|
74
|
+
FAIL/PASS keep their hues; the series/ramp colors are for panels sliced by
|
|
75
|
+
something other than the outcome. Heatmaps: diverging data (signed effects)
|
|
76
|
+
→ Red-Blue diverging around 0; magnitude-only data → a single-hue
|
|
77
|
+
sequential ramp.
|
|
78
|
+
|
|
79
|
+
Never print a raw column id (`generated_probe1_false_detection`) on an axis,
|
|
80
|
+
tick, or title — use a short human alias (≤ ~12 chars) and keep the raw name in
|
|
81
|
+
a caption/hover/table only.
|
|
82
|
+
|
|
83
|
+
## 2. Layout
|
|
84
|
+
|
|
85
|
+
- Distribution plots and scatters get full width; never squeeze a scatter into
|
|
86
|
+
a narrow subplot.
|
|
87
|
+
- One title per chart; don't repeat the section heading inside the figure.
|
|
88
|
+
- Label every axis with the *short* name + unit; annotate the zero/reference
|
|
89
|
+
line on effect axes.
|
|
90
|
+
|
|
91
|
+
## 3. Numbers and bins
|
|
92
|
+
|
|
93
|
+
- effects/correlations → 2 dp; p-values/statistics → 3 dp; percents → integer
|
|
94
|
+
(`41%`); counts → integer.
|
|
95
|
+
- **Never display raw `pandas.cut` edges** (`(113.844, 233.066]`). Render
|
|
96
|
+
human bins: `114–233`. Annotate per-bin n when bins are sparse.
|
|
97
|
+
|
|
98
|
+
## 4. Statistical honesty
|
|
99
|
+
|
|
100
|
+
- A signal that *is* the outcome re-measured (fail rate 1.00/0.00, effect ≈
|
|
101
|
+
perfect) is target leakage: demote it to a greyed sanity-check, never rank
|
|
102
|
+
it #1.
|
|
103
|
+
- Pick ONE primary effect metric for headline figures; relegate the rest to a
|
|
104
|
+
table, and label which metric every axis shows.
|
|
105
|
+
- Descriptive is descriptive: no "confirmed/validated" wording in titles.
|
|
106
|
+
|
|
107
|
+
## Scope note for EvalVitals sandboxes
|
|
108
|
+
|
|
109
|
+
Host-rendered chart *specs* stay deterministic (`kind` ∈ bar/line/scatter with
|
|
110
|
+
pre-aggregated CSVs) — apply §1/§3 to their data and titles. The chart-type
|
|
111
|
+
diversity of §0 lives in the PNGs you draw under `figures/` — that is where
|
|
112
|
+
violins, ECDFs, heatmaps, forests, and paired-slope figures belong. Styling
|
|
113
|
+
only: never change the data, the analysis, or the final result JSON.
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: evalvitals-report-ui
|
|
3
|
+
description: Generate EvalVitals exploratory results that read like diagnostic reports, not raw feature dumps.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# EvalVitals Report/UI Contract
|
|
7
|
+
|
|
8
|
+
Use this skill whenever you produce EvalVitals exploratory analysis JSON,
|
|
9
|
+
dashboard-ready artifacts, chart specs, or diagnostic report text.
|
|
10
|
+
|
|
11
|
+
## Audience
|
|
12
|
+
|
|
13
|
+
The reader is evaluating a model failure mode. They should not need to know
|
|
14
|
+
internal probe ids, generated column names, or implementation details to
|
|
15
|
+
understand the result.
|
|
16
|
+
|
|
17
|
+
## Stage Semantics
|
|
18
|
+
|
|
19
|
+
Use these stage names consistently:
|
|
20
|
+
|
|
21
|
+
- **M1 — Measurement:** frozen per-case data, analyzer/probe outputs, attention
|
|
22
|
+
maps, and derived signals. Dashboard role: Problem Setting.
|
|
23
|
+
- **M2 — Confirmatory analysis:** statistical comparisons of FAIL vs PASS,
|
|
24
|
+
effect sizes, CIs, e-values/e-BH, and deterministic charts. Dashboard role:
|
|
25
|
+
Analysis.
|
|
26
|
+
- **M3 — Hypothesis generation:** falsifiable explanations formed from M2
|
|
27
|
+
evidence plus exploratory context. Dashboard role: Hypotheses.
|
|
28
|
+
- **M4 — Mechanism test:** targeted experiments/interventions that test whether
|
|
29
|
+
a mechanism is real. Dashboard role: decision evidence.
|
|
30
|
+
- **M5 — Repair/surgery test:** proposed repair/fix outcomes and regression
|
|
31
|
+
checks. Dashboard role: final action gate.
|
|
32
|
+
|
|
33
|
+
Every dashboard/report should make clear which stage produced each claim,
|
|
34
|
+
chart, hypothesis, or artifact.
|
|
35
|
+
|
|
36
|
+
## Dashboard Storyboard
|
|
37
|
+
|
|
38
|
+
Do not produce an artifact dump. Produce a three-panel reader path:
|
|
39
|
+
|
|
40
|
+
1. **Problem Setting (M1 + run context)**
|
|
41
|
+
- What user question is being answered?
|
|
42
|
+
- What data/cases were provided?
|
|
43
|
+
- What is FAIL vs PASS?
|
|
44
|
+
- Which signals/analyzers are available?
|
|
45
|
+
|
|
46
|
+
2. **Analysis (M2)**
|
|
47
|
+
- For every analysis method, write: `method`, `evidence/chart`, `takeaway`.
|
|
48
|
+
- Put the main chart next to the method it supports.
|
|
49
|
+
- Put table details below the chart, not before the takeaway.
|
|
50
|
+
|
|
51
|
+
3. **Hypotheses & Artifacts (M3-M5)**
|
|
52
|
+
- List hypotheses only after M2 evidence.
|
|
53
|
+
- Link each hypothesis to cited charts/signals.
|
|
54
|
+
- Summarize M4/M5 test or fix outcomes.
|
|
55
|
+
- Keep raw artifacts in expandable drill-down sections.
|
|
56
|
+
|
|
57
|
+
Emit the reader path as `dashboard_storyboard` in the final JSON. The dashboard
|
|
58
|
+
renderer is intentionally stable and should only render this artifact. The
|
|
59
|
+
agent, not the Streamlit host, owns the run-specific narrative.
|
|
60
|
+
|
|
61
|
+
Required storyboard panel shape:
|
|
62
|
+
|
|
63
|
+
```json
|
|
64
|
+
{
|
|
65
|
+
"id": "problem_setting | analysis | hypotheses_artifacts",
|
|
66
|
+
"title": "Human panel title",
|
|
67
|
+
"stages": ["M1"],
|
|
68
|
+
"summary": "One paragraph written for the user",
|
|
69
|
+
"items": ["method -> evidence -> takeaway bullets"],
|
|
70
|
+
"artifact_refs": ["candidate_signals", "charts"]
|
|
71
|
+
}
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
## Required Output Discipline
|
|
75
|
+
|
|
76
|
+
- Every `candidate_signals` item must include:
|
|
77
|
+
- `name`: stable machine id, snake_case, suitable for recipes and joins.
|
|
78
|
+
- `display_name`: short human label, suitable for dashboard titles.
|
|
79
|
+
- `rationale`: one sentence in domain language.
|
|
80
|
+
- `suggested_test`: what would confirm or refute it.
|
|
81
|
+
- Every chart spec should include:
|
|
82
|
+
- `name`: stable machine id.
|
|
83
|
+
- `title` or `display_name`: human-readable, no raw probe/generated id unless
|
|
84
|
+
no interpretation is known.
|
|
85
|
+
- Every `visual_plan` item should include a readable `question` and avoid raw
|
|
86
|
+
field names in the user-facing wording.
|
|
87
|
+
- `chart_readings` and `claims` should cite what the chart means, not just which
|
|
88
|
+
column changed.
|
|
89
|
+
|
|
90
|
+
## Label-Like And Probe-Derived Signals
|
|
91
|
+
|
|
92
|
+
Signals such as `generated_probe1_false_detection`, `probe1_false_detection`,
|
|
93
|
+
or anything that perfectly re-measures the FAIL label are sanity checks. Treat
|
|
94
|
+
them as measurement checks, never as root causes.
|
|
95
|
+
|
|
96
|
+
Use labels like:
|
|
97
|
+
|
|
98
|
+
- `Sanity check: probe false-detection flag`
|
|
99
|
+
- `Probe says object is present`
|
|
100
|
+
- `Attention focus share`
|
|
101
|
+
- `Maximum relative attention`
|
|
102
|
+
- `Mean relative attention`
|
|
103
|
+
|
|
104
|
+
Do not lead a report with a label-reconstructing signal even when it has the largest
|
|
105
|
+
effect size. Mention it as a sanity check, then rank non-leaky explanatory
|
|
106
|
+
signals first.
|
|
107
|
+
|
|
108
|
+
## Overall Result Shape
|
|
109
|
+
|
|
110
|
+
The final answer should be claim-first:
|
|
111
|
+
|
|
112
|
+
1. State the supported non-leaky finding in plain language.
|
|
113
|
+
2. Mention label-reconstructing signals only as demoted sanity checks.
|
|
114
|
+
3. Link each claim to evidence: confirmed signal, chart, or downstream test.
|
|
115
|
+
4. Say what not to infer: association is not causality unless intervention tests
|
|
116
|
+
support it.
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
Version 2.0, January 2004
|
|
3
|
+
http://www.apache.org/licenses/
|
|
4
|
+
|
|
5
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
6
|
+
|
|
7
|
+
1. Definitions.
|
|
8
|
+
|
|
9
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
10
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
11
|
+
|
|
12
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
13
|
+
the copyright owner that is granting the License.
|
|
14
|
+
|
|
15
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
16
|
+
other entities that control, are controlled by, or are under common
|
|
17
|
+
control with that entity. For the purposes of this definition,
|
|
18
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
19
|
+
direction or management of such entity, whether by contract or
|
|
20
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
21
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
22
|
+
|
|
23
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
24
|
+
exercising permissions granted by this License.
|
|
25
|
+
|
|
26
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
27
|
+
including but not limited to software source code, documentation
|
|
28
|
+
source, and configuration files.
|
|
29
|
+
|
|
30
|
+
"Object" form shall mean any form resulting from mechanical
|
|
31
|
+
transformation or translation of a Source form, including but
|
|
32
|
+
not limited to compiled object code, generated documentation,
|
|
33
|
+
and conversions to other media types.
|
|
34
|
+
|
|
35
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
36
|
+
Object form, made available under the License, as indicated by a
|
|
37
|
+
copyright notice that is included in or attached to the work
|
|
38
|
+
(an example is provided in the Appendix below).
|
|
39
|
+
|
|
40
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
41
|
+
form, that is based on (or derived from) the Work and for which the
|
|
42
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
43
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
44
|
+
of this License, Derivative Works shall not include works that remain
|
|
45
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
46
|
+
the Work and Derivative Works thereof.
|
|
47
|
+
|
|
48
|
+
"Contribution" shall mean any work of authorship, including
|
|
49
|
+
the original version of the Work and any modifications or additions
|
|
50
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
51
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
52
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
53
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
54
|
+
means any form of electronic, verbal, or written communication sent
|
|
55
|
+
to the Licensor or its representatives, including but not limited to
|
|
56
|
+
communication on electronic mailing lists, source code control systems,
|
|
57
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
58
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
59
|
+
excluding communication that is conspicuously marked or otherwise
|
|
60
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
61
|
+
|
|
62
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
63
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
64
|
+
subsequently incorporated within the Work.
|
|
65
|
+
|
|
66
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
67
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
68
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
69
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
70
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
71
|
+
Work and such Derivative Works in Source or Object form.
|
|
72
|
+
|
|
73
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
74
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
75
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
76
|
+
(except as stated in this section) patent license to make, have made,
|
|
77
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
78
|
+
where such license applies only to those patent claims licensable
|
|
79
|
+
by such Contributor that are necessarily infringed by their
|
|
80
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
81
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
82
|
+
institute patent litigation against any entity (including a
|
|
83
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
84
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
85
|
+
or contributory patent infringement, then any patent licenses
|
|
86
|
+
granted to You under this License for that Work shall terminate
|
|
87
|
+
as of the date such litigation is filed.
|
|
88
|
+
|
|
89
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
90
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
91
|
+
modifications, and in Source or Object form, provided that You
|
|
92
|
+
meet the following conditions:
|
|
93
|
+
|
|
94
|
+
(a) You must give any other recipients of the Work or
|
|
95
|
+
Derivative Works a copy of this License; and
|
|
96
|
+
|
|
97
|
+
(b) You must cause any modified files to carry prominent notices
|
|
98
|
+
stating that You changed the files; and
|
|
99
|
+
|
|
100
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
101
|
+
that You distribute, all copyright, patent, trademark, and
|
|
102
|
+
attribution notices from the Source form of the Work,
|
|
103
|
+
excluding those notices that do not pertain to any part of
|
|
104
|
+
the Derivative Works; and
|
|
105
|
+
|
|
106
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
107
|
+
distribution, then any Derivative Works that You distribute must
|
|
108
|
+
include a readable copy of the attribution notices contained
|
|
109
|
+
within such NOTICE file, excluding those notices that do not
|
|
110
|
+
pertain to any part of the Derivative Works, in at least one
|
|
111
|
+
of the following places: within a NOTICE text file distributed
|
|
112
|
+
as part of the Derivative Works; within the Source form or
|
|
113
|
+
documentation, if provided along with the Derivative Works; or,
|
|
114
|
+
within a display generated by the Derivative Works, if and
|
|
115
|
+
wherever such third-party notices normally appear. The contents
|
|
116
|
+
of the NOTICE file are for informational purposes only and
|
|
117
|
+
do not modify the License. You may add Your own attribution
|
|
118
|
+
notices within Derivative Works that You distribute, alongside
|
|
119
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
120
|
+
that such additional attribution notices cannot be construed
|
|
121
|
+
as modifying the License.
|
|
122
|
+
|
|
123
|
+
You may add Your own copyright statement to Your modifications and
|
|
124
|
+
may provide additional or different license terms and conditions
|
|
125
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
126
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
127
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
128
|
+
the conditions stated in this License.
|
|
129
|
+
|
|
130
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
131
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
132
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
133
|
+
this License, without any additional terms or conditions.
|
|
134
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
135
|
+
the terms of any separate license agreement you may have executed
|
|
136
|
+
with Licensor regarding such Contributions.
|
|
137
|
+
|
|
138
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
139
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
140
|
+
except as required for reasonable and customary use in describing the
|
|
141
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
142
|
+
|
|
143
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
144
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
145
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
146
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
147
|
+
implied, including, without limitation, any warranties or conditions
|
|
148
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
149
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
150
|
+
appropriateness of using or redistributing the Work and assume any
|
|
151
|
+
risks associated with Your exercise of permissions under this License.
|
|
152
|
+
|
|
153
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
154
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
155
|
+
unless required by applicable law (such as deliberate and grossly
|
|
156
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
157
|
+
liable to You for damages, including any direct, indirect, special,
|
|
158
|
+
incidental, or consequential damages of any character arising as a
|
|
159
|
+
result of this License or out of the use or inability to use the
|
|
160
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
161
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
162
|
+
other commercial damages or losses), even if such Contributor
|
|
163
|
+
has been advised of the possibility of such damages.
|
|
164
|
+
|
|
165
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
166
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
167
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
168
|
+
or other liability obligations and/or rights consistent with this
|
|
169
|
+
License. However, in accepting such obligations, You may act only
|
|
170
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
171
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
172
|
+
defend, and hold each Contributor harmless for any liability
|
|
173
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
174
|
+
of your accepting any such warranty or additional liability.
|
|
175
|
+
|
|
176
|
+
END OF TERMS AND CONDITIONS
|
|
177
|
+
|
|
178
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
179
|
+
|
|
180
|
+
To apply the Apache License to your work, attach the following
|
|
181
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
182
|
+
replaced with your own identifying information. (Don't include
|
|
183
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
184
|
+
comment syntax for the file format. We also recommend that a
|
|
185
|
+
file or class name and description of purpose be included on the
|
|
186
|
+
same "printed page" as the copyright notice for easier
|
|
187
|
+
identification within third-party archives.
|
|
188
|
+
|
|
189
|
+
Copyright [yyyy] [name of copyright owner]
|
|
190
|
+
|
|
191
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
192
|
+
you may not use this file except in compliance with the License.
|
|
193
|
+
You may obtain a copy of the License at
|
|
194
|
+
|
|
195
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
196
|
+
|
|
197
|
+
Unless required by applicable law or agreed to in writing, software
|
|
198
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
199
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
200
|
+
See the License for the specific language governing permissions and
|
|
201
|
+
limitations under the License.
|