jevassert 0.1.0__py3-none-any.whl

jevassert/runner.py

@@ -0,0 +1,159 @@
+"""Record predictions against a live endpoint, and replay them offline.
+
+``record`` calls Jev once per case and writes one JSONL line per case.
+``load_predictions`` reads that file back for ``check`` — CI runs on the
+recording, so tests are deterministic, free and rate-limit free. Re-record
+when the pack changes or when the model version is bumped.
+
+Operational flags:
+
+- ``resume_from`` — skip cases that already have a successful record (retry
+  only the errored ones); the merged output keeps pack order.
+- ``rpm`` — requests-per-minute pacing for the early-access rate limit.
+- ``shuffle_options`` — seed for a robustness pass that permutes Choice option
+  order (Score levels are ordinal and are never shuffled).
+"""
+
+from __future__ import annotations
+
+import json
+import random
+import threading
+import time
+from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from typing import Any
+
+from .client import JevClient, JevError
+from .packs import Case, Pack
+
+
+def record(
+    pack: Pack,
+    client: JevClient,
+    concurrency: int = 8,
+    limit: int | None = None,
+    progress: Callable[[int, int], None] | None = None,
+    model: str | None = None,
+    resume_from: dict[str, dict[str, Any]] | None = None,
+    rpm: int = 0,
+    shuffle_options: int | None = None,
+) -> list[dict[str, Any]]:
+    """Evaluate every case in ``pack``; returns records in case order.
+
+    Auth failures (401/403, missing key) abort immediately instead of burning
+    one failed request per case.
+    """
+    if not client.api_key:
+        raise JevError("TYPESAFE_API_KEY is not set (env var or api_key=)")
+
+    selected: tuple[Case, ...] = pack.cases[:limit] if limit else pack.cases
+    api_questions = pack.to_api_questions()
+    if shuffle_options is not None:
+        api_questions = shuffle_choice_options(api_questions, shuffle_options)
+    api_model = model or pack.record_model
+    limiter = _RateLimiter(rpm) if rpm and rpm > 0 else None
+
+    existing = resume_from or {}
+    done = {case_id for case_id, item in existing.items() if not item.get("error")}
+    pending = [case for case in selected if case.id not in done]
+    completed = 0
+
+    def evaluate(case: Case) -> dict[str, Any]:
+        nonlocal completed
+        if limiter:
+            limiter.wait()
+        try:
+            response, latency_ms = client.system_one(case.state, api_questions, api_model)
+        except JevError as exc:
+            if exc.is_auth_error:
+                raise
+            result = {
+                "case_id": case.id,
+                "model": None,
+                "latency_ms": None,
+                "usage": None,
+                "answers": None,
+                "error": str(exc),
+            }
+        else:
+            result = {
+                "case_id": case.id,
+                "model": response.get("model"),
+                "latency_ms": round(latency_ms, 1),
+                "usage": response.get("usage"),
+                "answers": response.get("answers"),
+                "error": None,
+            }
+        completed += 1
+        if progress:
+            progress(completed, len(pending))
+        return result
+
+    with ThreadPoolExecutor(max_workers=max(1, concurrency)) as pool:
+        fresh = list(pool.map(evaluate, pending))
+
+    by_id = {item["case_id"]: item for item in existing.values()}
+    by_id.update({item["case_id"]: item for item in fresh})
+    return [by_id[case.id] for case in selected if case.id in by_id]
+
+
+def shuffle_choice_options(questions: dict[str, Any], seed: int) -> dict[str, Any]:
+    """Permute Choice option order deterministically (Score levels stay ordered)."""
+    rng = random.Random(seed)
+    shuffled: dict[str, Any] = {}
+    for qid, question in questions.items():
+        if question.get("type") == "choice" and isinstance(question.get("criteria"), dict):
+            items = list(question["criteria"].items())
+            rng.shuffle(items)
+            question = {**question, "criteria": dict(items)}
+        shuffled[qid] = question
+    return shuffled
+
+
+class _RateLimiter:
+    """Simple global pacer: at most ``rpm`` request starts per minute."""
+
+    def __init__(self, rpm: int) -> None:
+        self.min_interval = 60.0 / rpm
+        self._lock = threading.Lock()
+        self._next_start = 0.0
+
+    def wait(self) -> None:
+        with self._lock:
+            now = time.monotonic()
+            delay = max(0.0, self._next_start - now)
+            self._next_start = max(now, self._next_start) + self.min_interval
+        if delay:
+            time.sleep(delay)
+
+
+def write_predictions(path: str | Path, records: list[dict[str, Any]]) -> None:
+    path = Path(path)
+    with path.open("w", encoding="utf-8") as handle:
+        for record in records:
+            handle.write(json.dumps(record, ensure_ascii=False) + "\n")
+
+
+def load_predictions(path: str | Path) -> dict[str, dict[str, Any]]:
+    """Read a predictions JSONL file, keyed by case id."""
+    path = Path(path)
+    if not path.is_file():
+        raise FileNotFoundError(f"predictions file not found: {path}")
+    predictions: dict[str, dict[str, Any]] = {}
+    for lineno, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            record = json.loads(line)
+        except json.JSONDecodeError as exc:
+            raise ValueError(f"{path}:{lineno}: invalid JSON: {exc}") from exc
+        case_id = record.get("case_id")
+        if not isinstance(case_id, str) or not case_id:
+            raise ValueError(f"{path}:{lineno}: case_id is required")
+        if case_id in predictions:
+            raise ValueError(f"{path}:{lineno}: duplicate case_id '{case_id}'")
+        predictions[case_id] = record
+    return predictions

jevassert-0.1.0.dist-info/METADATA

@@ -0,0 +1,194 @@
+Metadata-Version: 2.5
+Name: jevassert
+Version: 0.1.0
+Summary: Record/replay regression tests for Jev (TypeSafe System One) question packs — accuracy, calibration and cost gates in CI.
+Project-URL: Homepage, https://github.com/dtduc-git/jevassert
+Project-URL: Issues, https://github.com/dtduc-git/jevassert/issues
+Project-URL: Changelog, https://github.com/dtduc-git/jevassert/releases
+Author-email: Duke - Duc Dinh <dtduc.contact@gmail.com>
+License-Expression: Apache-2.0
+License-File: LICENSE
+Keywords: calibration,ci,decisions,evals,jev,structured-output,system-one,testing,typesafe
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.10
+Requires-Dist: httpx>=0.27
+Requires-Dist: pyyaml>=6.0
+Description-Content-Type: text/markdown
+
+# jevassert
+
+[![CI](https://github.com/dtduc-git/jevassert/actions/workflows/ci.yml/badge.svg)](https://github.com/dtduc-git/jevassert/actions/workflows/ci.yml)
+[![License](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](LICENSE)
+
+Regression tests for [Jev](https://typesafe.ai) question packs: assert
+**accuracy, calibration and cost** in CI, with recordings so runs are
+deterministic, free and rate-limit free.
+
+> Independent community tool. Not affiliated with TypeSafe AI.
+
+Jev returns typed decisions with probabilities, which means two things your
+normal test suite cannot check: *does the decision match the labels*, and
+*does the stated probability mean what it says*. jevassert checks both — and
+never calls the model during `check`, because CI should run on a recording.
+
+```
+record  →  predictions.jsonl  →  check  (offline, deterministic)  →  exit 0/1/2
+```
+
+## Quickstart
+
+```sh
+uvx jevassert record examples/demo-triage -o triage.jsonl   # needs TYPESAFE_API_KEY
+uvx jevassert check  examples/demo-triage -p triage.jsonl --failures
+```
+
+`record` calls Jev once per case (concurrently, with retries) and writes one
+JSONL line per case — model version, usage, latency, answers. `check` replays
+that file: no API key, no network, same result every time.
+
+Re-record when the pack questions, the gates or the model version change, then
+compare:
+
+```sh
+uvx jevassert record examples/demo-triage -o triage-next.jsonl
+uvx jevassert compare examples/demo-triage --a triage.jsonl --b triage-next.jsonl
+```
+
+## Packs
+
+A pack follows **spec v0** (canonical:
+[jev-packs/SPEC.md](https://github.com/dtduc-git/jev-packs/blob/master/SPEC.md)):
+
+```
+examples/demo-triage/
+  pack.yaml     # state contract, questions, optional threshold floors
+  cases.jsonl   # {"id", "state", "expect"} per line
+  gates.yaml    # optional — jevassert quality gates
+```
+
+Questions are the SPEC shapes — `noul`, `choice` (`options`), `score`
+(`levels`) — and every closed set must include the label `unknown` (Jev cannot
+abstain). Score levels may carry `level_descriptions` so the API gets
+situational text instead of bare keys:
+
+```yaml
+questions:
+  queue:
+    type: choice
+    instructions: Which team should handle this message?
+    options: {billing: ..., technical: ..., unknown: Cannot tell.}
+  urgency:
+    type: score
+    instructions: How soon does this need a human response?
+    levels: [low, normal, high, unknown]
+    level_descriptions:
+      low: No stated impact or time pressure; can wait.
+      high: Stated impact or time pressure; needs a response today.
+```
+
+`thresholds` in the pack are the author's recommended operating point
+(per-label probability floors; labels without a floor always go to review).
+
+Gates live in `gates.yaml`, so the quality contract is explicit:
+
+```yaml
+min_accuracy: 0.85
+max_ece: 0.15
+max_cost_per_case_usd: 0.001
+max_p95_latency_ms: 800
+min_coverage_at_precision: {precision: 0.9, min_coverage: 0.6}
+min_accuracy_ci_lower: 0.8          # bootstrap CI lower bound (n small-safe gate)
+per_question:
+  queue: {min_accuracy: 0.9, max_ece: 0.1}
+```
+
+## What `check` reports
+
+- **accuracy** per question and overall, with a **bootstrap 95% CI**
+  (`--bootstrap N`, default 1000; `0` disables)
+- **ECE** — expected calibration error of the decision probability, equal-mass
+  bins (meaningful with a few hundred items; small packs get a warning)
+- **Brier** — for Noul questions
+- **coverage** — accept when `p >= threshold`: how much you automate, at what
+  precision
+- **author thresholds** — auto-accept coverage under the pack's own floors
+- **threshold suggestion** — `--target-precision 0.95` finds the highest-coverage
+  cut that still reaches that precision
+- **cost and latency** — dollars per case and p50/p95, from the recording
+- **`--failures`** — every mismatch with expected/got/probability and the state
+
+`check` warns when the recording's model version differs from `pack.tested`,
+when cases are missing or extra, and when `tested` is still null.
+
+Exit codes: `0` all gates pass, `1` a gate failed, `2` usage/IO error.
+`--junit FILE` writes JUnit XML; `--report FILE` writes a markdown report
+(the spec's `evidence.md`); `--json` prints machine-readable metrics.
+
+## GitHub Action
+
+```yaml
+- uses: dtduc-git/jevassert@v0
+  with:
+    pack: examples/demo-triage
+    predictions: triage.jsonl
+    report: jevassert-report.md
+```
+
+Record outside CI (or as a scheduled job), commit the predictions file, and
+the Action enforces the gates on every pull request.
+
+## Commands
+
+| command | what it does |
+|---|---|
+| `jevassert record PACK -o FILE [--model M] [--resume] [--rpm N] [--dry-run] [--shuffle-options SEED] [--repeat N]` | call Jev for every case, write predictions JSONL |
+| `jevassert check PACK -p FILE [--failures] [--bootstrap N] [--target-precision P] [--partition dev\|test]` | compute metrics, evaluate gates, exit 0/1/2 |
+| `jevassert compare PACK --a A --b B` | paired accuracy deltas + exact McNemar p-value |
+
+`record` extras: `--dry-run` estimates tokens/cost from the pack without sending
+anything; `--resume` retries only cases that errored; `--rpm` paces requests;
+`--shuffle-options` records a robustness pass with permuted Choice option order
+(Score levels stay ordinal) — compare it against the base recording; `--repeat N`
+records N independent rounds (`FILE-r2.jsonl`, ...) and prints the discordant
+decision count across rounds.
+
+`check --partition dev|test` evaluates only one deterministic hash-split half
+(`--partition-seed`, `--partition-ratio`) — tune thresholds on dev, then verify
+on test without fooling yourself.
+
+Environment: `TYPESAFE_API_KEY` (record only), `TYPESAFE_BASE_URL` (override
+the endpoint, e.g. a local Jev-compatible replica).
+
+## Reading the numbers
+
+- Accuracy on <30 items per question is noisy (±19pp at n=10). Prefer packs
+  with 100+ items per question before gating on tight thresholds, and let
+  `--bootstrap` show the CI — gate on `min_accuracy_ci_lower` when n is small.
+- ECE needs a few hundred items to be meaningful; with small n the bins
+  collapse to per-item gaps. Treat it as a smoke signal, not a measurement.
+- Stability: record twice and `compare` — the flip rate is your run-to-run
+  noise floor. Question wording, option order and model version all move it;
+  `--shuffle-options` isolates the option-order effect and `--repeat N` prints
+  the flip rate automatically.
+- Threshold tuning: suggest on one half, verify on the other —
+  `check --target-precision 0.95 --partition dev`, then `--partition test`.
+  (CIs are percentile bootstrap; with heavily tied probabilities the ECE CI can
+  be slightly skewed.)
+
+## Scope
+
+- Backend-neutral: anything that speaks the System One request/response shape.
+- File-based and offline-first: no server, no database, no telemetry.
+- Not a labeling tool, not a dashboard, not an observability product.
+
+`jevassert.packs` is the reference loader/validator for SPEC v0 and is meant
+to be imported by other tools (jev-table, packs CI).

jevassert-0.1.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+jevassert/__init__.py,sha256=g7nS1fVjWrA6PC90gOX8KET_1YnibZj0z-ReSEYTA0E,98
+jevassert/__main__.py,sha256=k1ocEWawweo1qCJWNFAAvyxz3tcY13dzvCenHszij30,48
+jevassert/cli.py,sha256=hhDED7PSweGzL2VJ4_LI6lyoAgviPJzzp1xYPKhyqE4,20215
+jevassert/client.py,sha256=2HR81GckC5coK1dQJ3IbYmnCft2vwkj6IUD3aqoLfgM,5140
+jevassert/compare.py,sha256=zTzSkvGu5TLH21pyN53Vj1AT_VdKuVBj8A0XOj6X4K4,3594
+jevassert/metrics.py,sha256=zpEp0jfEM08rCh4lV40xbYIui7UGj3cudY4iEFiXUU4,17861
+jevassert/packs.py,sha256=gSaTJU84CP64Tcfopu3ZZMaeoylg2QFaRBXtk1XHTHo,18237
+jevassert/report.py,sha256=gGUgiypi1oPm9Swg3y9CEuRdI6HEmBrdEG1gwL3Bin4,5149
+jevassert/runner.py,sha256=Th3cvmBaEPb0QyuqvarmmA8qAWcHyroUzPzZ0pjVe9o,5820
+jevassert-0.1.0.dist-info/METADATA,sha256=H3Tq0XToe_NkWFeGuoFO-mWsODYvtoNKt4aRd16BkbU,8233
+jevassert-0.1.0.dist-info/WHEEL,sha256=THafob7ofN-NsuMN7Mg4qZyHaQI7KkD-QlcQatYhXPo,87
+jevassert-0.1.0.dist-info/entry_points.txt,sha256=Zo0NHzrA4nrs74mvWCCf3oxynQmg1frHv-fiJwGIwZg,49
+jevassert-0.1.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+jevassert-0.1.0.dist-info/RECORD,,

jevassert-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.3
+Root-Is-Purelib: true
+Tag: py3-none-any

jevassert-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+jevassert = jevassert.cli:main

jevassert-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.