trialmark-forge 0.1.0__py3-none-any.whl

trialmark_forge/__init__.py

@@ -0,0 +1,242 @@
+"""A Trialmark runner that tests Forge's own AI steps through Forge's evaluation API.
+
+Each case is sent to ``POST /api/eval/run`` on a Forge deployment, which runs it
+through the same code a Forge user reaches (the Project's masking, grounding and
+instructions, or the app's fields, rules and guidance) on the model the
+configuration names, as a dry run that changes nothing. Forge returns the output
+with its cost, tokens and time, the Forge version and a fingerprint of the
+settings it ran under, and this runner hands them to Trialmark.
+
+Configuration::
+
+    "runners": { "forge": { "base_url": "https://www.forgeprivate.com" } },
+    "models": [
+      { "id": "claude-sonnet-5-5", "runner": "forge",
+        "settings": { "target": { "kind": "answer", "project_id": "…" } } }
+    ]
+
+Runner options:
+  base_url:   the Forge deployment (default https://www.forgeprivate.com).
+  token_env:  an environment variable holding the evaluation token
+              (default FORGE_EVAL_TOKEN); used when it is set.
+  token_file: a file holding the token, used otherwise (default ~/.forge-eval-token).
+  timeout:    seconds to wait for one case (default 300).
+
+Model settings:
+  target:     what to run: ``kind`` (answer, summarise, draft, extract, classify,
+              check) and ``project_id`` or ``tracker_id``. Required.
+  options:    passed to Forge with every case, such as ``{"today": "2026-11-06"}``
+              for extraction, so a relative date means the same on every run.
+  model:      the model Forge should run, when it differs from the entry's id.
+              ``"default"`` runs the step on the model Forge would choose.
+  expected_input_tokens, expected_output_tokens: for the cost estimate only.
+
+A case's input is a string or an object. A string becomes ``{"prompt": …}`` for
+answer, summarise and draft, and ``{"text": …}`` for extract; an object is sent as
+it is. For answer, summarise and draft the output Trialmark scores is the answer's
+text, so text scorers such as ``judge`` and ``checklist`` work unchanged; for the
+other kinds it is Forge's JSON output.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import ssl
+import threading
+import urllib.error
+import urllib.request
+from pathlib import Path
+from typing import Any
+
+from trialmark import RunOutput
+
+__version__ = "0.1.0"
+
+DEFAULT_BASE_URL = "https://www.forgeprivate.com"
+TEXT_KINDS = ("answer", "summarise", "draft")
+KINDS = (*TEXT_KINDS, "extract", "classify", "check")
+
+# Rough sizes for the estimate, from Forge's own runs: an answer reads about ten
+# thousand tokens of retrieved passages; the other steps read mostly their input.
+_EXPECTED_INPUT = {"answer": 11000, "summarise": 20000, "draft": 20000, "extract": 2000, "classify": 1500, "check": 0}
+_EXPECTED_OUTPUT = {"answer": 400, "summarise": 800, "draft": 1200, "extract": 600, "classify": 300, "check": 0}
+
+
+def _tls_context() -> ssl.SSLContext:
+    """Certificates from certifi, so HTTPS works on Pythons that ship without a
+    certificate list (python.org's macOS builds), and the system's otherwise."""
+    try:
+        import certifi
+
+        return ssl.create_default_context(cafile=certifi.where())
+    except ImportError:
+        return ssl.create_default_context()
+
+
+class ForgeError(Exception):
+    """A refusal from Forge that should stop a run before it spends anything."""
+
+
+class ForgeRunner:
+    """Runs each case through one of Forge's capabilities on the configured model."""
+
+    costs_money = True
+
+    def __init__(
+        self,
+        base_url: str = DEFAULT_BASE_URL,
+        token_env: str = "FORGE_EVAL_TOKEN",
+        token_file: str | None = None,
+        timeout: float = 300,
+        base_dir: Path | None = None,
+        opener: Any = None,
+    ) -> None:
+        self.base_url = base_url.rstrip("/")
+        self.token_env = token_env
+        self.token_file = Path(token_file).expanduser() if token_file else Path.home() / ".forge-eval-token"
+        self.timeout = float(timeout)
+        # For tests: anything with urllib's open(request, timeout=) interface.
+        self._opener = opener or urllib.request.build_opener(urllib.request.HTTPSHandler(context=_tls_context()))
+        self._targets: dict[str, Any] | None = None
+        self._lock = threading.Lock()
+
+    # --- talking to Forge -------------------------------------------------------
+
+    def _token(self) -> str:
+        token = os.environ.get(self.token_env, "").strip()
+        if token:
+            return token
+        if self.token_file.is_file():
+            token = self.token_file.read_text(encoding="utf-8").strip()
+            if token:
+                return token
+        raise ForgeError(
+            f"No evaluation token: set {self.token_env}, or save the token in {self.token_file}."
+        )
+
+    def _call(self, method: str, path: str, body: Any = None) -> tuple[int, dict[str, Any]]:
+        data = json.dumps(body).encode("utf-8") if body is not None else None
+        req = urllib.request.Request(
+            self.base_url + path,
+            data=data,
+            method=method,
+            headers={
+                "Authorization": f"Bearer {self._token()}",
+                "Content-Type": "application/json",
+                "User-Agent": f"trialmark-forge/{__version__}",
+            },
+        )
+        try:
+            with self._opener.open(req, timeout=self.timeout) as res:
+                return res.status, json.loads(res.read().decode("utf-8") or "{}")
+        except urllib.error.HTTPError as e:
+            try:
+                payload = json.loads(e.read().decode("utf-8") or "{}")
+            except ValueError:
+                payload = {}
+            return e.code, payload
+
+    def targets(self) -> dict[str, Any]:
+        """Forge's list of what this token may evaluate, fetched once per run."""
+        with self._lock:
+            if self._targets is None:
+                status, body = self._call("GET", "/api/eval/targets")
+                if status == 401:
+                    raise ForgeError("Forge refused the evaluation token. Check it, and save it again if it has changed.")
+                if status == 404:
+                    raise ForgeError(f"The evaluation API is not switched on at {self.base_url}.")
+                if status != 200:
+                    raise ForgeError(f"Forge answered {status} when listing targets: {body.get('error', body)}")
+                self._targets = body
+            return self._targets
+
+    # --- the runner interface ---------------------------------------------------
+
+    def prepare(self) -> None:
+        # Reaches Forge and checks the token once, before any case is run.
+        self.targets()
+
+    def _target(self, model: Any) -> dict[str, Any]:
+        t = model.settings.get("target")
+        if not isinstance(t, dict) or t.get("kind") not in KINDS:
+            raise ForgeError(
+                f"Model '{model.id}' needs settings.target with a kind from {', '.join(KINDS)} "
+                "and a project_id or tracker_id."
+            )
+        target_id = t.get("project_id") if t["kind"] in TEXT_KINDS else t.get("tracker_id")
+        if not target_id:
+            raise ForgeError(f"Model '{model.id}': a {t['kind']} target needs {'project_id' if t['kind'] in TEXT_KINDS else 'tracker_id'}.")
+        listed = next((x for x in self.targets().get("targets", []) if x.get("id") == target_id), None)
+        if listed is None:
+            raise ForgeError(f"Model '{model.id}': {target_id} is not among the targets this token may evaluate.")
+        if t["kind"] not in [c.get("kind") for c in listed.get("capabilities", [])]:
+            offered = ", ".join(c.get("kind", "?") for c in listed.get("capabilities", []))
+            raise ForgeError(f"Model '{model.id}': {listed.get('name')} does not support {t['kind']} (it offers {offered}).")
+        return t
+
+    def _forge_model(self, model: Any) -> str | None:
+        chosen = model.settings.get("model", model.id)
+        return None if chosen == "default" else chosen
+
+    def _input(self, kind: str, value: Any) -> dict[str, Any]:
+        if isinstance(value, dict):
+            return value
+        if isinstance(value, str):
+            return {"prompt": value} if kind in TEXT_KINDS else {"text": value}
+        raise ForgeError(f"A {kind} case's input must be a string or an object.")
+
+    def estimate_cost(self, model: Any, case: Any) -> float | None:
+        kind = self._target(model)["kind"]
+        if kind == "check":
+            return 0.0
+        chosen = self._forge_model(model)
+        price = next((m for m in self.targets().get("models", []) if m.get("id") == chosen), None)
+        if price is None:
+            if chosen is None:
+                return None  # the model Forge would choose; its price is not listed per target
+            raise ForgeError(f"Model '{model.id}': Forge cannot price {chosen}, so it will not run it.")
+        s = model.settings
+        size = len(json.dumps(case.input)) // 4
+        tokens_in = float(s.get("expected_input_tokens", _EXPECTED_INPUT[kind])) + size
+        tokens_out = float(s.get("expected_output_tokens", _EXPECTED_OUTPUT[kind]))
+        return (tokens_in * price["input_usd_per_mtok"] + tokens_out * price["output_usd_per_mtok"]) / 1_000_000
+
+    def run(self, model: Any, case: Any) -> RunOutput:
+        target = self._target(model)
+        kind = target["kind"]
+        body: dict[str, Any] = {"target": target, "input": self._input(kind, case.input)}
+        if model.settings.get("options"):
+            body["options"] = model.settings["options"]
+        chosen = self._forge_model(model)
+        if chosen and kind != "check":
+            body["model"] = chosen
+        status, r = self._call("POST", "/api/eval/run", body)
+        if status != 200:
+            return RunOutput(output=None, error=f"Forge {status}: {r.get('error', 'no message')}"[:500])
+
+        out = r.get("output")
+        if kind in TEXT_KINDS and isinstance(out, dict):
+            out = out.get("text", "")
+        error = None
+        if r.get("finish") == "length":
+            error = "The answer was cut off at Forge's output limit."
+        # The version and the fingerprint go on every result, so the record shows
+        # which Forge and which configuration each answer came from.
+        note = " ".join(
+            [f"forge {str(r.get('forge_version', '?'))[:12]}", f"settings {r.get('settings_fingerprint', '?')}"]
+            + [str(n) for n in r.get("notes", []) if n]
+        )
+        return RunOutput(
+            output=out,
+            cost_usd=float(r.get("cost_usd") or 0.0),
+            # Forge's own time for the step, without the network between here and Forge.
+            latency_ms=int(r["latency_ms"]) if isinstance(r.get("latency_ms"), (int, float)) else None,
+            input_tokens=int(r.get("input_tokens") or 0),
+            output_tokens=int(r.get("output_tokens") or 0),
+            error=error,
+            note=note,
+        )
+
+
+__all__ = ["ForgeRunner", "ForgeError", "__version__"]

trialmark_forge-0.1.0.dist-info/METADATA

@@ -0,0 +1,106 @@
+Metadata-Version: 2.5
+Name: trialmark-forge
+Version: 0.1.0
+Summary: A Trialmark runner that acceptance-tests Forge's AI steps (answering, summarising, drafting, extracting, classifying and checking) through Forge's evaluation API, on the models you choose.
+Project-URL: Homepage, https://github.com/peshwar9/trialmark-forge
+Project-URL: Trialmark, https://github.com/peshwar9/trialmark
+Author: PE (peshwar9)
+License-Expression: Apache-2.0
+License-File: LICENSE
+Keywords: acceptance testing,evaluation,forge,llm,trialmark
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.10
+Requires-Dist: certifi
+Requires-Dist: trialmark>=0.5.1
+Provides-Extra: dev
+Requires-Dist: pytest>=8; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# trialmark-forge
+
+A [Trialmark](https://github.com/peshwar9/trialmark) runner that acceptance-tests the AI steps of
+[Forge](https://www.forgeprivate.com) on the models you choose.
+
+Trialmark decides which model is good enough for a task, with a pass mark fixed before the results
+and a record anyone can check. This package lets it test Forge's own work rather than a bare model:
+each case runs through Forge's evaluation API, so the answer is the one a Forge user would get,
+with the Project's instructions, masking and grounding, or the app's fields, rules and guidance,
+applied. Nothing in Forge changes: every call is a dry run.
+
+## What it can test
+
+| Kind | Runs against | A case's input | The output Trialmark scores |
+|---|---|---|---|
+| `answer`, `summarise`, `draft` | A Project | The question or instruction, as a string | The answer's text, with its citations |
+| `extract` | An app that reads documents | The document's text, as a string, or `{"files": [...]}` | The fields or rows, as JSON |
+| `classify` | An app with a chart of accounts | `{"rows": [{"description": "...", "direction": "Debit"}]}` | Each row's account, as JSON |
+| `check` | An app with checks, such as vendor onboarding | `{"kind": "vendor", "fields": {...}, "detected": [...]}` | Each check's result, as JSON; no model is called |
+
+Options for each kind, such as `{"today": "2026-11-06"}` for extraction, go in the model's
+settings (below).
+
+## Install
+
+```
+pip install trialmark-forge
+```
+
+Trialmark finds the runner once the package is installed; `trialmark plugins` lists it as `forge`.
+
+## The token
+
+The evaluation API takes a bearer token issued by the Forge deployment. The runner reads it from
+the `FORGE_EVAL_TOKEN` environment variable when that is set, and otherwise from
+`~/.forge-eval-token`. Keep it out of configuration files and repositories.
+
+## A configuration
+
+```json
+{
+  "test_set": "testset.json",
+  "pass_mark": { "metric": "correct", "tolerance_points": 3, "floors": { "grounded": 0.95 }, "choose": "cheapest" },
+  "repeats": 3,
+  "concurrency": 4,
+  "runners": { "forge": { "base_url": "https://www.forgeprivate.com" } },
+  "models": [
+    { "id": "claude-haiku-4-5-20251001", "runner": "forge",
+      "settings": { "target": { "kind": "answer", "project_id": "<the Project's id>" } } },
+    { "id": "claude-sonnet-5-5", "runner": "forge",
+      "settings": { "target": { "kind": "answer", "project_id": "<the Project's id>" } } }
+  ],
+  "scorers": [{ "name": "judge", "options": { "model": "claude-opus-5-5", "documents": ["docs/contract.md"] } }],
+  "output": "runs/answers.json"
+}
+```
+
+`trialmark run config.json` prints the plan and an estimate from Forge's own prices, and checks
+that the token may evaluate each target and that Forge can price each model, before anything is
+spent. Add `--yes` to run it.
+
+**Runner options:** `base_url` (default `https://www.forgeprivate.com`), `token_env` (default
+`FORGE_EVAL_TOKEN`), `token_file` (default `~/.forge-eval-token`), `timeout` in seconds (default
+300).
+
+**Model settings:** `target` (required: `kind`, and `project_id` or `tracker_id`); `options`,
+sent with every case, such as `{"today": "2026-11-06"}` so that an extraction reads relative dates
+the same way on every run; `model`, when Forge should run a different model from the entry's id,
+or `"default"` for the model Forge would choose; `expected_input_tokens` and
+`expected_output_tokens`, to sharpen the estimate.
+
+## What the record holds
+
+Each result carries Forge's cost, tokens and time for the step (measured inside Forge, without the
+network), and a note with the Forge version and the fingerprint of the settings the step ran under
+(an RFC 8785 hash). Two runs with the same fingerprint tested the same configuration; a changed
+fingerprint means someone changed the Project's instructions or the app's setup in between.
+
+A refusal from Forge, such as the daily spend cap, is recorded as that case's error, and the run
+carries on. An answer cut off at Forge's output limit is recorded as an error too.
+
+## Licence
+
+Apache-2.0.

trialmark_forge-0.1.0.dist-info/RECORD

@@ -0,0 +1,6 @@
+trialmark_forge/__init__.py,sha256=ATQFxb4ZNibgI3OLXTOUeE1hruojdxc-Fk_uIu19HUk,11008
+trialmark_forge-0.1.0.dist-info/METADATA,sha256=Z4FI0GPky0Ua2KyGaCLc5uu8MQ6TOYTGHWYjxbTDhj4,5020
+trialmark_forge-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+trialmark_forge-0.1.0.dist-info/entry_points.txt,sha256=nD-ZgiOhBniGjTkRQmOGdpJqg5568OoINmIlSAorDGQ,56
+trialmark_forge-0.1.0.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+trialmark_forge-0.1.0.dist-info/RECORD,,

trialmark_forge-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any

trialmark_forge-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[trialmark.runners]
+forge = trialmark_forge:ForgeRunner

trialmark_forge-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.