hypothesis-helm 0.1.0__py3-none-any.whl

hypothesis_helm/__init__.py

@@ -0,0 +1,38 @@
+"""
+Property-based tests for Helm charts.
+"""
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from hypothesis_helm.charts.generate import coalesce, generate_tests
+    from hypothesis_helm.charts.model import Chart
+    from hypothesis_helm.charts.runner import check_chart
+
+__all__ = ["Chart", "check_chart", "coalesce", "generate_tests"]
+
+
+def __getattr__(name: str) -> object:
+    """
+    Load the public API without constructing strategies during pytest plugin startup.
+
+    Args:
+        name (str): Public package attribute requested by the caller.
+
+    Returns:
+        object: The exported chart type or framework function.
+    """
+    if name not in __all__:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+    from hypothesis_helm.charts.generate import coalesce, generate_tests
+    from hypothesis_helm.charts.model import Chart
+    from hypothesis_helm.charts.runner import check_chart
+
+    exports: dict[str, object] = {
+        "Chart": Chart,
+        "check_chart": check_chart,
+        "coalesce": coalesce,
+        "generate_tests": generate_tests,
+    }
+    globals().update(exports)
+    return exports[name]

hypothesis_helm/analysis/__init__.py

@@ -0,0 +1,3 @@
+"""
+Measure configuration mutations without changing compiler pruning guarantees.
+"""

hypothesis_helm/analysis/cli.py

@@ -0,0 +1,117 @@
+"""
+Measure the downstream effects of explicit values mutations without changing a chart.
+"""
+
+import argparse
+import json
+import time
+from pathlib import Path
+
+from hypothesis_helm.analysis.report import write_report
+from hypothesis_helm.analysis.sensitivity import Mutation, analyze
+from hypothesis_helm.charts.model import Chart
+from hypothesis_helm.charts.rendering import render
+from hypothesis_helm.reporting.budget import parse_time_limit
+from hypothesis_helm.schemas.contracts import mapping, sequence
+
+
+def load_mutations(path: Path) -> list[Mutation]:
+    """
+    Read explicit names, typed paths and replacement values from JSON.
+
+    Args:
+        path (Path): JSON array of mutation descriptors.
+
+    Returns:
+        list[Mutation]: Validated operation descriptors in input order.
+    """
+    mutations = []
+    for entry in sequence(json.loads(path.read_text())):
+        record = mapping(entry)
+        name, parts = record["name"], sequence(record["path"])
+        if not isinstance(name, str) or not name or not parts:
+            raise ValueError("each mutation requires a nonempty string name and path")
+        path_parts: list[str | int] = []
+        for part in parts:
+            if isinstance(part, str) or type(part) is int:
+                path_parts.append(part)
+            else:
+                raise ValueError("path components must be string keys or integer indices")
+        mutations.append(Mutation(name, tuple(path_parts), record["value"]))
+    return mutations
+
+
+def main(argv: list[str] | None = None) -> int:
+    """
+    Run a bounded local-chart sensitivity analysis and save its evidence.
+
+    Args:
+        argv (list[str] | None): Explicit arguments or process command line.
+
+    Returns:
+        int: Zero for a complete analysis, one for a budget-limited result.
+    """
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("chart", type=Path, help="local chart containing values.yaml and values.schema.json")
+    parser.add_argument("--mutations", required=True, type=Path, help="JSON array of name/path/value replacements")
+    parser.add_argument("--output", type=Path, default=Path(f"studies/sensitivity/runs/{time.time_ns()}"))
+    parser.add_argument("--max-pairs", type=int, default=100)
+    parser.add_argument("--max-mutations", type=int, default=100)
+    parser.add_argument("--time-limit", type=parse_time_limit, default=180)
+    parser.add_argument("--helm", default="helm")
+    parser.add_argument("--release", default="hypothesis")
+    parser.add_argument("--namespace", default="default")
+    parser.add_argument("--kube-version")
+    parser.add_argument("--plot", action="store_true", help="requires the benchmarking extra (matplotlib)")
+    args = parser.parse_args(argv)
+    if args.output.exists():
+        parser.error("choose a fresh output directory")
+    if args.max_pairs < 0 or args.max_mutations < 1:
+        parser.error("max-pairs must be nonnegative and max-mutations must be positive")
+    chart = Chart.load(args.chart)
+    mutations = load_mutations(args.mutations)
+    deadline = time.monotonic() + args.time_limit
+
+    def invoke(values: dict[str, object]) -> object:
+        """
+        Render with a fixed invocation context and the remaining execution budget.
+
+        Args:
+            values (dict[str, object]): Schema-valid values configuration.
+
+        Returns:
+            object: Parsed, validated manifest bundle.
+        """
+        remaining = deadline - time.monotonic()
+        if remaining <= 0:
+            raise TimeoutError("analysis render deadline reached")
+        return render(
+            chart,
+            values,
+            helm=args.helm,
+            release=args.release,
+            namespace=args.namespace,
+            kube_version=args.kube_version,
+            timeout=min(30, remaining),
+            stream=False,
+        )
+
+    document = analyze(
+        chart.defaults,
+        chart.schema,
+        mutations,
+        invoke,
+        max_pairs=args.max_pairs,
+        max_mutations=args.max_mutations,
+        time_limit=args.time_limit,
+    )
+    document["context"] = {
+        "chart": str(chart.path),
+        "helm": args.helm,
+        "release": args.release,
+        "namespace": args.namespace,
+        "kube_version": args.kube_version,
+    }
+    write_report(args.output, document, plots=args.plot)
+    print(f"Sensitivity analysis: {document['status']}; {document['renders']} renders; report: {args.output / 'README.md'}")
+    return 0 if document["status"] == "complete" else 1

hypothesis_helm/analysis/report.py

@@ -0,0 +1,172 @@
+"""
+Publish concise sensitivity evidence and optional diagnostic plots.
+"""
+
+import json
+from pathlib import Path
+
+from hypothesis_helm.reporting.contents import with_contents
+from hypothesis_helm.schemas.contracts import mapping, sequence
+
+
+def write_report(output: Path, document: dict[str, object], *, plots: bool = False) -> None:
+    """
+    Save measurements, a human-readable summary and optional matplotlib plots.
+
+    Args:
+        output (Path): Fresh report directory.
+        document (dict[str, object]): Completed or partial sensitivity evidence.
+        plots (bool): Generate a three-panel PNG and SVG.
+
+    Returns:
+        None: Report artifacts are written.
+    """
+    output.mkdir(parents=True, exist_ok=False)
+    (output / "results.json").write_text(json.dumps(document, indent=2, allow_nan=False) + "\n")
+    rows = [mapping(row) for row in sequence(document["mutations"])]
+    pairs = [mapping(row) for row in sequence(document["interactions"])]
+    identifiers = {str(row["name"]): index + 1 for index, row in enumerate(rows)}
+    lines = [
+        "# Mutation sensitivity",
+        "",
+        f"Status: **{document['status']}**. Helm render attempts: **{document['renders']}**.",
+        "",
+        "Distance counts added and removed JSON path/value indicators. A changed value counts twice.",
+        "Document and array order matter. These measurements do not prove equivalence or authorize pruning.",
+        "Distances assume deterministic rendering with fixed chart dependencies, release, namespace and Kubernetes version.",
+        "",
+    ]
+    if plots:
+        plot(output, document)
+        lines.extend(["![Sensitivity, interaction and sequence measurements](sensitivity.png)", ""])
+    lines += [
+        f"Measured {len(rows)} single mutations and {len(pairs)} pairs. The plots include every comparable measurement.",
+        "Tables show up to 20 of the largest effects. Mutation IDs follow the order in mutations.json.",
+        "",
+        "| ID | Mutation | Values path | Replacement | Output distance | Status |",
+        "| ---: | --- | --- | --- | ---: | --- |",
+    ]
+
+    def literal(value: object) -> str:
+        """
+        Escape untrusted labels and values for Markdown table cells.
+
+        Args:
+            value (object): JSON value to display.
+
+        Returns:
+            str: HTML-escaped JSON with table separators encoded.
+        """
+        import html
+
+        return html.escape(json.dumps(value, ensure_ascii=True)).replace("|", "|")
+
+    for row in sorted(rows, key=lambda row: -int(str(row.get("distance", -1))))[:20]:
+        lines.append(
+            f"| {identifiers[str(row['name'])]} | {literal(row['name'])} | {literal(row['path'])} | {literal(row['value'])} | "
+            f"{row.get('distance', 'N/A')} | {row['status']} |"
+        )
+    lines += [
+        "",
+        "## Parameter interactions",
+        "",
+        "A nonzero mixed difference means the selected output features respond non-additively.",
+        "Pairs with order-dependent inputs are excluded from this measure. Render failures have no assigned distance.",
+        "",
+        "| Mutations | Mixed difference | Status |",
+        "| --- | ---: | --- |",
+    ]
+    for row in sorted(pairs, key=lambda row: -int(str(row.get("mixed_difference_l1", -1))))[:20]:
+        lines.append(f"| {literal(row['mutations'])} | {row.get('mixed_difference_l1', 'N/A')} | {row['status']} |")
+    lines += [
+        "",
+        "The ordered sequence in results.json records both cumulative path length and displacement from the baseline.",
+        "They differ when later mutations reverse earlier changes. The sequence stops at its first invalid or failed step.",
+        "",
+        "[Full measurements and render errors](results.json)",
+    ]
+    (output / "README.md").write_text(with_contents("\n".join(lines) + "\n"))
+
+
+def plot(output: Path, document: dict[str, object]) -> None:
+    """
+    Plot mutation distances, pair interactions and ordered path accumulation.
+
+    Args:
+        output (Path): Destination for plot files.
+        document (dict[str, object]): Recorded sensitivity observations.
+
+    Returns:
+        None: PNG and SVG plots are saved.
+    """
+    import numpy as np
+    from matplotlib.backends.backend_agg import FigureCanvasAgg
+    from matplotlib.figure import Figure
+
+    figure = Figure(figsize=(20, 7), layout="constrained")
+    FigureCanvasAgg(figure)
+    axes = figure.subplots(1, 3)
+    mutations = [mapping(row) for row in sequence(document["mutations"])]
+    names = [str(row["name"]) for row in mutations]
+    singles = [(index + 1, int(str(row["distance"]))) for index, row in enumerate(mutations) if "distance" in row]
+    if singles:
+        axes[0].scatter(*zip(*singles, strict=True), s=22, alpha=0.8)
+    else:
+        axes[0].text(0.5, 0.5, "No comparable single mutations", transform=axes[0].transAxes, ha="center")
+    axes[0].set(
+        title=f"Which single changes affect output most?\n{len(singles)} measured mutations; IDs follow the input file",
+        xlabel="Mutation ID",
+        ylabel="Changed leaf indicators",
+    )
+    pairs = [mapping(row) for row in sequence(document["interactions"]) if "mixed_difference_l1" in mapping(row)]
+    positions = {name: index for index, name in enumerate(names)}
+    matrix = np.full((len(names), len(names)), np.nan)
+    for row in pairs:
+        left, right = (positions[str(name)] for name in sequence(row["mutations"]))
+        matrix[left, right] = matrix[right, left] = int(str(row["mixed_difference_l1"]))
+    if pairs:
+        from matplotlib import colormaps
+
+        colors = colormaps["viridis"].with_extremes(bad="#dddddd")
+        heatmap = axes[1].imshow(
+            np.ma.masked_invalid(matrix),
+            origin="lower",
+            interpolation="nearest",
+            cmap=colors,
+            vmin=0,
+            vmax=max(1, float(np.nanmax(matrix))),
+            extent=(0.5, len(names) + 0.5, 0.5, len(names) + 0.5),
+        )
+        figure.colorbar(heatmap, ax=axes[1], label="Interaction magnitude", shrink=0.75)
+    else:
+        axes[1].text(0.5, 0.5, "No comparable pairs", transform=axes[1].transAxes, ha="center")
+    axes[1].set(
+        title=f"Which pairs interact?\n{len(pairs)} measured pairs; gray = unmeasured or inapplicable",
+        xlabel="Mutation ID",
+        ylabel="Mutation ID",
+    )
+    steps = [mapping(row) for row in sequence(document["sequence"]) if "cumulative_path_length" in mapping(row)]
+    indices = list(range(len(steps) + 1))
+    if steps:
+        axes[2].plot(indices, [0, *(int(str(row["cumulative_path_length"])) for row in steps)], marker=".", label="Cumulative path length")
+        axes[2].plot(
+            indices, [0, *(int(str(row["endpoint_displacement"])) for row in steps)], marker=".", label="Displacement from baseline"
+        )
+        axes[2].legend()
+    else:
+        axes[2].text(0.5, 0.5, "No comparable sequence", transform=axes[2].transAxes, ha="center")
+    axes[2].set(
+        title=f"Do later changes undo earlier ones?\n{len(steps)} measured steps in explicit input order",
+        xlabel="Completed mutation steps",
+        ylabel="Leaf indicators",
+    )
+    for index, axis in enumerate(axes):
+        axis.title.set_fontsize(10)
+        axis.tick_params(labelsize=9)
+        if index != 1:
+            axis.grid(alpha=0.2)
+    heading = figure.suptitle("Which values changes have the largest effects, and which interact?")
+    heading.set_gid("plot-question")
+    for suffix in ("png", "svg"):
+        figure.savefig(output / f"sensitivity.{suffix}", dpi=240)
+    figure.clear()

hypothesis_helm/analysis/sensitivity.py

@@ -0,0 +1,268 @@
+"""
+Measure output sensitivity and interactions of explicit chart-value mutations.
+"""
+
+import copy
+import itertools
+import json
+import time
+from collections import Counter
+from collections.abc import Callable, Sequence
+from typing import cast
+
+from attrs import define
+from jsonschema import validators
+
+from hypothesis_helm.schemas.contracts import Json, configuration_key
+
+
+@define(frozen=True)
+class Mutation:
+    """
+    Replace one value at an explicit object or array path.
+
+    Attributes:
+        name (str): Unique mutation label.
+        path (tuple[str | int, ...]): Object keys and array indices.
+        value (object): JSON-compatible replacement value.
+    """
+
+    name: str
+    path: tuple[str | int, ...]
+    value: object
+
+    def apply(self, values: dict[str, object]) -> dict[str, object]:
+        """
+        Apply a replacement without modifying the original configuration.
+
+        Args:
+            values (dict[str, object]): Source configuration.
+
+        Returns:
+            dict[str, object]: Independently owned mutated configuration.
+        """
+        if not self.name or not self.path:
+            raise ValueError("mutations need a name and a nonempty path")
+        result = copy.deepcopy(values)
+        node: object = result
+        for part in self.path[:-1]:
+            if isinstance(node, dict) and isinstance(part, str):
+                node = node[part]
+            elif isinstance(node, list) and type(part) is int and 0 <= part < len(node):
+                node = node[part]
+            else:
+                raise ValueError("mutation path does not address an existing container")
+        part = self.path[-1]
+        if isinstance(node, dict) and isinstance(part, str):
+            node[part] = copy.deepcopy(self.value)
+        elif isinstance(node, list) and type(part) is int and 0 <= part < len(node):
+            node[part] = copy.deepcopy(self.value)
+        else:
+            raise ValueError("mutation path does not address a replaceable value")
+        configuration_key(result)
+        return result
+
+
+def features(value: object, path: tuple[str | int, ...] = ()) -> Counter[str]:
+    """
+    Encode manifest leaves as exact path-and-value indicators.
+
+    Args:
+        value (object): JSON manifest bundle or subtree.
+        path (tuple[str | int, ...]): Current structural position.
+
+    Returns:
+        Counter[str]: One-hot leaf features, including empty containers.
+    """
+    result: Counter[str] = Counter()
+    if isinstance(value, dict) and value:
+        for key, item in value.items():
+            result.update(features(item, (*path, str(key))))
+    elif isinstance(value, list) and value:
+        for index, item in enumerate(value):
+            result.update(features(item, (*path, index)))
+    else:
+        result[json.dumps([path, value], sort_keys=True, allow_nan=False)] += 1
+    return result
+
+
+def distance(left: object, right: object) -> int:
+    """
+    Compute L1 displacement between leaf-indicator representations.
+
+    Args:
+        left (object): First parsed manifest bundle.
+        right (object): Second parsed manifest bundle.
+
+    Returns:
+        int: Added plus removed path/value indicators; a replacement counts twice.
+    """
+    a, b = features(left), features(right)
+    return sum(abs(a[key] - b[key]) for key in a.keys() | b.keys())
+
+
+def analyze(
+    baseline: dict[str, object],
+    schema: dict[str, object],
+    mutations: Sequence[Mutation],
+    render: Callable[[dict[str, object]], object],
+    *,
+    max_pairs: int = 100,
+    max_mutations: int = 100,
+    time_limit: float = 180,
+) -> dict[str, object]:
+    """
+    Measure single mutations, pairwise interactions and a cumulative mutation sequence.
+
+    Args:
+        baseline (dict[str, object]): Starting values configuration.
+        schema (dict[str, object]): JSON Schema restricting all measured configurations.
+        mutations (Sequence[Mutation]): Explicit replacement operations in sequence order.
+        render (Callable[[dict[str, object]], object]): Renderer with fixed chart and invocation context.
+        max_pairs (int): Maximum unordered pairs of selected mutations to examine.
+        max_mutations (int): Maximum supplied mutations to examine.
+        time_limit (float): Admission deadline; callers must also bound individual render invocations.
+
+    Returns:
+        dict[str, object]: Sensitivity rankings, interactions, sequence distances and explicit failures.
+    """
+    if max_pairs < 0 or max_mutations < 1 or not 0 < time_limit < float("inf"):
+        raise ValueError("analysis limits must be nonnegative pairs, positive mutations and finite positive time")
+    if not mutations or len({mutation.name for mutation in mutations}) != len(mutations):
+        raise ValueError("provide at least one mutation with unique names")
+    validator = validators.validator_for(schema)(schema)
+    validator.validate(cast(Json, baseline))
+    deadline = time.monotonic() + time_limit
+    selected = list(mutations[:max_mutations])
+    observations: dict[str, dict[str, object]] = {}
+    singles: list[dict[str, object]] = []
+    interactions: list[dict[str, object]] = []
+    sequence_rows: list[dict[str, object]] = []
+    input_values: dict[str, dict[str, object]] = {}
+    renders = 0
+
+    def observe(values: dict[str, object]) -> dict[str, object]:
+        """
+        Render a schema-valid configuration once within this analysis.
+
+        Args:
+            values (dict[str, object]): Candidate chart input.
+
+        Returns:
+            dict[str, object]: Output evidence or a schema/render failure.
+        """
+        nonlocal renders
+        if time.monotonic() >= deadline:
+            raise TimeoutError("analysis admission deadline reached")
+        key = configuration_key(values)
+        if key in observations:
+            return observations[key]
+        from jsonschema.exceptions import ValidationError
+
+        try:
+            validator.validate(cast(Json, values))
+        except ValidationError as error:
+            return {"status": "schema-rejected", "error": error.message}
+        renders += 1
+        try:
+            output = render(copy.deepcopy(values))
+            json.dumps(output, sort_keys=True, allow_nan=False)
+            observation: dict[str, object] = {"status": "rendered", "output": output}
+        except Exception as error:
+            observation = {"status": "render-error", "error": str(error), "error_type": type(error).__name__}
+        observations[key] = observation
+        return observation
+
+    origin: dict[str, object] = {"status": "unobserved"}
+    status = "complete"
+    try:
+        origin = observe(baseline)
+        for mutation in selected:
+            row: dict[str, object] = {"name": mutation.name, "path": list(mutation.path), "value": mutation.value}
+            try:
+                values = mutation.apply(baseline)
+            except (ValueError, KeyError, IndexError, TypeError) as error:
+                singles.append({**row, "status": "invalid-mutation", "error": str(error)})
+                continue
+            observation = observe(values)
+            row.update(observation)
+            if observation["status"] == "rendered" and origin["status"] == "rendered":
+                row["distance"] = distance(origin["output"], observation["output"])
+            input_values[mutation.name] = values
+            singles.append(row)
+        for first, second in itertools.islice(itertools.combinations(selected, 2), max_pairs):
+            row = {"mutations": [first.name, second.name]}
+            if first.name not in input_values or second.name not in input_values:
+                interactions.append({**row, "status": "unavailable-single"})
+                continue
+            a, b = input_values[first.name], input_values[second.name]
+            try:
+                ab, ba = second.apply(a), first.apply(b)
+            except (ValueError, KeyError, IndexError, TypeError) as error:
+                interactions.append({**row, "status": "invalid-mutation", "error": str(error)})
+                continue
+            if configuration_key(ab) != configuration_key(ba):
+                interactions.append({**row, "status": "order-dependent"})
+                continue
+            joint = observe(ab)
+            row.update(joint)
+            observation_a, observation_b = observe(a), observe(b)
+            comparable = all(item["status"] == "rendered" for item in (origin, observation_a, observation_b, joint))
+            if comparable:
+                fa, fb, fab, f0 = (
+                    features(observation_a["output"]),
+                    features(observation_b["output"]),
+                    features(joint["output"]),
+                    features(origin["output"]),
+                )
+                keys = fa.keys() | fb.keys() | fab.keys() | f0.keys()
+                row["mixed_difference_l1"] = sum(abs(fab[key] - fa[key] - fb[key] + f0[key]) for key in keys)
+                row["joint_distance"] = distance(origin["output"], joint["output"])
+            else:
+                row["measurement_status"] = "unavailable-reference"
+            interactions.append(row)
+        current, previous = copy.deepcopy(baseline), origin
+        cumulative = 0
+        for mutation in selected:
+            try:
+                next_values = mutation.apply(current)
+            except (ValueError, KeyError, IndexError, TypeError) as error:
+                sequence_rows.append({"name": mutation.name, "status": "invalid-mutation", "error": str(error)})
+                break
+            next_observation = observe(next_values)
+            row = {"name": mutation.name, "status": next_observation["status"]}
+            if next_observation["status"] != "rendered" or previous["status"] != "rendered":
+                if previous["status"] != "rendered":
+                    row["status"] = "preceding-output-unavailable"
+                row["error"] = next_observation.get("error", "preceding output unavailable")
+                sequence_rows.append(row)
+                break
+            step_distance = distance(previous["output"], next_observation["output"])
+            cumulative += step_distance
+            row.update(
+                step_distance=step_distance,
+                cumulative_path_length=cumulative,
+                endpoint_displacement=distance(origin["output"], next_observation["output"]),
+            )
+            sequence_rows.append(row)
+            current, previous = next_values, next_observation
+    except TimeoutError:
+        status = "time-limit"
+    if time.monotonic() >= deadline:
+        status = "time-limit"
+    if status == "complete" and (len(selected) < len(mutations) or len(selected) * (len(selected) - 1) // 2 > max_pairs):
+        status = "selection-limit"
+    return {
+        "version": 1,
+        "status": status,
+        "metric": "L1 of JSON path/value leaf indicators; arrays and documents remain ordered",
+        "baseline": origin,
+        "mutations": singles,
+        "interactions": interactions,
+        "sequence": sequence_rows,
+        "selected_mutations": len(selected),
+        "provided_mutations": len(mutations),
+        "renders": renders,
+        "pruning_authorized": False,
+        "assumption": "The renderer is deterministic under a fixed chart, dependency and invocation context.",
+    }

hypothesis_helm/benchmarking/__init__.py

@@ -0,0 +1,3 @@
+"""
+Reproducible renderer, pruning and worker-scaling benchmarks.
+"""

hypothesis_helm/benchmarking/analysis/__init__.py

@@ -0,0 +1,3 @@
+"""
+Analyze output populations, calibration and filtering selections.
+"""