statistical-comparison-helpers 0.1.0__py3-none-any.whl

statistical_comparison_helpers/__init__.py

@@ -0,0 +1,27 @@
+"""Method-agnostic helpers for policy comparison."""
+
+from statistical_comparison_helpers.cld import compact_letter_display
+from statistical_comparison_helpers.posterior import (
+    Binary,
+    Continuous,
+    Discrete,
+    draw_samples_from_beta_posterior,
+    draw_samples_from_dirichlet_posterior,
+)
+
+__all__ = [
+    "compact_letter_display",
+    "draw_samples_from_beta_posterior",
+    "draw_samples_from_dirichlet_posterior",
+    "Binary",
+    "Discrete",
+    "Continuous",
+    "plot_model_comparison",
+]
+
+
+def __getattr__(name: str):
+    if name == "plot_model_comparison":
+        from statistical_comparison_helpers.plotting import plot_model_comparison
+        return plot_model_comparison
+    raise AttributeError(f"module 'statistical_comparison_helpers' has no attribute {name!r}")

statistical_comparison_helpers/cld.py

@@ -0,0 +1,153 @@
+"""Compact Letter Display (CLD) implementation.
+
+Based on "An Algorithm for a Letter-Based Representation of All-Pairwise
+Comparisons" by Piepho (2004).
+"""
+
+from typing import List, Tuple
+
+
+def compact_letter_display(
+    significant_pair_list: List[Tuple[str, str]],
+    sorted_model_list: List[str],
+) -> List[str]:
+    """Generates Compact Letter Display (CLD) given a list of significant
+    pairs and a list of models.
+
+    Args:
+        significant_pair_list: A list containing tuples of model names that
+            were deemed significantly different by each A/B test.
+        sorted_model_list: A list of model names sorted by performance in
+            descending order.
+
+    Returns:
+        A list of letter strings representing CLD for the corresponding
+        models, in the same order as sorted_model_list.
+
+    Raises:
+        ValueError: If significant_pair_list contains unknown models,
+            duplicate models in sorted_model_list, self-comparisons,
+            or duplicate pairs.
+    """
+    _validate_inputs(significant_pair_list, sorted_model_list)
+
+    num_models = len(sorted_model_list)
+    if num_models == 0:
+        return []
+
+    # Map model names to indices.
+    model_to_index = {model: idx for idx, model in enumerate(sorted_model_list)}
+
+    # Canonicalize pairs: always (lower_index, higher_index) and deduplicate.
+    significant_index_pairs = []
+    for m1, m2 in significant_pair_list:
+        i, j = model_to_index[m1], model_to_index[m2]
+        significant_index_pairs.append((min(i, j), max(i, j)))
+
+    # --- Inner helper to remove redundant columns ---
+    def remove_redundant_columns(matrix):
+        changed = True
+        while changed:
+            changed = False
+            for i in range(len(matrix)):
+                for j in range(len(matrix)):
+                    if i != j:
+                        indices_i = {
+                            idx for idx, char in enumerate(matrix[i]) if char
+                        }
+                        indices_j = {
+                            idx for idx, char in enumerate(matrix[j]) if char
+                        }
+                        if indices_i.issubset(indices_j):
+                            matrix.pop(i)
+                            changed = True
+                            break
+                if changed:
+                    break
+        return matrix
+
+    # --- Main algorithm ---
+    # Start with a single column of True for all models.
+    letter_matrix: List[List[bool]] = [[True] * num_models]
+
+    # For each significant pair, update the letter matrix.
+    for model_idx1, model_idx2 in significant_index_pairs:
+        while any(col[model_idx1] and col[model_idx2] for col in letter_matrix):
+            for col_index, letter_column in enumerate(letter_matrix):
+                if letter_column[model_idx1] and letter_column[model_idx2]:
+                    new_column = letter_column.copy()
+                    new_column[model_idx1] = False
+                    letter_column[model_idx2] = False
+                    letter_matrix[col_index] = letter_column
+                    letter_matrix.append(new_column)
+                    letter_matrix = remove_redundant_columns(letter_matrix)
+                    break
+
+    # --- Deterministic letter assignment ---
+    # Sort columns with a principled tie-break:
+    #   1. Primary: first non-empty position (ascending) — top-ranked groups first.
+    #   2. Secondary: number of covered models (descending) — larger groups get
+    #      earlier letters.
+    #   3. Tertiary: tuple of covered positions (ascending) — deterministic
+    #      tie-break for equal-size groups.
+    def column_sort_key(column):
+        positions = tuple(i for i, v in enumerate(column) if v)
+        first_pos = positions[0] if positions else num_models
+        count = len(positions)
+        return (first_pos, -count, positions)
+
+    letter_matrix.sort(key=column_sort_key)
+
+    # Assign letters and build final output.
+    final_display = []
+    for model_idx in range(num_models):
+        letters = "".join(
+            chr(ord("a") + col_idx)
+            for col_idx in range(len(letter_matrix))
+            if letter_matrix[col_idx][model_idx]
+        )
+        final_display.append("".join(sorted(letters)))
+
+    return final_display
+
+
+def _validate_inputs(
+    significant_pair_list: List[Tuple[str, str]],
+    sorted_model_list: List[str],
+) -> None:
+    """Validate CLD inputs."""
+    # Check for duplicate models.
+    if len(sorted_model_list) != len(set(sorted_model_list)):
+        seen = set()
+        for m in sorted_model_list:
+            if m in seen:
+                raise ValueError(
+                    f"Duplicate model in sorted_model_list: '{m}'"
+                )
+            seen.add(m)
+
+    model_set = set(sorted_model_list)
+
+    seen_pairs = set()
+    for m1, m2 in significant_pair_list:
+        # Check for unknown models.
+        if m1 not in model_set:
+            raise ValueError(
+                f"Unknown model in significant_pair_list: '{m1}'"
+            )
+        if m2 not in model_set:
+            raise ValueError(
+                f"Unknown model in significant_pair_list: '{m2}'"
+            )
+        # Check for self-comparisons.
+        if m1 == m2:
+            raise ValueError(
+                f"Self-comparison in significant_pair_list: ('{m1}', '{m2}')"
+            )
+        # Check for duplicate pairs (order-independent).
+        canonical = (min(m1, m2), max(m1, m2))
+        if canonical in seen_pairs:
+            raise ValueError(
+                f"Duplicate pair in significant_pair_list: ('{m1}', '{m2}')"
+            )
+        seen_pairs.add(canonical)

statistical_comparison_helpers/plotting.py

@@ -0,0 +1,175 @@
+"""Violin plot visualization for policy comparison."""
+
+from typing import List, Optional, Union
+
+import matplotlib.pyplot as plt
+import numpy as np
+
+from statistical_comparison_helpers.posterior import (
+    Binary,
+    Continuous,
+    Discrete,
+    draw_posterior_samples,
+)
+
+
+def plot_model_comparison(
+    model_name_list: List[str],
+    result_arrays: List[np.ndarray],
+    cld_letters: List[str],
+    rng: np.random.Generator,
+    score: Union[Binary, Discrete, Continuous] = Binary(),
+    plot_mode: str = "posterior",
+    show_empirical_means: bool = True,
+    overlay_on_bars: bool = False,
+    output_path: Optional[str] = None,
+    title: Optional[str] = None,
+    add_legend: bool = False,
+    unit_width: int = 6,
+    height: int = 4,
+    dpi: int = 100,
+) -> Optional[plt.Figure]:
+    """Makes a violin plot of score estimates with corresponding CLD letters.
+
+    Args:
+        model_name_list: A list of model names.
+        result_arrays: A list of score arrays for each model.
+        cld_letters: A list of CLD letters corresponding to each model.
+        rng: A numpy random Generator instance for posterior sampling.
+        score: Score type specification (Binary, Discrete, or Continuous).
+        plot_mode: "posterior" for posterior violin or "raw" for raw data violin.
+        show_empirical_means: Whether to show empirical mean dots.
+        overlay_on_bars: Whether to draw bar overlays behind violins.
+        output_path: Optional file path to save the plot.
+        title: Optional title for the plot.
+        add_legend: Whether to show legend.
+        unit_width: Figure width per model.
+        height: Figure height.
+        dpi: Resolution of the saved plot.
+
+    Returns:
+        If output_path is None, returns a matplotlib Figure object.
+        Otherwise, saves the plot and returns None.
+    """
+    num_models = len(model_name_list)
+
+    if plot_mode == "posterior":
+        violin_data = []
+        posterior_means = []
+        for result_array in result_arrays:
+            samples = draw_posterior_samples(
+                np.asarray(result_array), score, rng
+            )
+            violin_data.append(samples)
+            posterior_means.append(np.mean(samples))
+    elif plot_mode == "raw":
+        violin_data = [np.asarray(arr) for arr in result_arrays]
+    else:
+        raise ValueError(f"Unknown plot_mode: '{plot_mode}'. Use 'posterior' or 'raw'.")
+
+    empirical_means = [np.mean(arr) for arr in result_arrays]
+
+    fig, ax = plt.subplots(figsize=(max(unit_width, num_models), height), dpi=dpi)
+
+    cmap = plt.get_cmap("tab10")
+    colors = [cmap(i % 10) for i in range(num_models)]
+
+    if overlay_on_bars:
+        ax.bar(
+            np.arange(num_models),
+            empirical_means,
+            color=colors,
+            alpha=0.15,
+            width=0.8,
+            zorder=1,
+        )
+
+    parts = ax.violinplot(
+        violin_data,
+        positions=np.arange(num_models),
+        showmeans=False,
+        showmedians=False,
+        showextrema=False,
+        widths=0.8,
+    )
+    for pc, color in zip(parts["bodies"], colors):
+        pc.set_facecolor(color)
+        pc.set_alpha(0.6)
+
+    # Posterior mode: show posterior mean line + empirical mean dot.
+    # Raw mode: do NOT draw posterior mean line.
+    if plot_mode == "posterior":
+        # Posterior mean horizontal lines.
+        for i, mean_val in enumerate(posterior_means):
+            ax.hlines(
+                mean_val,
+                i - 0.3,
+                i + 0.3,
+                colors="black",
+                linewidths=1.2,
+                zorder=3,
+                label="Posterior mean" if i == 0 else None,
+            )
+
+    if show_empirical_means:
+        ax.scatter(
+            np.arange(num_models),
+            empirical_means,
+            color="black",
+            s=30,
+            zorder=4,
+            marker="o",
+            label="Empirical mean",
+        )
+
+    # Add CLD labels.
+    label_y = posterior_means if plot_mode == "posterior" else empirical_means
+    for i, (x, y, label) in enumerate(
+        zip(np.arange(num_models), label_y, cld_letters)
+    ):
+        ax.text(
+            x + 0.15,
+            y + 0.03,
+            label,
+            fontsize=12,
+            fontweight="bold",
+            color="black",
+            verticalalignment="center",
+            zorder=5,
+        )
+
+    ax.set_xticks(np.arange(num_models))
+    ax.set_xticklabels(model_name_list, rotation=0, ha="center")
+
+    # Derive y-limits from score bounds / grid / data.
+    if isinstance(score, Binary):
+        y_lo, y_hi = 0.0, 1.0
+    elif isinstance(score, Continuous):
+        if score.grid is not None:
+            grid = np.asarray(sorted(score.grid), dtype=float)
+            y_lo, y_hi = float(grid[0]), float(grid[-1])
+        else:
+            y_lo, y_hi = score.bounds
+    elif isinstance(score, Discrete):
+        y_lo = float(min(score.support))
+        y_hi = float(max(score.support))
+    else:
+        # Fallback: derive from data.
+        all_vals = np.concatenate([np.asarray(a) for a in result_arrays])
+        y_lo, y_hi = float(np.min(all_vals)), float(np.max(all_vals))
+
+    padding = (y_hi - y_lo) * 0.05
+    ax.set_ylim(y_lo - padding, y_hi + padding)
+    ax.set_ylabel("Score")
+    if title is not None:
+        ax.set_title(title)
+    if add_legend:
+        ax.legend(loc="best")
+    plt.tight_layout()
+
+    if output_path is not None:
+        plt.savefig(output_path, dpi=300)
+        plt.close()
+        return None
+    else:
+        return fig

statistical_comparison_helpers/posterior.py

@@ -0,0 +1,241 @@
+"""Posterior sampling and score-type specifications."""
+
+from dataclasses import dataclass, field
+from typing import List, Optional, Union
+
+import numpy as np
+from scipy import stats
+
+
+@dataclass(frozen=True)
+class Binary:
+    """Score spec for binary success/failure data. Uses Beta posterior.
+
+    Args:
+        alpha_prior: Alpha parameter for the Beta prior. Defaults to 1.
+        beta_prior: Beta parameter for the Beta prior. Defaults to 1.
+    """
+
+    alpha_prior: float = 1.0
+    beta_prior: float = 1.0
+
+    def __post_init__(self):
+        if self.alpha_prior <= 0:
+            raise ValueError("alpha_prior must be > 0.")
+        if self.beta_prior <= 0:
+            raise ValueError("beta_prior must be > 0.")
+
+
+@dataclass(frozen=True)
+class Discrete:
+    """Score spec for discrete partial-progress values.
+
+    Uses Dirichlet posterior over the exact support. Observed values must lie
+    exactly on the support points.
+
+    Args:
+        support: The exact set of possible score values, e.g. [0, 0.25, 0.5, 0.75, 1.0].
+        alpha_prior: Symmetric Dirichlet prior concentration per bin. Defaults to 1.
+    """
+
+    support: List[float] = field(default_factory=lambda: [0.0, 1.0])
+    alpha_prior: float = 1.0
+
+    def __post_init__(self):
+        if len(self.support) < 2:
+            raise ValueError("Discrete support must have at least 2 points.")
+        arr = np.asarray(self.support, dtype=float)
+        if not np.all(np.isfinite(arr)):
+            raise ValueError("Discrete support must contain only finite values.")
+        if len(np.unique(arr)) != len(arr):
+            raise ValueError("Discrete support must not contain duplicates.")
+        if self.alpha_prior <= 0:
+            raise ValueError("alpha_prior must be > 0.")
+
+
+@dataclass(frozen=True)
+class Continuous:
+    """Score spec for bounded continuous scores.
+
+    Uses Dirichlet posterior over an approximation grid. Observed values are
+    snapped to the nearest grid point.
+
+    Args:
+        bounds: (min, max) bounds for the score range. Defaults to (0.0, 1.0).
+        num_bins: Number of bins for the approximation grid. The grid has
+            num_bins + 1 points. Defaults to 10.
+        grid: Explicit grid points. If provided, num_bins and bounds are ignored.
+        alpha_prior: Symmetric Dirichlet prior concentration per bin. Defaults to 1.
+    """
+
+    bounds: tuple = (0.0, 1.0)
+    num_bins: int = 10
+    grid: Optional[List[float]] = None
+    alpha_prior: float = 1.0
+
+    def __post_init__(self):
+        if self.grid is not None:
+            arr = np.asarray(self.grid, dtype=float)
+            if len(arr) < 2:
+                raise ValueError("Continuous grid must have at least 2 points.")
+            if not np.all(np.isfinite(arr)):
+                raise ValueError("Continuous grid must contain only finite values.")
+            if len(np.unique(arr)) != len(arr):
+                raise ValueError("Continuous grid must not contain duplicates.")
+        else:
+            lo, hi = self.bounds
+            if lo >= hi:
+                raise ValueError(f"bounds min ({lo}) must be < max ({hi}).")
+            if self.num_bins <= 0:
+                raise ValueError("num_bins must be > 0.")
+        if self.alpha_prior <= 0:
+            raise ValueError("alpha_prior must be > 0.")
+
+    def get_grid(self) -> np.ndarray:
+        """Return the approximation grid as a numpy array."""
+        if self.grid is not None:
+            return np.asarray(sorted(self.grid), dtype=float)
+        lo, hi = self.bounds
+        return np.linspace(lo, hi, self.num_bins + 1)
+
+
+def draw_samples_from_beta_posterior(
+    success_array: np.ndarray,
+    rng: np.random.Generator,
+    num_samples: int = 10000,
+    alpha_prior: float = 1,
+    beta_prior: float = 1,
+) -> np.ndarray:
+    """Draw samples from the beta posterior distribution given a success array.
+
+    These samples can be used to estimate the posterior distribution of the
+    success rate of a Bernoulli process. The default prior parameters of (1, 1)
+    correspond to a uniform prior.
+
+    Args:
+        success_array: A binary array with 0/1 or bool values indicating failure/success.
+        rng: A numpy random Generator instance.
+        num_samples: Number of samples to draw. Defaults to 10000.
+        alpha_prior: Alpha parameter of the beta prior. Defaults to 1.
+        beta_prior: Beta parameter of the beta prior. Defaults to 1.
+
+    Returns:
+        Samples drawn from the beta posterior distribution.
+    """
+    if num_samples <= 0:
+        raise ValueError("num_samples must be > 0.")
+    if alpha_prior <= 0:
+        raise ValueError("alpha_prior must be > 0.")
+    if beta_prior <= 0:
+        raise ValueError("beta_prior must be > 0.")
+    success_array = np.asarray(success_array)
+    if success_array.size > 0:
+        unique_vals = set(np.unique(success_array).tolist())
+        if not unique_vals.issubset({0, 1, 0.0, 1.0, True, False}):
+            raise ValueError(
+                "Binary data must contain only 0/1 or boolean values. "
+                f"Found: {unique_vals}"
+            )
+    n_trials = len(success_array)
+    n_successes = np.sum(success_array)
+    n_failures = n_trials - n_successes
+    posterior = stats.beta(alpha_prior + n_successes, beta_prior + n_failures)
+    return posterior.rvs(num_samples, random_state=rng)
+
+
+def draw_samples_from_dirichlet_posterior(
+    progress_array: np.ndarray,
+    progress_bins: np.ndarray,
+    rng: np.random.Generator,
+    num_samples: int = 10000,
+    alpha_prior: float = 1,
+) -> np.ndarray:
+    """Draw posterior mean samples via Dirichlet posterior over a discrete grid.
+
+    Args:
+        progress_array: Observed score values. Must lie exactly on progress_bins
+            for Discrete, or will be snapped for Continuous.
+        progress_bins: Sorted grid points for the Dirichlet posterior.
+        rng: A numpy random Generator instance.
+        num_samples: Number of samples to draw. Defaults to 10000.
+        alpha_prior: Symmetric Dirichlet prior concentration per bin.
+            Defaults to 1.
+
+    Returns:
+        Samples of the posterior mean (weighted average of grid points under
+        Dirichlet-sampled probability vectors).
+    """
+    if num_samples <= 0:
+        raise ValueError("num_samples must be > 0.")
+    if alpha_prior <= 0:
+        raise ValueError("alpha_prior must be > 0.")
+
+    progress_bins = np.asarray(sorted(progress_bins), dtype=float)
+    progress_array = np.asarray(progress_array, dtype=float)
+
+    if len(progress_bins) < 2:
+        raise ValueError("progress_bins must have at least 2 points.")
+    if len(np.unique(progress_bins)) != len(progress_bins):
+        raise ValueError("progress_bins must not contain duplicates.")
+    if not np.all(np.isfinite(progress_bins)):
+        raise ValueError("progress_bins must contain only finite values.")
+
+    if not np.all(np.isin(np.round(progress_array, 10), np.round(progress_bins, 10))):
+        raise ValueError("All progress values must be in progress_bins.")
+
+    alpha_vec = alpha_prior * np.ones(len(progress_bins))
+    progress_counts = np.array(
+        [np.sum(np.isclose(progress_array, b)) for b in progress_bins]
+    )
+    posterior = stats.dirichlet(alpha_vec + progress_counts)
+    p_samples = posterior.rvs(size=num_samples, random_state=rng)
+    mean_samples = p_samples.dot(progress_bins)
+    return mean_samples
+
+
+def _snap_to_grid(values: np.ndarray, grid: np.ndarray) -> np.ndarray:
+    """Snap continuous values to the nearest grid point."""
+    values = np.asarray(values, dtype=float)
+    grid = np.asarray(grid, dtype=float)
+    # For each value, find the nearest grid point.
+    indices = np.abs(values[:, None] - grid[None, :]).argmin(axis=1)
+    return grid[indices]
+
+
+def draw_posterior_samples(
+    data: np.ndarray,
+    score: Union[Binary, Discrete, Continuous],
+    rng: np.random.Generator,
+    num_samples: int = 10000,
+) -> np.ndarray:
+    """Draw posterior samples for the given score type.
+
+    Args:
+        data: Observed score values.
+        score: A score spec (Binary, Discrete, or Continuous).
+        rng: A numpy random Generator instance.
+        num_samples: Number of samples to draw.
+
+    Returns:
+        Posterior mean samples.
+    """
+    if isinstance(score, Binary):
+        return draw_samples_from_beta_posterior(
+            data, rng, num_samples=num_samples,
+            alpha_prior=score.alpha_prior, beta_prior=score.beta_prior,
+        )
+    elif isinstance(score, Discrete):
+        bins = np.asarray(sorted(score.support), dtype=float)
+        return draw_samples_from_dirichlet_posterior(
+            data, bins, rng, num_samples=num_samples,
+            alpha_prior=score.alpha_prior,
+        )
+    elif isinstance(score, Continuous):
+        grid = score.get_grid()
+        snapped = _snap_to_grid(data, grid)
+        return draw_samples_from_dirichlet_posterior(
+            snapped, grid, rng, num_samples=num_samples,
+            alpha_prior=score.alpha_prior,
+        )
+    else:
+        raise TypeError(f"Unknown score type: {type(score)}")

statistical_comparison_helpers-0.1.0.dist-info/METADATA

@@ -0,0 +1,22 @@
+Metadata-Version: 2.4
+Name: statistical_comparison_helpers
+Version: 0.1.0
+Summary: Method-agnostic helpers for policy comparison: CLD, posterior sampling, and plotting.
+License: CC BY-NC 4.0
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.21
+Requires-Dist: scipy>=1.7
+Requires-Dist: matplotlib>=3.5
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == "dev"
+Dynamic: license-file
+
+# License statement
+
+The code is provided under a Attribution-NonCommercial 4.0 International (CC BY-NC 4.0) license. Under the license, the code is provided royalty free for non-commercial purposes only. The code may be covered by patents and if you want to use the code for commercial purposes, please contact us for a different license.
+
+# statistical_comparison_helpers
+
+This library is a part of [AnyEval](https://github.com/TRI-ML/AnyEval) software. It provides method-agnostic helpers for policy comparison: CLD, posterior sampling, and plotting.

statistical_comparison_helpers-0.1.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+statistical_comparison_helpers/__init__.py,sha256=S76FUSxbd1HhjW-yba9DsLzfmKZCMinpBwnruABI8Nw,795
+statistical_comparison_helpers/cld.py,sha256=UBcd70I4OOLwUx3sz8bX9q1etrXGwK3dLb_l1MbwzDM,5674
+statistical_comparison_helpers/plotting.py,sha256=LAirBD2j8ERadFl2nnirwgJl3c1MmEJpzXAcvgf9Yes,5526
+statistical_comparison_helpers/posterior.py,sha256=5HVL9m4f3c5jiYx_ggTUhrpFLflKG5pWMxmeZ5cJHYY,9065
+statistical_comparison_helpers-0.1.0.dist-info/licenses/LICENSE,sha256=mgNWNPeRFQBBSFB9tlT-Tu2q7RNx96r4s1dKJ6RExKE,19916
+statistical_comparison_helpers-0.1.0.dist-info/METADATA,sha256=9C-9ZLd6628I9vCG_ShxrdWG4BXf0FefgIJJ_khH3VY,1000
+statistical_comparison_helpers-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+statistical_comparison_helpers-0.1.0.dist-info/top_level.txt,sha256=K4h3UgVUQdWiELE7iHPOF9XdpLS8X6mYMQIZiemebq0,31
+statistical_comparison_helpers-0.1.0.dist-info/RECORD,,

statistical_comparison_helpers-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

statistical_comparison_helpers-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,352 @@
+Creative Commons Attribution-NonCommercial 4.0 International
+
+Creative Commons Corporation ("Creative Commons") is not a law firm and
+does not provide legal services or legal advice. Distribution of
+Creative Commons public licenses does not create a lawyer-client or
+other relationship. Creative Commons makes its licenses and related
+information available on an "as-is" basis. Creative Commons gives no
+warranties regarding its licenses, any material licensed under their
+terms and conditions, or any related information. Creative Commons
+disclaims all liability for damages resulting from their use to the
+fullest extent possible.
+
+Using Creative Commons Public Licenses
+
+Creative Commons public licenses provide a standard set of terms and
+conditions that creators and other rights holders may use to share
+original works of authorship and other material subject to copyright and
+certain other rights specified in the public license below. The
+following considerations are for informational purposes only, are not
+exhaustive, and do not form part of our licenses.
+
+-   Considerations for licensors: Our public licenses are intended for
+    use by those authorized to give the public permission to use
+    material in ways otherwise restricted by copyright and certain other
+    rights. Our licenses are irrevocable. Licensors should read and
+    understand the terms and conditions of the license they choose
+    before applying it. Licensors should also secure all rights
+    necessary before applying our licenses so that the public can reuse
+    the material as expected. Licensors should clearly mark any material
+    not subject to the license. This includes other CC-licensed
+    material, or material used under an exception or limitation to
+    copyright. More considerations for licensors :
+    wiki.creativecommons.org/Considerations_for_licensors
+
+-   Considerations for the public: By using one of our public licenses,
+    a licensor grants the public permission to use the licensed material
+    under specified terms and conditions. If the licensor's permission
+    is not necessary for any reason–for example, because of any
+    applicable exception or limitation to copyright–then that use is not
+    regulated by the license. Our licenses grant only permissions under
+    copyright and certain other rights that a licensor has authority to
+    grant. Use of the licensed material may still be restricted for
+    other reasons, including because others have copyright or other
+    rights in the material. A licensor may make special requests, such
+    as asking that all changes be marked or described. Although not
+    required by our licenses, you are encouraged to respect those
+    requests where reasonable. More considerations for the public :
+    wiki.creativecommons.org/Considerations_for_licensees
+
+Creative Commons Attribution-NonCommercial 4.0 International Public
+License
+
+By exercising the Licensed Rights (defined below), You accept and agree
+to be bound by the terms and conditions of this Creative Commons
+Attribution-NonCommercial 4.0 International Public License ("Public
+License"). To the extent this Public License may be interpreted as a
+contract, You are granted the Licensed Rights in consideration of Your
+acceptance of these terms and conditions, and the Licensor grants You
+such rights in consideration of benefits the Licensor receives from
+making the Licensed Material available under these terms and conditions.
+
+-   Section 1 – Definitions.
+
+    -   a. Adapted Material means material subject to Copyright and
+        Similar Rights that is derived from or based upon the Licensed
+        Material and in which the Licensed Material is translated,
+        altered, arranged, transformed, or otherwise modified in a
+        manner requiring permission under the Copyright and Similar
+        Rights held by the Licensor. For purposes of this Public
+        License, where the Licensed Material is a musical work,
+        performance, or sound recording, Adapted Material is always
+        produced where the Licensed Material is synched in timed
+        relation with a moving image.
+    -   b. Adapter's License means the license You apply to Your
+        Copyright and Similar Rights in Your contributions to Adapted
+        Material in accordance with the terms and conditions of this
+        Public License.
+    -   c. Copyright and Similar Rights means copyright and/or similar
+        rights closely related to copyright including, without
+        limitation, performance, broadcast, sound recording, and Sui
+        Generis Database Rights, without regard to how the rights are
+        labeled or categorized. For purposes of this Public License, the
+        rights specified in Section 2(b)(1)-(2) are not Copyright and
+        Similar Rights.
+    -   d. Effective Technological Measures means those measures that,
+        in the absence of proper authority, may not be circumvented
+        under laws fulfilling obligations under Article 11 of the WIPO
+        Copyright Treaty adopted on December 20, 1996, and/or similar
+        international agreements.
+    -   e. Exceptions and Limitations means fair use, fair dealing,
+        and/or any other exception or limitation to Copyright and
+        Similar Rights that applies to Your use of the Licensed
+        Material.
+    -   f. Licensed Material means the artistic or literary work,
+        database, or other material to which the Licensor applied this
+        Public License.
+    -   g. Licensed Rights means the rights granted to You subject to
+        the terms and conditions of this Public License, which are
+        limited to all Copyright and Similar Rights that apply to Your
+        use of the Licensed Material and that the Licensor has authority
+        to license.
+    -   h. Licensor means the individual(s) or entity(ies) granting
+        rights under this Public License.
+    -   i. NonCommercial means not primarily intended for or directed
+        towards commercial advantage or monetary compensation. For
+        purposes of this Public License, the exchange of the Licensed
+        Material for other material subject to Copyright and Similar
+        Rights by digital file-sharing or similar means is NonCommercial
+        provided there is no payment of monetary compensation in
+        connection with the exchange.
+    -   j. Share means to provide material to the public by any means or
+        process that requires permission under the Licensed Rights, such
+        as reproduction, public display, public performance,
+        distribution, dissemination, communication, or importation, and
+        to make material available to the public including in ways that
+        members of the public may access the material from a place and
+        at a time individually chosen by them.
+    -   k. Sui Generis Database Rights means rights other than copyright
+        resulting from Directive 96/9/EC of the European Parliament and
+        of the Council of 11 March 1996 on the legal protection of
+        databases, as amended and/or succeeded, as well as other
+        essentially equivalent rights anywhere in the world.
+    -   l. You means the individual or entity exercising the Licensed
+        Rights under this Public License. Your has a corresponding
+        meaning.
+
+-   Section 2 – Scope.
+
+    -   a. License grant.
+        -   1. Subject to the terms and conditions of this Public
+            License, the Licensor hereby grants You a worldwide,
+            royalty-free, non-sublicensable, non-exclusive, irrevocable
+            license to exercise the Licensed Rights in the Licensed
+            Material to:
+            -   A. reproduce and Share the Licensed Material, in whole
+                or in part, for NonCommercial purposes only; and
+            -   B. produce, reproduce, and Share Adapted Material for
+                NonCommercial purposes only.
+        -   2. Exceptions and Limitations. For the avoidance of doubt,
+            where Exceptions and Limitations apply to Your use, this
+            Public License does not apply, and You do not need to comply
+            with its terms and conditions.
+        -   3. Term. The term of this Public License is specified in
+            Section 6(a).
+        -   4. Media and formats; technical modifications allowed. The
+            Licensor authorizes You to exercise the Licensed Rights in
+            all media and formats whether now known or hereafter
+            created, and to make technical modifications necessary to do
+            so. The Licensor waives and/or agrees not to assert any
+            right or authority to forbid You from making technical
+            modifications necessary to exercise the Licensed Rights,
+            including technical modifications necessary to circumvent
+            Effective Technological Measures. For purposes of this
+            Public License, simply making modifications authorized by
+            this Section 2(a)(4) never produces Adapted Material.
+        -   5. Downstream recipients.
+            -   A. Offer from the Licensor – Licensed Material. Every
+                recipient of the Licensed Material automatically
+                receives an offer from the Licensor to exercise the
+                Licensed Rights under the terms and conditions of this
+                Public License.
+            -   B. No downstream restrictions. You may not offer or
+                impose any additional or different terms or conditions
+                on, or apply any Effective Technological Measures to,
+                the Licensed Material if doing so restricts exercise of
+                the Licensed Rights by any recipient of the Licensed
+                Material.
+        -   6. No endorsement. Nothing in this Public License
+            constitutes or may be construed as permission to assert or
+            imply that You are, or that Your use of the Licensed
+            Material is, connected with, or sponsored, endorsed, or
+            granted official status by, the Licensor or others
+            designated to receive attribution as provided in Section
+            3(a)(1)(A)(i).
+    -   b. Other rights.
+        -   1. Moral rights, such as the right of integrity, are not
+            licensed under this Public License, nor are publicity,
+            privacy, and/or other similar personality rights; however,
+            to the extent possible, the Licensor waives and/or agrees
+            not to assert any such rights held by the Licensor to the
+            limited extent necessary to allow You to exercise the
+            Licensed Rights, but not otherwise.
+        -   2. Patent and trademark rights are not licensed under this
+            Public License.
+        -   3. To the extent possible, the Licensor waives any right to
+            collect royalties from You for the exercise of the Licensed
+            Rights, whether directly or through a collecting society
+            under any voluntary or waivable statutory or compulsory
+            licensing scheme. In all other cases the Licensor expressly
+            reserves any right to collect such royalties, including when
+            the Licensed Material is used other than for NonCommercial
+            purposes.
+
+-   Section 3 – License Conditions.
+
+    Your exercise of the Licensed Rights is expressly made subject to
+    the following conditions.
+
+    -   a. Attribution.
+        -   1. If You Share the Licensed Material (including in modified
+            form), You must:
+            -   A. retain the following if it is supplied by the
+                Licensor with the Licensed Material:
+                -   i. identification of the creator(s) of the Licensed
+                    Material and any others designated to receive
+                    attribution, in any reasonable manner requested by
+                    the Licensor (including by pseudonym if designated);
+                -   ii. a copyright notice;
+                -   iii. a notice that refers to this Public License;
+                -   iv. a notice that refers to the disclaimer of
+                    warranties;
+                -   v. a URI or hyperlink to the Licensed Material to
+                    the extent reasonably practicable;
+            -   B. indicate if You modified the Licensed Material and
+                retain an indication of any previous modifications; and
+            -   C. indicate the Licensed Material is licensed under this
+                Public License, and include the text of, or the URI or
+                hyperlink to, this Public License.
+        -   2. You may satisfy the conditions in Section 3(a)(1) in any
+            reasonable manner based on the medium, means, and context in
+            which You Share the Licensed Material. For example, it may
+            be reasonable to satisfy the conditions by providing a URI
+            or hyperlink to a resource that includes the required
+            information.
+        -   3. If requested by the Licensor, You must remove any of the
+            information required by Section 3(a)(1)(A) to the extent
+            reasonably practicable.
+        -   4. If You Share Adapted Material You produce, the Adapter's
+            License You apply must not prevent recipients of the Adapted
+            Material from complying with this Public License.
+
+-   Section 4 – Sui Generis Database Rights.
+
+    Where the Licensed Rights include Sui Generis Database Rights that
+    apply to Your use of the Licensed Material:
+
+    -   a. for the avoidance of doubt, Section 2(a)(1) grants You the
+        right to extract, reuse, reproduce, and Share all or a
+        substantial portion of the contents of the database for
+        NonCommercial purposes only;
+    -   b. if You include all or a substantial portion of the database
+        contents in a database in which You have Sui Generis Database
+        Rights, then the database in which You have Sui Generis Database
+        Rights (but not its individual contents) is Adapted Material;
+        and
+    -   c. You must comply with the conditions in Section 3(a) if You
+        Share all or a substantial portion of the contents of the
+        database.
+
+    For the avoidance of doubt, this Section 4 supplements and does not
+    replace Your obligations under this Public License where the
+    Licensed Rights include other Copyright and Similar Rights.
+
+-   Section 5 – Disclaimer of Warranties and Limitation of Liability.
+
+    -   a. Unless otherwise separately undertaken by the Licensor, to
+        the extent possible, the Licensor offers the Licensed Material
+        as-is and as-available, and makes no representations or
+        warranties of any kind concerning the Licensed Material, whether
+        express, implied, statutory, or other. This includes, without
+        limitation, warranties of title, merchantability, fitness for a
+        particular purpose, non-infringement, absence of latent or other
+        defects, accuracy, or the presence or absence of errors, whether
+        or not known or discoverable. Where disclaimers of warranties
+        are not allowed in full or in part, this disclaimer may not
+        apply to You.
+    -   b. To the extent possible, in no event will the Licensor be
+        liable to You on any legal theory (including, without
+        limitation, negligence) or otherwise for any direct, special,
+        indirect, incidental, consequential, punitive, exemplary, or
+        other losses, costs, expenses, or damages arising out of this
+        Public License or use of the Licensed Material, even if the
+        Licensor has been advised of the possibility of such losses,
+        costs, expenses, or damages. Where a limitation of liability is
+        not allowed in full or in part, this limitation may not apply to
+        You.
+    -   c. The disclaimer of warranties and limitation of liability
+        provided above shall be interpreted in a manner that, to the
+        extent possible, most closely approximates an absolute
+        disclaimer and waiver of all liability.
+
+-   Section 6 – Term and Termination.
+
+    -   a. This Public License applies for the term of the Copyright and
+        Similar Rights licensed here. However, if You fail to comply
+        with this Public License, then Your rights under this Public
+        License terminate automatically.
+    -   b. Where Your right to use the Licensed Material has terminated
+        under Section 6(a), it reinstates:
+
+        -   1. automatically as of the date the violation is cured,
+            provided it is cured within 30 days of Your discovery of the
+            violation; or
+        -   2. upon express reinstatement by the Licensor.
+
+        For the avoidance of doubt, this Section 6(b) does not affect
+        any right the Licensor may have to seek remedies for Your
+        violations of this Public License.
+
+    -   c. For the avoidance of doubt, the Licensor may also offer the
+        Licensed Material under separate terms or conditions or stop
+        distributing the Licensed Material at any time; however, doing
+        so will not terminate this Public License.
+    -   d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
+        License.
+
+-   Section 7 – Other Terms and Conditions.
+
+    -   a. The Licensor shall not be bound by any additional or
+        different terms or conditions communicated by You unless
+        expressly agreed.
+    -   b. Any arrangements, understandings, or agreements regarding the
+        Licensed Material not stated herein are separate from and
+        independent of the terms and conditions of this Public License.
+
+-   Section 8 – Interpretation.
+
+    -   a. For the avoidance of doubt, this Public License does not, and
+        shall not be interpreted to, reduce, limit, restrict, or impose
+        conditions on any use of the Licensed Material that could
+        lawfully be made without permission under this Public License.
+    -   b. To the extent possible, if any provision of this Public
+        License is deemed unenforceable, it shall be automatically
+        reformed to the minimum extent necessary to make it enforceable.
+        If the provision cannot be reformed, it shall be severed from
+        this Public License without affecting the enforceability of the
+        remaining terms and conditions.
+    -   c. No term or condition of this Public License will be waived
+        and no failure to comply consented to unless expressly agreed to
+        by the Licensor.
+    -   d. Nothing in this Public License constitutes or may be
+        interpreted as a limitation upon, or waiver of, any privileges
+        and immunities that apply to the Licensor or You, including from
+        the legal processes of any jurisdiction or authority.
+
+Creative Commons is not a party to its public licenses. Notwithstanding,
+Creative Commons may elect to apply one of its public licenses to
+material it publishes and in those instances will be considered the
+"Licensor." The text of the Creative Commons public licenses is
+dedicated to the public domain under the CC0 Public Domain Dedication.
+Except for the limited purpose of indicating that material is shared
+under a Creative Commons public license or as otherwise permitted by the
+Creative Commons policies published at creativecommons.org/policies,
+Creative Commons does not authorize the use of the trademark "Creative
+Commons" or any other trademark or logo of Creative Commons without its
+prior written consent including, without limitation, in connection with
+any unauthorized modifications to any of its public licenses or any
+other arrangements, understandings, or agreements concerning use of
+licensed material. For the avoidance of doubt, this paragraph does not
+form part of the public licenses.
+
+Creative Commons may be contacted at creativecommons.org.

statistical_comparison_helpers-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+statistical_comparison_helpers