sverify-spark 0.8.0__py3-none-any.whl

sverify_spark/__init__.py

@@ -0,0 +1,186 @@
+"""sverify's Spark engine (#1189): the Spark SQL dialect
+(`sverify_spark.dialect`: `SPARK`, `SPARK_CONF`), with shims where Spark
+differs from the reference, proved against Spark's facts (`Rel.spark_sound`),
+and the runner on a Spark session (`sverify_spark.engine`), registered as the
+`ProfilePlugin` `PLUGIN` under the entry-point group `sverify.profiles`.
+
+Spark's settings: a run reads the session's `spark.conf`: sverify's local
+session (`sverify_spark.engine.session`, which starts with `SPARK_CONF`) or
+the session passed as the engine. The facts were probed under ANSI mode,
+with `TIMESTAMP` as `TIMESTAMP_NTZ` and the session time zone UTC
+(spark-exec-settings); the writer's SQL relies on those and on
+`caseSensitive` and `escapedStringLiterals`, so a session that changes them
+refuses every run. The legacy statistics, decimal-precision and
+`percentile_disc` settings refuse only plans whose options they change.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+from sverify._unsupported import ISSUE_271, RANGE_GUARD
+from sverify.profiles import ProfilePlugin, Setting
+from sverify_spark.store import ArrowStore
+
+#: The settings Spark's profile assumes.
+SETTINGS: tuple[Setting, ...] = (
+    Setting(
+        "spark.sql.ansi.enabled",
+        (True,),
+        "refuse",
+        ("spark-exec-settings", "spark-int-arith", "spark-int-divide", "spark-double-divide"),
+        "whether integer overflow, a zero divisor and a bad cast raise or give NULL or a "
+        "wrapped value (overflow, on_division_by_zero, cast_failure)",
+    ),
+    Setting(
+        "spark.sql.timestampType",
+        ("timestamp_ntz",),
+        "refuse",
+        ("spark-time-zones", "spark-type-names"),
+        "what TIMESTAMP means: an instant in the session time zone or a wall-clock time",
+    ),
+    Setting(
+        "spark.sql.session.timeZone",
+        ("utc", "etc/utc"),
+        "refuse",
+        ("spark-time-zones",),
+        "the wall clock the written SQL reads microseconds as (`unix_micros`, `timestamp_micros`)",
+    ),
+    Setting(
+        "spark.sql.caseSensitive",
+        (True,),
+        "refuse",
+        ("spark-exec-settings",),
+        "whether columns `a` and `A` are one column",
+    ),
+    Setting(
+        "spark.sql.parser.escapedStringLiterals",
+        (False,),
+        "refuse",
+        ("spark-exec-settings",),
+        "how a backslash in a string literal reads",
+    ),
+    Setting(
+        "spark.sql.legacy.statisticalAggregate",
+        (False,),
+        "refuse",
+        ("spark-var-stddev", "spark-covar", "spark-corr"),
+        "whether a sample variance, covariance or correlation of one value is NULL or NaN "
+        "(stat_degenerate)",
+        ("stat_degenerate", "stat_algo"),
+    ),
+    Setting(
+        "spark.sql.decimalOperations.allowPrecisionLoss",
+        (True,),
+        "refuse",
+        ("spark-decimal-arith",),
+        "the scale of a decimal result past 38 digits (decimal_rescale)",
+        ("decimal_rescale",),
+    ),
+    Setting(
+        "spark.sql.legacy.percentileDiscCalculation",
+        (False,),
+        "refuse",
+        ("spark-quantile",),
+        "which value percentile_disc takes (quantile_interp)",
+        ("quantile_interp",),
+    ),
+)
+
+#: Spark 4.3.0's defaults for the settings `SETTINGS` reads (spark-exec-settings).
+DEFAULTS: Mapping[str, str] = {
+    "spark.sql.ansi.enabled": "true",
+    "spark.sql.timestampType": "TIMESTAMP_LTZ",
+    "spark.sql.caseSensitive": "false",
+    "spark.sql.parser.escapedStringLiterals": "false",
+    "spark.sql.legacy.statisticalAggregate": "false",
+    "spark.sql.decimalOperations.allowPrecisionLoss": "true",
+    "spark.sql.legacy.percentileDiscCalculation": "false",
+}
+
+
+def _run(
+    con: Any, plan_json: str, session: Any, *, keep: bool, finite: bool, tie_free: bool
+) -> Any:
+    from sverify.engine import _rename_tables, read_tables
+    from sverify_spark import engine as spark_engine
+
+    plan = json.loads(plan_json)
+    # Spark's temporary views take plain names, not the delta plans' `t$delta`.
+    names = {t: f"__sverify_{i}" for i, t in enumerate(read_tables(plan))}
+    _rename_tables(plan, names)
+    tables = {alias: con._arrow[t] for t, alias in names.items()}
+    return spark_engine.run(plan, tables, session, deterministic=tie_free, finite=finite)
+
+
+def _is_session(obj: Any) -> bool:
+    """A Spark session (any object that is not an engine's name was one,
+    before engines were plugins; a session's class is pyspark's)."""
+    return type(obj).__module__.startswith("pyspark.")
+
+
+def _version() -> str | None:
+    import pyspark
+
+    return pyspark.__version__
+
+
+def _local_session() -> Any:
+    """The session a run on `"spark"` uses: sverify's local one, if it has
+    started (`sverify_spark.engine.session`); None before, when the run
+    starts it with `SPARK_CONF`."""
+    engine = sys.modules.get("sverify_spark.engine")
+    if engine is None or engine.session.cache_info().currsize == 0:
+        return None
+    return engine.session()
+
+
+def _read_settings(names: Sequence[str], session: Any) -> dict[str, Any]:
+    spark = session if session is not None else _local_session()
+    if spark is None:
+        return {}
+    return {n: spark.conf.get(n, DEFAULTS.get(n)) for n in names}
+
+
+def _restore(setting: Setting) -> str:
+    from sverify_spark.dialect import SPARK_CONF
+
+    v = SPARK_CONF.get(setting.name, DEFAULTS.get(setting.name, setting.expected[0]))
+    return f'spark.conf.set("{setting.name}", "{v}")'
+
+
+def _dialect() -> Any:
+    from sverify_spark.dialect import SPARK
+
+    return SPARK
+
+
+PLUGIN = ProfilePlugin(
+    "spark",
+    store=ArrowStore,
+    title="Spark",
+    profile=True,
+    translation="sverify's Spark SQL writer, with shims where Spark differs from the "
+    "reference, proved against Spark's facts (Rel.spark_sound) and tested against "
+    "the reference",
+    settings=SETTINGS,
+    installed_version=_version,
+    restore=_restore,
+    runner=_run,
+    session=_is_session,
+    modules=("pyspark",),
+    read_settings=_read_settings,
+    dialect=_dialect,
+    refuses={
+        "sverify_raise": (
+            "Spark's SQL of the date guards exhausts its JVM's heap (a date from columns "
+            "is 38 KB of nested CASEs)",
+            ISSUE_271,
+        )
+    },
+    guards=(RANGE_GUARD,),
+    shim_text={"null_order": "NULLs sorted as the reference sorts them"},
+)

sverify_spark/dialect.py

@@ -0,0 +1,408 @@
+"""Apache Spark SQL (and Databricks SQL) with the Lean model's semantics.
+
+Targets Spark 4 with the session configuration of `SPARK_CONF`, above all
+ANSI mode: without it integer overflow wraps around and a bad cast is NULL,
+where the Lean model (and this SQL) raises. Every hook either emits SQL whose
+result is the Lean value, checked on a local Spark session by
+`engines/spark/tests/test_sql_spark.py`, or raises `Unsupported`. The divergences:
+
+* **Integers** are `BIGINT`. In ANSI mode `+`, `-`, `*`, negation and
+  `MIN div -1` raise on overflow. `/` returns a double, so integer division
+  is `div`, which truncates; `%` has the sign of the dividend. A literal
+  without a cast is an `INT` (or a `DECIMAL` beyond 64 bits), so every
+  integer constant is cast. `SUM` of `BIGINT` raises as soon as a partial
+  sum overflows, though Lean's is the true total: it adds `DECIMAL(38,0)`
+  values instead (exact) and checks the total against 64 bits.
+* **Doubles** are IEEE, but in ANSI mode `/` raises on a zero divisor, so
+  `x / 0.0` is spelled out (±inf by the sign of `x`, or NaN).
+  `CAST(x AS BIGINT)` truncates, raises on NaN and ±inf, but saturates at
+  2⁶³: the range is checked first. `FLOOR` and `CEIL` return `BIGINT`, so
+  they keep NaN, ±inf and large values as they are. `POWER` is Java's
+  `Math.pow`, which is only within 1 ulp, so double powers are refused.
+* **Zero and NaN**: `=` and `<` are the model's: NaN equals itself and is
+  largest, and -0.0 equals 0.0. `GROUP BY`, `DISTINCT` and window
+  partitions treat them the same way (`NormalizeFloatingNumbers`), and
+  `canon` keeps -0.0 out of results.
+* **NULL order**: `NULLS FIRST` is the default for ascending sorts; every
+  key says where NULLs go.
+* **Timestamps** are `TIMESTAMP_NTZ`: wall-clock microseconds like Lean.
+  Plain `TIMESTAMP` is `TIMESTAMP_LTZ` (by `spark.sql.timestampType`), whose
+  calendar fields follow the session time zone, and `date_trunc` returns
+  one, so truncation is computed on `TIMESTAMP_NTZ` and `DATE` values
+  instead and never touches the time zone. Intervals are `INTERVAL DAY TO
+  SECOND`, microseconds like Lean. `extract(SECOND …)` is a decimal with the
+  microseconds; `WEEK` is ISO, and `dayofweek` counts from Sunday = 1. Spark
+  holds timestamps beyond the reference's (years 0 and +10000), so the
+  writer raises outside 0001-01-01 to 9999-12-31 before `date_part` (D32).
+* **Strings**: string literals take backslash escapes (with
+  `spark.sql.parser.escapedStringLiterals` off), so backslashes are
+  doubled. Strings compare by UTF-8 bytes under the default collation
+  `UTF8_BINARY`, which is Lean's code point order; a column with another
+  collation (`UTF8_LCASE`, ICU) would compare differently. `length`,
+  `locate` and `substring` count characters. Regular expressions are
+  Java's, which agree with Rust's on simple patterns (black boxes in Lean);
+  `RLIKE` searches. `upper`, `lower` and patterns with `\\d`, `\\w` or
+  `\\s` raise on non-ASCII text, where Java's tables differ from Rust's (#70).
+* **Decimals** (#27) follow the reference's types, which are Spark's own
+  (`DecimalPrecision`) up to 38 digits; past them Spark cuts the scale to no
+  less than 6 (`spark.sql.decimalOperations.allowPrecisionLoss`), so a sum
+  or product whose type would pass 38 digits at a scale above 6 is refused.
+  A smaller scale rounds half away from zero (`ROUND_HALF_UP`), as the
+  reference does. `-x` rounds past 34 significant digits (SP-1), so a
+  negation of more than 34 digits is `0 - x`, exact up to scale 6 at 38
+  digits, and refused past it. A double goes to a decimal as
+  `round(x · 10ˢ)` in doubles, through `BIGINT`, since Spark reads a double
+  as its shortest text. Division by an integer (an `AVG`) is refused.
+* **Rows**: tables (Parquet, Delta, views) have no row order, so nothing
+  can stand for Lean's list order, which FETCH and ties in a sort or window
+  follow: `row_ordinal` is refused. Windows and sorts run with
+  `to_sql(..., deterministic=True)`, when the caller knows there are no ties.
+  A `SUM` or `AVG` of doubles is Spark's own, within the plan's bound of the
+  reference's correctly rounded sum (D25, `float_sum`).
+* **Identifiers** are quoted with backticks; `spark.sql.caseSensitive` keeps
+  `a` and `A` apart.
+"""
+
+from __future__ import annotations
+
+import math
+import re
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+from sverify.sql.core import I64_MAX, I64_MIN, Dec, Dialect, Kind, Unsupported, _timestamp_text
+
+#: Session settings the SQL relies on; `SparkSession.builder.config` each.
+SPARK_CONF: dict[str, str] = {
+    # Overflow, division by zero and bad casts raise instead of wrapping or
+    # giving NULL.
+    "spark.sql.ansi.enabled": "true",
+    # Only LTZ values (none here) depend on it; pinned so nothing does.
+    "spark.sql.session.timeZone": "UTC",
+    "spark.sql.timestampType": "TIMESTAMP_NTZ",
+    "spark.sql.caseSensitive": "true",
+    # Backslash escapes in string literals, as `str_lit` writes them.
+    "spark.sql.parser.escapedStringLiterals": "false",
+}
+
+_TYPES: dict[str, str] = {
+    "int": "BIGINT",
+    "float": "DOUBLE",
+    "time": "TIMESTAMP_NTZ",
+    "date": "DATE",
+    "interval": "INTERVAL DAY TO SECOND",
+    "str": "STRING",
+    "bool": "BOOLEAN",
+}
+
+
+def _digits(k: Kind) -> tuple[int, int]:
+    """The precision and scale Spark gives an operand of kind `k` in decimal
+    arithmetic: a `BIGINT` is a `DECIMAL(20, 0)`."""
+    if isinstance(k, Dec):
+        return k.p, k.s
+    if k == "int":
+        return 20, 0
+    raise Unsupported(f"decimal arithmetic with a {k} operand on Spark")
+
+
+@dataclass(frozen=True)
+class SparkDialect(Dialect):
+    # Decimals (#27), as the module's docstring says.
+    decimals: bool = True
+
+    def quote(self, ident: str) -> str:
+        return "`" + ident.replace("`", "``") + "`"
+
+    def type_name(self, kind: Kind) -> str:
+        if isinstance(kind, Dec):
+            return super().type_name(kind)
+        if kind not in _TYPES:
+            raise self.to_duckdb(f"spark has no {kind} type (TIME needs Spark 4.1)")
+        return _TYPES[kind]
+
+    def time_of_day_lit(self, micros: int) -> str:
+        raise self.to_duckdb("spark has no tod type (TIME needs Spark 4.1)")
+
+    def raise_(self, message: str, dep: str, kind: Kind) -> str:
+        text = f"concat({self.str_lit(f'sverify: {message}: ')}, CAST({dep} AS STRING))"
+        fail = self.cast(f"raise_error({text})", kind)
+        return f"(CASE WHEN {dep} IS NULL THEN {self.null(kind)} ELSE {fail} END)"
+
+    # Literals
+
+    def str_lit(self, s: str) -> str:
+        s = s.replace("\\", "\\\\").replace("'", "\\'")
+        # Control characters as escapes, so every query is one line of text.
+        return "'" + "".join(f"\\u{ord(c):04x}" if ord(c) < 32 else c for c in s) + "'"
+
+    def timestamp_lit(self, micros: int) -> str:
+        return f"TIMESTAMP_NTZ '{_timestamp_text(micros)}'"
+
+    # Integers: ANSI mode raises on BIGINT overflow.
+
+    def trunc_div(self, a: str, b: str) -> str:
+        return f"({a} div {b})"
+
+    def trunc_mod(self, a: str, b: str) -> str:
+        return f"({a} % {b})"
+
+    def int_product(self, a: str, n: str) -> str:
+        # Partial products grow in magnitude, so the checked `*` raises
+        # exactly when the result leaves 64 bits.
+        reps = f"array_repeat({a}, CAST({n} AS INT))"
+        return f"aggregate({reps}, {self.int_lit(1)}, (acc, e) -> acc * e)"
+
+    # Doubles
+
+    def float_div(self, a: str, b: str) -> str:
+        if (c := _float_const(b)) is not None and c != 0:
+            return f"({a} / {b})"
+        nan, inf, ninf = (self.float_lit(x) for x in (math.nan, math.inf, -math.inf))
+        # The divisor is the one zero, +0.0.
+        by_zero = (
+            f"(CASE WHEN {a} IS NULL THEN {self.null('float')} "
+            f"WHEN isnan({a}) OR {a} = 0 THEN {nan} WHEN {a} > 0 THEN {inf} ELSE {ninf} END)"
+        )
+        return f"(CASE WHEN {b} = 0 THEN {by_zero} ELSE {a} / {b} END)"
+
+    def whole(self, fn: str, x: str) -> str:
+        """`fn` (FLOOR or CEIL, which return BIGINT) on a double. Doubles of
+        magnitude 2⁵² and more are whole already; NaN and ±inf stay."""
+        big = self.float_lit(2.0**52)
+        value = f"CAST({fn}({x}) AS DOUBLE)"
+        return f"(CASE WHEN isnan({x}) OR abs({x}) >= {big} THEN {x} ELSE {value} END)"
+
+    def float_floor(self, x: str) -> str:
+        return self.whole("FLOOR", x)
+
+    def float_ceil(self, x: str) -> str:
+        return self.whole("CEIL", x)
+
+    def float_round(self, x: str) -> str:
+        # Half away from zero on the shortest decimal of the double, which is
+        # the double's own rounding (a halfway double prints as one).
+        return f"ROUND({x})"
+
+    def float_logb(self, x: str, b: str) -> str:
+        raise self.to_duckdb(
+            "Spark's LN is Java's StrictMath, not the C libm the Lean model uses "
+            "(spark-elem-rounding)"
+        )
+
+    def float_pow(self, a: str, b: str) -> str:
+        raise self.to_duckdb(
+            "Spark's POWER is Java's Math.pow, within 1 ulp of the C pow the Lean model uses "
+            "(spark-elem-rounding)"
+        )
+
+    def float_to_int(self, x: str) -> str:
+        # The cast raises on NaN and ±inf but saturates at 2⁶³: check first.
+        lo, hi = self.float_lit(-(2.0**63)), self.float_lit(2.0**63)
+        fail = self.raise_("CAST to BIGINT out of range", x, "int")
+        return f"(CASE WHEN {x} >= {lo} AND {x} < {hi} THEN CAST({x} AS BIGINT) ELSE {fail} END)"
+
+    def is_nan(self, x: str) -> str:
+        return f"isnan({x})"
+
+    def float_eq(self, a: str, b: str) -> str:
+        return f"({a} = {b})"
+
+    def float_lt(self, a: str, b: str) -> str:
+        return f"({a} < {b})"
+
+    # Temporal
+
+    def date_part(self, part: str, ts: str) -> str:
+        sec = f"extract(SECOND FROM {ts})"
+        value = {
+            "week": f"extract(WEEK FROM {ts})",
+            "dow": f"(dayofweek({ts}) - 1)",
+            "doy": f"dayofyear({ts})",
+            "second": f"CAST(FLOOR({sec}) AS BIGINT)",
+            "millisecond": f"CAST(FLOOR({sec} * 1000) AS BIGINT)",
+            "microsecond": f"CAST({sec} * 1000000 AS BIGINT)",
+        }.get(part, f"{part}({ts})")
+        return self.cast(value, "int")
+
+    def date_trunc(self, unit: str, ts: str) -> str:
+        # `date_trunc` returns TIMESTAMP_LTZ; these stay in NTZ and DATE.
+        sec = f"extract(SECOND FROM {ts})"
+        day = f"CAST({ts} AS DATE)"
+        match unit:
+            case "microsecond":
+                return ts
+            case "millisecond":
+                return f"({ts} - make_dt_interval(0, 0, 0, {sec} - FLOOR({sec} * 1000) / 1000))"
+            case "second":
+                return f"({ts} - make_dt_interval(0, 0, 0, {sec} - FLOOR({sec})))"
+            case "minute":
+                return f"({ts} - make_dt_interval(0, 0, 0, {sec}))"
+            case "hour":
+                return f"({ts} - make_dt_interval(0, 0, minute({ts}), {sec}))"
+            case "day":
+                return f"CAST({day} AS TIMESTAMP_NTZ)"
+            case "week":
+                # dayofweek: Sunday 1, Monday 2.
+                return f"CAST(date_sub({day}, (dayofweek({ts}) + 5) % 7) AS TIMESTAMP_NTZ)"
+        return f"CAST(trunc({day}, {self.str_lit(unit.upper())}) AS TIMESTAMP_NTZ)"
+
+    def date_days(self, d: str) -> str:
+        return f"CAST(unix_date({d}) AS BIGINT)"
+
+    def date_add_days(self, d: str, n: str) -> str:
+        return f"date_add({d}, CAST({n} AS INT))"
+
+    def add_months(self, x: str, months: int, kind: Kind) -> str:
+        # `add_months` clamps to the end of a shorter month, as
+        # `Time.addMonths` (since Spark 3.0 also from a month's last day).
+        if kind == "time":
+            return f"({x} + make_ym_interval(0, {months}))"
+        return f"add_months({x}, {months})"
+
+    def as_time(self, us: str) -> str:
+        # The session time zone is UTC, so the NTZ cast keeps the instant's clock.
+        return f"CAST(timestamp_micros({us}) AS TIMESTAMP_NTZ)"
+
+    def time_micros(self, ts: str) -> str:
+        # The session time zone is UTC, so the LTZ cast keeps the wall clock.
+        return f"unix_micros(CAST({ts} AS TIMESTAMP_LTZ))"
+
+    # Aggregates
+
+    def int_sum(self, v: str) -> str:
+        total = f"SUM(CAST({v} AS DECIMAL(38,0)))"
+        fail = self.raise_("integer overflow in SUM", total, "int")
+        inside = f"{total} BETWEEN {I64_MIN} AND {I64_MAX}"
+        return f"(CASE WHEN {inside} THEN CAST({total} AS BIGINT) ELSE {fail} END)"
+
+    def int_avg(self, v: str, over: str) -> str:
+        # An exact DECIMAL(38,0) sum; `div` truncates toward zero.
+        total = f"SUM(CAST({v} AS DECIMAL(38,0))) {over}"
+        return f"CAST({total} div COUNT({v}) {over} AS BIGINT)"
+
+    def window_int_sum(self, v: str, over: str) -> str:
+        total = f"SUM(CAST({v} AS DECIMAL(38,0))) {over}"
+        fail = self.raise_("integer overflow in SUM", total, "int")
+        inside = f"{total} BETWEEN {I64_MIN} AND {I64_MAX}"
+        return f"(CASE WHEN {inside} THEN CAST({total} AS BIGINT) ELSE {fail} END)"
+
+    def day_interval(self, days: int) -> str:
+        return f"INTERVAL {days} DAYS"
+
+    # Decimals
+
+    def decimal_arith(
+        self, op: str, a: str, b: str, sa: int, k: Dec, kinds: tuple[Kind, Kind] | None = None
+    ) -> str:
+        if kinds is None:
+            raise Unsupported(f"decimal {op} of operands of unknown types on Spark")
+        (p1, s1), (p2, s2) = (_digits(x) for x in kinds)
+        if op == "*":
+            p, sc = p1 + p2 + 1, s1 + s2
+        else:
+            sc = max(s1, s2)
+            p = max(p1 - s1, p2 - s2) + sc + 1
+        if p > 38 and sc > 6:
+            # spark-decimal-arith: Spark cuts the scale to fit 38 digits.
+            raise Unsupported(
+                f"decimal {op} of DECIMAL({p1}, {s1}) and DECIMAL({p2}, {s2}) on Spark: "
+                f"its type DECIMAL({p}, {sc}) has more than 38 digits, and Spark rounds it "
+                "to fewer decimal places; cast an operand to fewer digits"
+            )
+        return self.cast(f"({a} {op} {b})", k)
+
+    def decimal_neg(self, a: str, k: Dec | None = None) -> str:
+        # SP-1: `-x` rounds past 34 significant digits. `0 - x` is exact where
+        # its type keeps the scale: up to 37 digits, or 38 at scale 6 or less.
+        if k is not None and k.p <= 34:
+            return f"(-{a})"
+        if k is not None and (k.p <= 37 or k.s <= 6):
+            return self.cast(f"({self.decimal_lit(0, Dec(1, 0))} - {a})", k)
+        raise self.to_duckdb(
+            f"negation of a DECIMAL({k.p if k else '?'}, {k.s if k else '?'}) on Spark: "
+            "Spark rounds -x past 34 digits and 0 - x past scale 6 at 38 digits (SP-1)",
+            "see #27",
+            "cast it to 34 digits or fewer",
+        )
+
+    def decimal_abs(self, a: str, k: Dec | None = None) -> str:
+        # SP-1: `abs` negates a negative value as `-x` does.
+        return f"(CASE WHEN {a} < 0 THEN {self.decimal_neg(a, k)} ELSE {a} END)"
+
+    def decimal_cast(self, a: str, k: Dec, src: Kind | None = None) -> str:
+        if src != "float":
+            return self.cast(a, k)
+        if k.p > 18:
+            raise self.to_duckdb(
+                f"CAST of a double to DECIMAL({k.p}, {k.s}) on Spark: Spark reads the "
+                "double as its shortest text, which past 18 digits is not its value",
+                "see #27",
+                "cast to 18 digits or fewer",
+            )
+        # `round(x · 10ˢ)` in doubles, halves away from zero (Spark's `ROUND` of
+        # a double rounds its shortest text, which keeps every half where it
+        # is); whole doubles below 2⁶³ are exact as `BIGINT`s.
+        scale = self.float_lit(float(10**k.s))
+        units = self.cast(f"CAST(ROUND({a} * {scale}) AS BIGINT)", Dec(k.p, 0))
+        return self.cast(f"({units} * {self.decimal_lit(1, Dec(k.s + 1, k.s))})", k)
+
+    def decimal_to_int(self, a: str) -> str:
+        # Truncates toward zero; ANSI raises outside 64 bits.
+        return f"CAST({a} AS BIGINT)"
+
+    def decimal_div(self, a: str, b: str, sa: int, k: Dec) -> str:
+        raise self.to_duckdb(
+            "decimal division (an AVG of decimals) on Spark: Spark's decimal quotient "
+            "rounds at a scale of its own first (spark-decimal-arith)",
+            "see #27",
+            "cast it to DOUBLE",
+        )
+
+    def float_extreme(self, fn: str, v: str, over: str) -> str:
+        # NaN is the greatest double, and -0.0 equals 0.0, keeping whichever
+        # came first (spark-nan-order, spark-zero-order): the codes' order up
+        # to the zero, which `canon` makes the one zero (`Shim.one_zero`).
+        return self.canon(f"{fn}({v}) {over}" if over else f"{fn}({v})")
+
+    def like_pattern(self, p: str) -> str:
+        odd = f"({p} RLIKE {self.str_lit(r'(^|[^\\])(\\\\)*\\$')})"
+        return f"(CASE WHEN {odd} THEN concat({p}, {self.str_lit(chr(92))}) ELSE {p} END)"
+
+    def substring(self, s: str, start: str, length: str | None) -> str:
+        return (
+            f"substring({s}, {start})" if length is None else f"substring({s}, {start}, {length})"
+        )
+
+    def regex_search(self, s: str, pattern: str) -> str:
+        return f"({s} RLIKE {self.str_lit(pattern)})"
+
+    # Windows
+
+    def over(self, partition: Sequence[str], order: Sequence[str]) -> str:
+        if not order:
+            raise self.to_duckdb("Spark window functions need an ORDER BY")
+        return super().over(partition, order)
+
+    def lag(self, x: str, offset: int, default: str, over: str) -> str:
+        return x if offset == 0 else super().lag(x, offset, default, over)
+
+    def lead(self, x: str, offset: int, default: str, over: str) -> str:
+        return x if offset == 0 else super().lead(x, offset, default, over)
+
+    # Relations
+
+    def row_ordinal(self) -> str:
+        raise Unsupported(
+            "Spark tables have no row order, so FETCH and windows over a named table "
+            "cannot follow the Lean model's list order (windows run with deterministic=True)"
+        )
+
+
+def _float_const(sql: str) -> float | None:
+    """The value of a `float_lit` constant, or None for other SQL."""
+    m = re.fullmatch(r"CAST\('([^']*)' AS DOUBLE\)", sql)
+    return float(m.group(1)) if m else None
+
+
+SPARK: Dialect = SparkDialect("spark")

sverify_spark/engine.py

@@ -0,0 +1,194 @@
+"""Spark as an engine, through SQL generated from the plan (`pip install
+'sverify[spark]'`, and a JVM: Java 17 or 21 on `PATH` or in `JAVA_HOME`).
+
+Internal: not part of sverify-spark's public API, which is `PLUGIN` and the
+dialect (`sverify_spark.dialect`: `SPARK`, `SPARK_CONF`); it may change in any
+release.
+
+`run` writes the plan as Spark SQL with `sverify.sql.to_sql` (whose semantics
+are Lean's `Rel.eval`, see `sverify_spark.dialect`) and runs it on a Spark
+session holding the given tables as temporary views. `session()` is a local
+session with the settings of `SPARK_CONF`; on a cluster or Databricks, pass
+your own session, configured the same way. Plans that need a table's row
+order are refused (`Unsupported`) unless `deterministic=True`.
+"""
+
+from __future__ import annotations
+
+import functools
+import os
+import re
+import sys
+import weakref
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Any
+
+import pyarrow as pa
+
+from sverify.sql import to_sql
+from sverify_spark.dialect import SPARK, SPARK_CONF
+
+try:
+    import pyspark  # noqa: F401
+except ImportError as e:
+    raise ImportError("the Spark engine needs pyspark: pip install 'sverify[spark]'") from e
+
+if TYPE_CHECKING:
+    from pyspark.sql import SparkSession
+
+
+#: Settings of the local session beyond `SPARK_CONF`. The parser's ANTLR DFA
+#: cache is static and unbounded unless Spark manages it: the nested SQL that
+#: `to_sql` writes for date arithmetic (one 51 kB query of
+#: `tests/test_backends.py`) left about 740 MB of a 1 GB driver heap in it,
+#: and later queries failed with `AGGREGATE_OUT_OF_MEMORY` or could not
+#: broadcast. Flushing it at 10 % of the heap keeps it near 100 MB.
+LOCAL_CONF: dict[str, str] = {
+    "spark.ui.enabled": "false",
+    "spark.sql.shuffle.partitions": "2",
+    "spark.log.level": "OFF",
+    "spark.sql.parser.manageParserCaches": "true",
+    "spark.sql.parser.parserDfaCacheFlushRatio": "10",
+}
+
+
+def driver_memory(limit: str) -> str:
+    """`SVERIFY_MEMORY_LIMIT` (#783) as Spark's `spark.driver.memory`:
+    `"2GB"` is `"2g"`, `"512MB"` is `"512m"`."""
+    m = re.fullmatch(r"\s*(\d+)\s*([KMGT])i?B?\s*", limit, re.IGNORECASE)
+    if m is None:
+        raise ValueError(f"memory limit {limit!r}: write a size such as 2GB or 512MB")
+    return f"{m[1]}{m[2].lower()}"
+
+
+@functools.cache
+def session() -> SparkSession:
+    """A local Spark session (one per process) with the settings of
+    `SPARK_CONF` and `LOCAL_CONF`. `SVERIFY_MEMORY_LIMIT` sets the driver's
+    memory (Spark's default is 1 GB) when the session's JVM starts. Python
+    workers run this process's interpreter unless `PYSPARK_PYTHON` names
+    another."""
+    from pyspark.sql import SparkSession
+
+    builder = SparkSession.builder.master("local[2]").appName("sverify")
+    conf = SPARK_CONF | LOCAL_CONF
+    if limit := os.environ.get("SVERIFY_MEMORY_LIMIT", "").strip():
+        conf["spark.driver.memory"] = driver_memory(limit)
+    for k, v in conf.items():
+        builder = builder.config(k, v)
+    spark = builder.getOrCreate()
+    spark.sparkContext.setLogLevel("OFF")
+    if not os.environ.get("PYSPARK_PYTHON"):
+        # Python workers run this interpreter, not whichever `python3` is
+        # first on `PATH` (a virtual environment need not be on it).
+        spark.sparkContext.pythonExec = sys.executable
+    return spark
+
+
+def spark_type(t: pa.DataType) -> str:
+    """The Spark type of an Arrow column, as `sverify_spark.dialect` writes it."""
+    if pa.types.is_integer(t):
+        return SPARK.type_name("int")
+    if pa.types.is_decimal(t):
+        return f"DECIMAL({t.precision}, {t.scale})"
+    if pa.types.is_floating(t):
+        return SPARK.type_name("float")
+    if pa.types.is_timestamp(t):
+        if t.tz is not None:
+            raise TypeError(f"{t}: only timestamps without a time zone (TIMESTAMP_NTZ)")
+        return SPARK.type_name("time")
+    if pa.types.is_date(t):
+        return SPARK.type_name("date")
+    if pa.types.is_boolean(t):
+        return SPARK.type_name("bool")
+    if pa.types.is_string(t) or pa.types.is_large_string(t):
+        return SPARK.type_name("str")
+    if pa.types.is_list(t) or pa.types.is_large_list(t):
+        return f"ARRAY<{spark_type(t.value_type)}>"
+    raise TypeError(f"no Spark type for {t}")
+
+
+def arrow_type(t: pa.DataType) -> pa.DataType:
+    """The Arrow type Spark reads as `spark_type(t)`: 64-bit integers and
+    doubles, microsecond timestamps, plain strings and lists."""
+    if pa.types.is_integer(t):
+        return pa.int64()
+    if pa.types.is_floating(t):
+        return pa.float64()
+    if pa.types.is_timestamp(t):
+        return pa.timestamp("us")
+    if pa.types.is_string(t) or pa.types.is_large_string(t):
+        return pa.string()
+    if pa.types.is_list(t) or pa.types.is_large_list(t):
+        return pa.list_(arrow_type(t.value_type))
+    return t
+
+
+#: The views sverify registered under its own names (`__sverify_<i>`, which
+#: `sverify.engine.run_plan` gives a plan's tables), per session: the Arrow
+#: table each holds. A run over the same table does not send it again.
+_VIEWS: weakref.WeakKeyDictionary[Any, dict[str, pa.Table]] = weakref.WeakKeyDictionary()
+
+
+def put_tables(tables: Mapping[str, pa.Table], spark: SparkSession | None = None) -> None:
+    """Register each table as a temporary view of that name. The columns go
+    to Spark as Arrow, cast to the types `spark_type` names, so NULL and NaN
+    stay apart, and no Python worker runs (a table of 100,000 rows takes a
+    quarter of the time it took as Python values). A view of sverify's own
+    names that holds the same Arrow table already is kept as it is."""
+    spark = spark or session()
+    held = _VIEWS.setdefault(spark, {})
+    for name, table in tables.items():
+        own = name.startswith("__sverify_")
+        if own and held.get(name) is table:
+            continue
+        schema = ", ".join(f"{SPARK.quote(f.name)} {spark_type(f.type)}" for f in table.schema)
+        target = pa.schema([pa.field(f.name, arrow_type(f.type)) for f in table.schema])
+        spark.createDataFrame(table.cast(target), schema).createOrReplaceTempView(name)
+        if own:
+            held[name] = table
+
+
+def run(
+    plan: str | dict,
+    tables: Mapping[str, pa.Table],
+    spark: SparkSession | None = None,
+    *,
+    deterministic: bool = False,
+    finite: bool = True,
+) -> pa.Table:
+    """Run a Substrait JSON plan on Spark over `tables` (temporary views).
+
+    The plan is written as Spark SQL by `sverify.engines.to_sql` in the
+    `sverify_spark.dialect.SPARK` dialect. `sverify.engines.run(con, plan,
+    "spark")` runs a plan over a connection's tables the same way.
+
+    Args:
+        plan: A Substrait JSON plan, as a string or parsed.
+        tables: The tables the plan reads, by name. Each is registered as a
+            temporary view of that name, replacing any view of the same name.
+        spark: The Spark session to run on, configured with
+            `sverify_spark.dialect.SPARK_CONF`. Default: a local session with
+            those settings.
+        deterministic: Promise that no two rows tie on any window's
+            `PARTITION BY` and `ORDER BY` columns (what a contract report's
+            `deterministic` says), so windows run without a row-order
+            tiebreak (see `sverify.engines.to_sql`). Default False.
+        finite: Check that no operation on finite doubles gives `±∞`, the
+            reference's precondition (#575); False for a plan verified
+            under a profile that allows `±∞`, as Spark's does. Default True.
+
+    Returns:
+        The plan's result, as an Arrow table.
+
+    Raises:
+        sverify.engines.UnsupportedError: Spark SQL cannot express the plan with
+            Lean's meaning, for example a plan that needs a table's row order
+            without `deterministic=True`.
+        TypeError: A column has a type Spark cannot hold here, such as a
+            timestamp with a time zone.
+    """
+    spark = spark or session()
+    sql = to_sql(plan, SPARK, deterministic=deterministic, finite=finite)
+    put_tables(tables, spark)
+    return spark.sql(sql).toArrow()

sverify_spark/store.py

@@ -0,0 +1,12 @@
+"""Spark's store, for now (#1189, interim): its views' state as Arrow
+tables in the Python process, with the operations of `sverify._arrow_state`,
+until the cluster holds it keeps it natively (`SparkStore`).
+"""
+
+from __future__ import annotations
+
+from sverify._arrow_state import ArrowOps
+
+
+class ArrowStore(ArrowOps):
+    """Spark's views' state as Arrow tables in the Python process."""

sverify_spark-0.8.0.dist-info/METADATA

@@ -0,0 +1,39 @@
+Metadata-Version: 2.5
+Name: sverify-spark
+Version: 0.8.0
+Summary: Spark as an engine of sverify: plans verified in Lean, written as Spark SQL
+Project-URL: Source, https://github.com/AlexanderNenninger/substrait-lean/tree/main/engines/spark
+Project-URL: Issues, https://github.com/AlexanderNenninger/substrait-lean/issues
+License-Expression: MPL-2.0
+License-File: LICENSE
+Requires-Python: >=3.12
+Requires-Dist: sverify==0.8.0
+Provides-Extra: run
+Requires-Dist: pyspark>=4.0; extra == 'run'
+Description-Content-Type: text/markdown
+
+# sverify-spark
+
+Sverify-spark is a working prototype, and so is sverify, which it extends.
+Their proofs and code still need a human audit before anyone relies on their
+results.
+
+Spark as an engine of [sverify](https://github.com/AlexanderNenninger/substrait-lean):
+a plan verified in Lean is written as Spark SQL, with a shim where Spark
+differs from the reference, and runs on a Spark session under ANSI mode.
+`pip install sverify[spark]` installs it with PySpark; `sverify-spark` alone
+writes Spark SQL (`sverify_spark.dialect.SPARK`) without PySpark. sverify finds
+it through its `ProfilePlugin` in the entry-point group `sverify.profiles`.
+
+The directory holds the whole engine package:
+
+- `src/sverify_spark/`: the plugin (`PLUGIN`), the dialect (`dialect`:
+  `SPARK`, `SPARK_CONF`) and the runner on a Spark session (`engine`);
+- `facts/`: what Spark does at each of the core's options, the evidence for
+  `Profile.spark` ([facts/README.md](facts/README.md));
+- `probes/`: the probes' and the fuzzer's recordings the facts cite;
+- `tests/`: its tests, which run in sverify's test session
+  (`python/pyproject.toml`'s `testpaths`).
+
+Its Lean half, the profile and its shims, is in the root Lean package for now
+(`SubstraitLean/SparkProfile.lean`, `SparkShims.lean`).

sverify_spark-0.8.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+sverify_spark/__init__.py,sha256=xx4yRu0cF1XE5EASNwnfWL5daVM1dW784KkZ3iNiKec,6435
+sverify_spark/dialect.py,sha256=L2dgLaVpzl3HKFm772Ujkf_51bAha77sWaNCvSy9K3Y,18345
+sverify_spark/engine.py,sha256=18MsfMbcT3Uz3lMxli03_iswy-X7GFmGprR3NLeTowk,8119
+sverify_spark/store.py,sha256=RD-w7zyFl_pvLPPPMDs9bxWIGMqEYcQqSQN0eOepO00,391
+sverify_spark-0.8.0.dist-info/METADATA,sha256=ZuObsEA3mWjBpZz9tcofDp5LECYN1m9P_dPuQOpIJnc,1803
+sverify_spark-0.8.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+sverify_spark-0.8.0.dist-info/entry_points.txt,sha256=SNWfL6iSa5w1rHwnco_4NuvRoR4f_ZH3UH6GwTzNbv8,48
+sverify_spark-0.8.0.dist-info/licenses/LICENSE,sha256=Pz2eACSxkhsGfW9_iN60pgy-enjnbGTj8df8O3ebnQQ,16726
+sverify_spark-0.8.0.dist-info/RECORD,,

sverify_spark-0.8.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any

sverify_spark-0.8.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[sverify.profiles]
+spark = sverify_spark:PLUGIN

sverify_spark-0.8.0.dist-info/licenses/LICENSE

@@ -0,0 +1,373 @@
+Mozilla Public License Version 2.0
+==================================
+
+1. Definitions
+--------------
+
+1.1. "Contributor"
+    means each individual or legal entity that creates, contributes to
+    the creation of, or owns Covered Software.
+
+1.2. "Contributor Version"
+    means the combination of the Contributions of others (if any) used
+    by a Contributor and that particular Contributor's Contribution.
+
+1.3. "Contribution"
+    means Covered Software of a particular Contributor.
+
+1.4. "Covered Software"
+    means Source Code Form to which the initial Contributor has attached
+    the notice in Exhibit A, the Executable Form of such Source Code
+    Form, and Modifications of such Source Code Form, in each case
+    including portions thereof.
+
+1.5. "Incompatible With Secondary Licenses"
+    means
+
+    (a) that the initial Contributor has attached the notice described
+        in Exhibit B to the Covered Software; or
+
+    (b) that the Covered Software was made available under the terms of
+        version 1.1 or earlier of the License, but not also under the
+        terms of a Secondary License.
+
+1.6. "Executable Form"
+    means any form of the work other than Source Code Form.
+
+1.7. "Larger Work"
+    means a work that combines Covered Software with other material, in
+    a separate file or files, that is not Covered Software.
+
+1.8. "License"
+    means this document.
+
+1.9. "Licensable"
+    means having the right to grant, to the maximum extent possible,
+    whether at the time of the initial grant or subsequently, any and
+    all of the rights conveyed by this License.
+
+1.10. "Modifications"
+    means any of the following:
+
+    (a) any file in Source Code Form that results from an addition to,
+        deletion from, or modification of the contents of Covered
+        Software; or
+
+    (b) any new file in Source Code Form that contains any Covered
+        Software.
+
+1.11. "Patent Claims" of a Contributor
+    means any patent claim(s), including without limitation, method,
+    process, and apparatus claims, in any patent Licensable by such
+    Contributor that would be infringed, but for the grant of the
+    License, by the making, using, selling, offering for sale, having
+    made, import, or transfer of either its Contributions or its
+    Contributor Version.
+
+1.12. "Secondary License"
+    means either the GNU General Public License, Version 2.0, the GNU
+    Lesser General Public License, Version 2.1, the GNU Affero General
+    Public License, Version 3.0, or any later versions of those
+    licenses.
+
+1.13. "Source Code Form"
+    means the form of the work preferred for making modifications.
+
+1.14. "You" (or "Your")
+    means an individual or a legal entity exercising rights under this
+    License. For legal entities, "You" includes any entity that
+    controls, is controlled by, or is under common control with You. For
+    purposes of this definition, "control" means (a) the power, direct
+    or indirect, to cause the direction or management of such entity,
+    whether by contract or otherwise, or (b) ownership of more than
+    fifty percent (50%) of the outstanding shares or beneficial
+    ownership of such entity.
+
+2. License Grants and Conditions
+--------------------------------
+
+2.1. Grants
+
+Each Contributor hereby grants You a world-wide, royalty-free,
+non-exclusive license:
+
+(a) under intellectual property rights (other than patent or trademark)
+    Licensable by such Contributor to use, reproduce, make available,
+    modify, display, perform, distribute, and otherwise exploit its
+    Contributions, either on an unmodified basis, with Modifications, or
+    as part of a Larger Work; and
+
+(b) under Patent Claims of such Contributor to make, use, sell, offer
+    for sale, have made, import, and otherwise transfer either its
+    Contributions or its Contributor Version.
+
+2.2. Effective Date
+
+The licenses granted in Section 2.1 with respect to any Contribution
+become effective for each Contribution on the date the Contributor first
+distributes such Contribution.
+
+2.3. Limitations on Grant Scope
+
+The licenses granted in this Section 2 are the only rights granted under
+this License. No additional rights or licenses will be implied from the
+distribution or licensing of Covered Software under this License.
+Notwithstanding Section 2.1(b) above, no patent license is granted by a
+Contributor:
+
+(a) for any code that a Contributor has removed from Covered Software;
+    or
+
+(b) for infringements caused by: (i) Your and any other third party's
+    modifications of Covered Software, or (ii) the combination of its
+    Contributions with other software (except as part of its Contributor
+    Version); or
+
+(c) under Patent Claims infringed by Covered Software in the absence of
+    its Contributions.
+
+This License does not grant any rights in the trademarks, service marks,
+or logos of any Contributor (except as may be necessary to comply with
+the notice requirements in Section 3.4).
+
+2.4. Subsequent Licenses
+
+No Contributor makes additional grants as a result of Your choice to
+distribute the Covered Software under a subsequent version of this
+License (see Section 10.2) or under the terms of a Secondary License (if
+permitted under the terms of Section 3.3).
+
+2.5. Representation
+
+Each Contributor represents that the Contributor believes its
+Contributions are its original creation(s) or it has sufficient rights
+to grant the rights to its Contributions conveyed by this License.
+
+2.6. Fair Use
+
+This License is not intended to limit any rights You have under
+applicable copyright doctrines of fair use, fair dealing, or other
+equivalents.
+
+2.7. Conditions
+
+Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
+in Section 2.1.
+
+3. Responsibilities
+-------------------
+
+3.1. Distribution of Source Form
+
+All distribution of Covered Software in Source Code Form, including any
+Modifications that You create or to which You contribute, must be under
+the terms of this License. You must inform recipients that the Source
+Code Form of the Covered Software is governed by the terms of this
+License, and how they can obtain a copy of this License. You may not
+attempt to alter or restrict the recipients' rights in the Source Code
+Form.
+
+3.2. Distribution of Executable Form
+
+If You distribute Covered Software in Executable Form then:
+
+(a) such Covered Software must also be made available in Source Code
+    Form, as described in Section 3.1, and You must inform recipients of
+    the Executable Form how they can obtain a copy of such Source Code
+    Form by reasonable means in a timely manner, at a charge no more
+    than the cost of distribution to the recipient; and
+
+(b) You may distribute such Executable Form under the terms of this
+    License, or sublicense it under different terms, provided that the
+    license for the Executable Form does not attempt to limit or alter
+    the recipients' rights in the Source Code Form under this License.
+
+3.3. Distribution of a Larger Work
+
+You may create and distribute a Larger Work under terms of Your choice,
+provided that You also comply with the requirements of this License for
+the Covered Software. If the Larger Work is a combination of Covered
+Software with a work governed by one or more Secondary Licenses, and the
+Covered Software is not Incompatible With Secondary Licenses, this
+License permits You to additionally distribute such Covered Software
+under the terms of such Secondary License(s), so that the recipient of
+the Larger Work may, at their option, further distribute the Covered
+Software under the terms of either this License or such Secondary
+License(s).
+
+3.4. Notices
+
+You may not remove or alter the substance of any license notices
+(including copyright notices, patent notices, disclaimers of warranty,
+or limitations of liability) contained within the Source Code Form of
+the Covered Software, except that You may alter any license notices to
+the extent required to remedy known factual inaccuracies.
+
+3.5. Application of Additional Terms
+
+You may choose to offer, and to charge a fee for, warranty, support,
+indemnity or liability obligations to one or more recipients of Covered
+Software. However, You may do so only on Your own behalf, and not on
+behalf of any Contributor. You must make it absolutely clear that any
+such warranty, support, indemnity, or liability obligation is offered by
+You alone, and You hereby agree to indemnify every Contributor for any
+liability incurred by such Contributor as a result of warranty, support,
+indemnity or liability terms You offer. You may include additional
+disclaimers of warranty and limitations of liability specific to any
+jurisdiction.
+
+4. Inability to Comply Due to Statute or Regulation
+---------------------------------------------------
+
+If it is impossible for You to comply with any of the terms of this
+License with respect to some or all of the Covered Software due to
+statute, judicial order, or regulation then You must: (a) comply with
+the terms of this License to the maximum extent possible; and (b)
+describe the limitations and the code they affect. Such description must
+be placed in a text file included with all distributions of the Covered
+Software under this License. Except to the extent prohibited by statute
+or regulation, such description must be sufficiently detailed for a
+recipient of ordinary skill to be able to understand it.
+
+5. Termination
+--------------
+
+5.1. The rights granted under this License will terminate automatically
+if You fail to comply with any of its terms. However, if You become
+compliant, then the rights granted under this License from a particular
+Contributor are reinstated (a) provisionally, unless and until such
+Contributor explicitly and finally terminates Your grants, and (b) on an
+ongoing basis, if such Contributor fails to notify You of the
+non-compliance by some reasonable means prior to 60 days after You have
+come back into compliance. Moreover, Your grants from a particular
+Contributor are reinstated on an ongoing basis if such Contributor
+notifies You of the non-compliance by some reasonable means, this is the
+first time You have received notice of non-compliance with this License
+from such Contributor, and You become compliant prior to 30 days after
+Your receipt of the notice.
+
+5.2. If You initiate litigation against any entity by asserting a patent
+infringement claim (excluding declaratory judgment actions,
+counter-claims, and cross-claims) alleging that a Contributor Version
+directly or indirectly infringes any patent, then the rights granted to
+You by any and all Contributors for the Covered Software under Section
+2.1 of this License shall terminate.
+
+5.3. In the event of termination under Sections 5.1 or 5.2 above, all
+end user license agreements (excluding distributors and resellers) which
+have been validly granted by You or Your distributors under this License
+prior to termination shall survive termination.
+
+************************************************************************
+*                                                                      *
+*  6. Disclaimer of Warranty                                           *
+*  -------------------------                                           *
+*                                                                      *
+*  Covered Software is provided under this License on an "as is"       *
+*  basis, without warranty of any kind, either expressed, implied, or  *
+*  statutory, including, without limitation, warranties that the       *
+*  Covered Software is free of defects, merchantable, fit for a        *
+*  particular purpose or non-infringing. The entire risk as to the     *
+*  quality and performance of the Covered Software is with You.        *
+*  Should any Covered Software prove defective in any respect, You     *
+*  (not any Contributor) assume the cost of any necessary servicing,   *
+*  repair, or correction. This disclaimer of warranty constitutes an   *
+*  essential part of this License. No use of any Covered Software is   *
+*  authorized under this License except under this disclaimer.         *
+*                                                                      *
+************************************************************************
+
+************************************************************************
+*                                                                      *
+*  7. Limitation of Liability                                          *
+*  --------------------------                                          *
+*                                                                      *
+*  Under no circumstances and under no legal theory, whether tort      *
+*  (including negligence), contract, or otherwise, shall any           *
+*  Contributor, or anyone who distributes Covered Software as          *
+*  permitted above, be liable to You for any direct, indirect,         *
+*  special, incidental, or consequential damages of any character      *
+*  including, without limitation, damages for lost profits, loss of    *
+*  goodwill, work stoppage, computer failure or malfunction, or any    *
+*  and all other commercial damages or losses, even if such party      *
+*  shall have been informed of the possibility of such damages. This   *
+*  limitation of liability shall not apply to liability for death or   *
+*  personal injury resulting from such party's negligence to the       *
+*  extent applicable law prohibits such limitation. Some               *
+*  jurisdictions do not allow the exclusion or limitation of           *
+*  incidental or consequential damages, so this exclusion and          *
+*  limitation may not apply to You.                                    *
+*                                                                      *
+************************************************************************
+
+8. Litigation
+-------------
+
+Any litigation relating to this License may be brought only in the
+courts of a jurisdiction where the defendant maintains its principal
+place of business and such litigation shall be governed by laws of that
+jurisdiction, without reference to its conflict-of-law provisions.
+Nothing in this Section shall prevent a party's ability to bring
+cross-claims or counter-claims.
+
+9. Miscellaneous
+----------------
+
+This License represents the complete agreement concerning the subject
+matter hereof. If any provision of this License is held to be
+unenforceable, such provision shall be reformed only to the extent
+necessary to make it enforceable. Any law or regulation which provides
+that the language of a contract shall be construed against the drafter
+shall not be used to construe this License against a Contributor.
+
+10. Versions of the License
+---------------------------
+
+10.1. New Versions
+
+Mozilla Foundation is the license steward. Except as provided in Section
+10.3, no one other than the license steward has the right to modify or
+publish new versions of this License. Each version will be given a
+distinguishing version number.
+
+10.2. Effect of New Versions
+
+You may distribute the Covered Software under the terms of the version
+of the License under which You originally received the Covered Software,
+or under the terms of any subsequent version published by the license
+steward.
+
+10.3. Modified Versions
+
+If you create software not governed by this License, and you want to
+create a new license for such software, you may create and use a
+modified version of this License if you rename the license and remove
+any references to the name of the license steward (except to note that
+such modified license differs from this License).
+
+10.4. Distributing Source Code Form that is Incompatible With Secondary
+Licenses
+
+If You choose to distribute Source Code Form that is Incompatible With
+Secondary Licenses under the terms of this version of the License, the
+notice described in Exhibit B of this License must be attached.
+
+Exhibit A - Source Code Form License Notice
+-------------------------------------------
+
+  This Source Code Form is subject to the terms of the Mozilla Public
+  License, v. 2.0. If a copy of the MPL was not distributed with this
+  file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+If it is not possible or desirable to put the notice in a particular
+file, then You may include the notice in a location (such as a LICENSE
+file in a relevant directory) where a recipient would be likely to look
+for such a notice.
+
+You may add additional accurate notices of copyright ownership.
+
+Exhibit B - "Incompatible With Secondary Licenses" Notice
+---------------------------------------------------------
+
+  This Source Code Form is "Incompatible With Secondary Licenses", as
+  defined by the Mozilla Public License, v. 2.0.