dagster-dataframely 0.0.1__py3-none-any.whl

dagster_dataframely/__init__.py

@@ -0,0 +1,32 @@
+from dagster_dataframely.asset import dataframely_asset
+from dagster_dataframely.checks import check_specs
+from dagster_dataframely.errors import (
+    CheckNameCollisionError,
+    CollectionNotSupportedError,
+    DagsterDataframelyError,
+    ReservedColumnError,
+    SchemaGateError,
+    UnwritableDtypeError,
+    ValidationAbortError,
+)
+from dagster_dataframely.io_managers import DataframelyParquetIOManager
+from dagster_dataframely.metadata import schema_metadata, table_schema
+from dagster_dataframely.naming import check_name
+from dagster_dataframely.runtime import process
+
+__all__ = [
+    "CheckNameCollisionError",
+    "CollectionNotSupportedError",
+    "DagsterDataframelyError",
+    "DataframelyParquetIOManager",
+    "ReservedColumnError",
+    "SchemaGateError",
+    "UnwritableDtypeError",
+    "ValidationAbortError",
+    "check_name",
+    "check_specs",
+    "dataframely_asset",
+    "process",
+    "schema_metadata",
+    "table_schema",
+]

dagster_dataframely/asset.py

@@ -0,0 +1,195 @@
+"""The front door: one decorator argument attaches a dataframely schema to a Dagster asset.
+
+The door coordinates four artifacts no single `@dg.asset` parameter accepts as a bundle: the out, the check specs, the definition metadata, and the wrapped runtime. First-party precedent for a decorator that does this is `@dbt_assets`.
+
+This module carries no `from __future__ import annotations`. At a 3.12 floor it would buy only unquoted forward references, while turning user-facing annotations into strings that Dagster's runtime introspection rejects. The counter-trap is that typing-only names such as `dg.CoercibleToAssetDep` are absent at runtime, so they are spelled here with runtime-real types.
+"""
+
+import functools
+from collections.abc import Callable, Iterable, Mapping, Sequence
+from collections.abc import Set as AbstractSet
+from typing import Any
+
+import dagster as dg
+import dataframely as dy
+import polars as pl
+
+from dagster_dataframely.checks import check_specs
+from dagster_dataframely.errors import CollectionNotSupportedError
+from dagster_dataframely.metadata import schema_metadata
+from dagster_dataframely.runtime import AssetYield, process
+
+TransformFn = Callable[..., pl.DataFrame | pl.LazyFrame]
+
+# The union `@dg.asset` accepts, spelled out because `AutomationCondition` is generic and its two parameterizations are not interchangeable.
+AutomationCondition = (
+    dg.AutomationCondition[dg.AssetKey]
+    | dg.AutomationCondition[dg.AssetKey | dg.AssetCheckKey]
+)
+
+#: Runtime-real spelling of Dagster's `CoercibleToAssetDep`, which is typing-only.
+AssetDep = (
+    dg.AssetKey
+    | str
+    | Sequence[str]
+    | dg.AssetSpec
+    | dg.AssetsDefinition
+    | dg.SourceAsset
+    | dg.AssetDep
+)
+
+
+def dataframely_asset(  # noqa: PLR0913 - the forwarded surface is the point
+    *,
+    # --- door-owned ---
+    schema: type[dy.Schema],
+    key_prefix: str | Sequence[str] | None = None,
+    # --- carried to the asset itself, matching @dg.asset's vocabulary ---
+    io_manager_key: str | None = None,
+    metadata: Mapping[str, Any] | None = None,
+    tags: Mapping[str, str] | None = None,
+    owners: Sequence[str] | None = None,
+    kinds: AbstractSet[str] | None = None,
+    automation_condition: AutomationCondition | None = None,
+    freshness_policy: dg.FreshnessPolicy | None = None,
+    # --- forwarded to @dg.multi_asset, verbatim ---
+    name: str | None = None,
+    ins: Mapping[str, dg.AssetIn] | None = None,
+    deps: Iterable[AssetDep] | None = None,
+    description: str | None = None,
+    # Narrower than Dagster's own six-member union, deliberately: a mapping is the
+    # spelling worth a static guarantee, and the rest are legacy.
+    config_schema: Mapping[str, Any] | None = None,
+    required_resource_keys: AbstractSet[str] | None = None,
+    partitions_def: dg.PartitionsDefinition[str] | None = None,
+    hooks: AbstractSet[dg.HookDefinition] | None = None,
+    backfill_policy: dg.BackfillPolicy | None = None,
+    op_tags: Mapping[str, Any] | None = None,
+    resource_defs: Mapping[str, object] | None = None,
+    group_name: str | None = None,
+    retry_policy: dg.RetryPolicy | None = None,
+    code_version: str | None = None,
+    pool: str | None = None,
+) -> Callable[[TransformFn], dg.AssetsDefinition]:
+    """Turns a polars transform into an asset that validates its output against `schema`.
+
+    The contract then lives in exactly one place. From the single declaration, the Columns tab fills in before the asset has ever run, every dataframely rule becomes an asset check with its own pass/fail history, and a frame whose shape does not match the schema aborts the run before a single row is filtered.
+
+    The transform keeps plain polars annotations: nothing rewrites the signature, upstream dependencies bind as ordinary parameters, and the return may be a `DataFrame` or a `LazyFrame`. It takes no `context` parameter; the wrapper reaches the context itself.
+
+    **Every row has to be good.** A run that rejects even one row fails and writes nothing, leaving the last-known-good table in place. There is no lenient mode and no strict flag, deliberately: landing the survivors and dropping the rest is precisely the failure this package exists to make visible, so it is not reachable by configuration. To drop rows anyway, filter in the asset body, where the drop is a line you wrote:
+
+        good, _ = Orders.filter(raw_orders)
+        return good
+
+    Routing rejected rows to a sibling asset instead of failing is planned work, tracked in issue #19.
+
+    Every parameter is declared explicitly with its runtime-real type, so editors autocomplete them and `group_nme="sales"` is a static error rather than an import-time crash. `outs`, `check_specs` and `specs` are surfaces this decorator owns and are simply absent, so they cannot be contested. `can_subset` is absent too: a subset executes but saves nothing.
+
+    `@dg.multi_asset` is the mechanism, but the vocabulary is `@dg.asset`'s, because this decorator is designed for a single table that happens to grow a quarantine sibling. Anything `@dg.asset` lets you say about one asset is sayable here under the same name, and a test asserts that in both directions.
+
+    Args:
+        schema: The dataframely schema the transform's output must satisfy.
+        key_prefix: Prefix for the asset key. The checks follow it automatically.
+        io_manager_key: Resource key the table is stored under. The quarantine inherits it unless its own `dg.AssetOut` names a different one (#19).
+        metadata: Definition metadata to carry alongside the schema's own. `dagster/column_schema` and `dagster_dataframely/schema` are the package's and win a collision.
+        tags: Asset tags, for filtering and grouping in the catalog.
+        owners: Asset owners, as emails or `team:<name>`.
+        kinds: Kind badges shown on the asset in the graph.
+        automation_condition: Declarative automation condition for the asset.
+        freshness_policy: Freshness policy for the asset.
+        name: Asset name. Defaults to the function name.
+        ins: Explicit input mapping, for the cases a parameter name cannot express.
+        deps: Upstream assets this one depends on without loading.
+        description: Asset description. Defaults to the function's docstring.
+        config_schema: Run configuration schema for the underlying op.
+        required_resource_keys: Resources the transform reaches through the context.
+        partitions_def: Partitioning for the asset. The state machine then runs per partition, on that partition's frame.
+        hooks: Hooks to attach to the underlying op.
+        backfill_policy: How Dagster backfills this asset's partitions.
+        op_tags: Tags on the underlying op, for run launcher and executor routing.
+        resource_defs: Resources bound to this asset specifically.
+        group_name: Asset group.
+        retry_policy: Retry policy for the underlying op.
+        code_version: Version string for change-based staleness.
+        pool: Concurrency pool the underlying op runs in.
+
+    Returns:
+        A decorator producing a `multi_asset` with one out and one check per rule.
+
+    Raises:
+        CollectionNotSupportedError: `schema` is a `dy.Collection`.
+
+    Example:
+        >>> import dagster as dg
+        >>> import dataframely as dy
+        >>> import polars as pl
+        >>> import dagster_dataframely as dd
+        >>> class Orders(dy.Schema):
+        ...     order_id = dy.String(primary_key=True)
+        ...     amount = dy.Float64(nullable=False, min=0.0)
+        >>> @dd.dataframely_asset(schema=Orders, group_name="sales")
+        ... def orders(raw_orders: pl.DataFrame) -> pl.DataFrame:
+        ...     return raw_orders.select("order_id", "amount")
+    """
+    # Deliberately narrow: anything else keeps failing however it already fails.
+    if isinstance(schema, type) and issubclass(schema, dy.Collection):
+        raise CollectionNotSupportedError(schema.__name__)
+
+    forwarded: dict[str, Any] = {
+        "ins": ins,
+        "deps": deps,
+        "description": description,
+        "config_schema": config_schema,
+        "required_resource_keys": required_resource_keys,
+        "partitions_def": partitions_def,
+        "hooks": hooks,
+        "backfill_policy": backfill_policy,
+        "op_tags": op_tags,
+        "resource_defs": resource_defs,
+        "group_name": group_name,
+        "retry_policy": retry_policy,
+        "code_version": code_version,
+        "pool": pool,
+    }
+
+    def decorate(fn: TransformFn) -> dg.AssetsDefinition:
+        asset_name = name or fn.__name__
+        prefix: list[str] = []
+        if key_prefix is not None:
+            prefix = [key_prefix] if isinstance(key_prefix, str) else list(key_prefix)
+        key = dg.AssetKey([*prefix, asset_name])
+
+        @dg.multi_asset(
+            name=asset_name,
+            outs={
+                asset_name: dg.AssetOut(
+                    key=key,
+                    # The gate and the abort path both end the step without yielding.
+                    is_required=False,
+                    # The package's two keys are applied last, so a user cannot accidentally displace the Columns tab or the schema carrier.
+                    metadata={**(metadata or {}), **schema_metadata(schema)},
+                    io_manager_key=io_manager_key,
+                    tags=tags,
+                    owners=owners,
+                    kinds=set(kinds) if kinds else None,
+                    automation_condition=automation_condition,
+                    freshness_policy=freshness_policy,
+                )
+            },
+            check_specs=check_specs(schema, asset=key),
+            **forwarded,
+        )
+        @functools.wraps(fn)
+        def compute(*args: object, **kwargs: object) -> AssetYield:
+            # No `context` parameter, deliberately: a user-side postponed-annotations import makes Dagster reject a qualified annotation for one.
+            yield from process(
+                schema,
+                fn(*args, **kwargs),
+                context=dg.AssetExecutionContext.get(),
+                good_out=asset_name,
+            )
+
+        return compute
+
+    return decorate

dagster_dataframely/checks.py

@@ -0,0 +1,94 @@
+"""Asset-check specs derived from a schema, and the results a run reports against them.
+
+Specs come off the schema, never off a run's `FailureInfo`. A rule that rejected nothing still gets a spec and still reports `0 failed`, so a clean run is a row in every rule's history rather than a gap in it.
+"""
+
+import dagster as dg
+import dataframely as dy
+
+from dagster_dataframely.naming import (
+    GATE_CHECK,
+    check_name,
+    rule_description,
+    validate_namespace,
+    validation_rules,
+)
+
+
+def check_specs(
+    schema: type[dy.Schema],
+    *,
+    # Not `dg.CoercibleToAssetKey`: typing-only, so absent at runtime.
+    asset: str | dg.AssetKey,
+) -> list[dg.AssetCheckSpec]:
+    """Builds one check spec per validation rule, plus the schema gate.
+
+    Args:
+        schema: The schema the checks are derived from.
+        asset: The asset key the checks hang off. Build it once and pass the same key to the asset's out, so the two cannot drift.
+
+    Returns:
+        The gate spec first, then one spec per rule in the schema's own order.
+
+    Raises:
+        ReservedColumnError: A user column sits inside the reserved namespace.
+        CheckNameCollisionError: Two rules rewrite to the same check name.
+    """
+    validate_namespace(schema)
+    gate = dg.AssetCheckSpec(
+        GATE_CHECK,
+        asset=asset,
+        description=f"Columns and dtypes match {schema.__name__}.",
+        blocking=True,
+    )
+    return [
+        gate,
+        *(
+            dg.AssetCheckSpec(
+                check_name(rule),
+                asset=asset,
+                # The rendered-constraint rung of the ladder arrives with #20.
+                description=rule_description(schema, rule) or rule,
+            )
+            for rule in validation_rules(schema)
+        ),
+    ]
+
+
+def _rule_results(
+    schema: type[dy.Schema],
+    counts: dict[str, int],
+    *,
+    asset_key: dg.AssetKey,
+    severity: dg.AssetCheckSeverity,
+) -> list[dg.AssetCheckResult]:
+    """Builds one result per rule from `FailureInfo.counts()`.
+
+    Severity is the run's outcome rather than the rule's: when nothing lands, no failure is a warning.
+
+    Args:
+        schema: The schema the results report against.
+        counts: Failure count per rule; rules that rejected nothing are absent.
+        asset_key: The asset the results hang off. Stated explicitly because the abort path yields results on their own, with no materialization to infer it from.
+        severity: Severity for every failing result in this run.
+    """
+    results: list[dg.AssetCheckResult] = []
+    for rule, definition in validation_rules(schema).items():
+        failed: int = counts.get(rule, 0)
+        metadata: dict[str, str | int] = {
+            "dy_rule": rule,
+            # The expression, not the bound: tightening `min` must not rename the check and orphan its history.
+            "dy_rule__expr": str(definition.expr),
+        }
+        if failed:
+            metadata["dy_failed_count"] = failed
+        results.append(
+            dg.AssetCheckResult(
+                check_name=check_name(rule),
+                asset_key=asset_key,
+                passed=not failed,
+                severity=severity,
+                metadata=metadata,
+            )
+        )
+    return results

dagster_dataframely/errors.py

@@ -0,0 +1,154 @@
+"""The package's exception family, all subclassing `DagsterDataframelyError` so they can be caught together.
+
+Every message names the schema, the culprit and the fix, because the message is the whole of what a user sees. It carries no colon: Python already prints `ModuleError: ` ahead of it, and a second colon in the first clause reads as a stutter. Each error takes its culprits as data and builds its own message; none of them knows how the culprits were found.
+"""
+
+from collections.abc import Mapping, Sequence
+
+import polars as pl
+
+
+class DagsterDataframelyError(Exception):
+    """Base for every error this package raises."""
+
+
+class ReservedColumnError(DagsterDataframelyError):
+    """A user column sits inside the reserved `dy_` namespace.
+
+    Raised at definition time. Left to runtime, the collision would surface as a check name that quietly means two different things.
+    """
+
+    def __init__(self, schema_name: str, columns: list[str], prefix: str) -> None:
+        """Names the offending columns only, never the whole schema.
+
+        Args:
+            schema_name: The schema the columns belong to.
+            columns: The column names inside the reserved namespace.
+            prefix: The reserved prefix itself.
+        """
+        culprits = ", ".join(f"'{column}'" for column in columns)
+        plural, verb, pronoun = (
+            ("", "uses", "it") if len(columns) == 1 else ("s", "use", "them")
+        )
+        super().__init__(
+            f"Column{plural} {culprits} of {schema_name} {verb} the reserved "
+            f"'{prefix}' prefix. Rename {pronoun}. This package generates every "
+            f"check name and quarantine column under that namespace."
+        )
+
+
+class CheckNameCollisionError(DagsterDataframelyError):
+    """Two rules rewrite to the same asset-check name.
+
+    Raised at definition time, ahead of Dagster's own `Duplicate check specs`, which names the collision but not the rules that caused it.
+    """
+
+    def __init__(self, schema_name: str, first: str, second: str, name: str) -> None:
+        """Names both culprits and the name they collide on.
+
+        Args:
+            schema_name: The schema both rules belong to.
+            first: The rule seen first.
+            second: The rule that collided with it.
+            name: The asset-check name they both rewrite to.
+        """
+        super().__init__(
+            f"Rules '{first}' and '{second}' of {schema_name} both become "
+            f"asset-check name '{name}' after the '|' -> '__' rewrite. "
+            f"Rename one of them."
+        )
+
+
+class CollectionNotSupportedError(DagsterDataframelyError):
+    """`schema=` received a `dy.Collection`.
+
+    Raised at decoration time. The guard exists because a Collection is real, adjacent, and the most plausible wrong thing a dataframely user reaches for; it is deliberately not generalised into a type check on `schema=`.
+    """
+
+    def __init__(self, collection_name: str) -> None:
+        """States the boundary and makes no promise about a future release.
+
+        Args:
+            collection_name: The collection class that was passed.
+        """
+        super().__init__(
+            f"{collection_name} is a dataframely Collection. This decorator takes a single `dy.Schema`. Declare one asset per member, each with the member's own schema."
+        )
+
+
+class SchemaGateError(DagsterDataframelyError):
+    """A frame arrived with wrong dtypes or missing columns.
+
+    A pipeline defect rather than a data defect, so the whole asset aborts: no rows are filtered and nothing is written.
+    """
+
+    def __init__(self, schema_name: str, problems: Sequence[Mapping[str, str]]) -> None:
+        """Names each offending column with its expected and actual dtype.
+
+        Args:
+            schema_name: The schema the frame failed to match.
+            problems: One mapping of `column`, `expected` and `actual` per offending column.
+        """
+        culprits = ", ".join(
+            f"'{problem['column']}' (expected {problem['expected']}, "
+            f"got {problem['actual']})"
+            for problem in problems
+        )
+        plural, verb = ("", "does") if len(problems) == 1 else ("s", "do")
+        super().__init__(
+            f"Column{plural} {culprits} {verb} not match {schema_name}. "
+            f"Fix the transform, or cast deliberately with `{schema_name}.cast(frame)` "
+            f"in the asset body. This package never casts on your behalf."
+        )
+
+
+class ValidationAbortError(DagsterDataframelyError):
+    """Rows were rejected, so the asset writes nothing.
+
+    Without somewhere to route rejected rows, every row has to be good. Landing the survivors and dropping the rest is the failure this package exists to make visible, so it is not reachable by configuration: a drop is a line the engineer writes in the asset body, the way a cast is.
+    """
+
+    def __init__(
+        self, schema_name: str, rejected: int, counts: Mapping[str, int]
+    ) -> None:
+        """States the damage per rule, and the two fixes that exist today.
+
+        The fix clause will name `quarantine=` once that out exists (#19). Until then it would send the reader to a keyword argument the door does not accept, and an error message is a promise.
+
+        Args:
+            schema_name: The schema that rejected the rows.
+            rejected: How many rows were rejected.
+            counts: Failure count per rule, for the rules that rejected anything. The counts can sum past `rejected`, because one row can break several rules.
+        """
+        culprits = ", ".join(f"{count} by '{rule}'" for rule, count in counts.items())
+        plural = "" if rejected == 1 else "s"
+        super().__init__(
+            f"{schema_name} rejected {rejected} row{plural}, {culprits}. Nothing "
+            f"was written, so the last-known-good table survives. Fix the rows "
+            f"upstream, or drop them deliberately in the asset body. This package "
+            f"never discards rows on your behalf."
+        )
+
+
+class UnwritableDtypeError(DagsterDataframelyError):
+    """A column holds a dtype the bound IO manager cannot write.
+
+    Raised from `handle_output` before the write. Left to polars, the same frame fails with a `ComputeError` from inside the writer, or with a Rust panic.
+    """
+
+    def __init__(self, extension: str, columns: Mapping[str, pl.DataType]) -> None:
+        """Names the culprits and the fix.
+
+        Dagster's wrapping `DagsterExecutionHandleOutputError` already names the step, so naming the asset again here would only repeat it.
+
+        Args:
+            extension: The file extension being written, e.g. `.parquet`.
+            columns: The offending column names, mapped to their dtypes.
+        """
+        culprits = ", ".join(f"'{name}' ({dtype})" for name, dtype in columns.items())
+        plural, pronoun = ("", "it") if len(columns) == 1 else ("s", "them")
+        super().__init__(
+            f"Column{plural} {culprits} cannot be written to {extension}. "
+            f"Convert or drop {pronoun} in the asset body. "
+            f"This IO manager never casts on your behalf."
+        )

dagster_dataframely/io_managers.py

@@ -0,0 +1,137 @@
+"""The package's IO managers.
+
+The asset body owns what the data is; the manager owns where and how it lands. So the manager emits only what varied this run: `path`, `bytes_written` and `dagster/storage_kind`.
+
+Design decisions:
+    - `dagster/column_schema` describes the data, not the write. The asset definition emits it, not the IO manager.
+"""
+
+from typing import TYPE_CHECKING, override
+
+import polars as pl
+from dagster import (
+    ConfigurableIOManagerFactory,
+    DagsterInvariantViolationError,
+    MetadataValue,
+    UPathIOManager,
+)
+from pydantic import Field
+from upath import UPath
+
+from dagster_dataframely.errors import UnwritableDtypeError
+
+if TYPE_CHECKING:
+    from dagster import InitResourceContext, InputContext, OutputContext
+
+_STORAGE_KIND_KEY = "dagster/storage_kind"
+_PARQUET_EXTENSION = ".parquet"
+
+# Parquet's only refusal. Polars cannot nest an `Object`, so scanning top-level dtypes is enough.
+_UNWRITABLE_DTYPES: tuple[pl.DataType | type[pl.DataType], ...] = (pl.Object,)
+
+
+class _ParquetIOManager(UPathIOManager):
+    """Writes polars frames to `.parquet`, built by the `DataframelyParquetIOManager` users bind."""
+
+    extension = _PARQUET_EXTENSION
+
+    def __init__(self, base_dir: str) -> None:
+        """Roots the manager at `base_dir`, a directory or cloud URI."""
+        super().__init__(base_path=UPath(base_dir))
+
+    @override
+    def handle_output(
+        self, context: "OutputContext", obj: pl.DataFrame | pl.LazyFrame
+    ) -> None:
+        """Rejects what parquet cannot represent, then hands the frame to the base manager.
+
+        Here rather than in `dump_to_path` so that a rejection writes nothing and creates no directory, and not at definition time because an asset cannot know which IO manager it will be bound to.
+        """
+        if not isinstance(obj, (pl.DataFrame, pl.LazyFrame)):
+            wrong_type = f"This manager writes polars frames, but the output is a {type(obj).__name__}. Annotate the asset `-> None` if it manages its own storage, so that Dagster skips the IO manager entirely."
+            raise DagsterInvariantViolationError(wrong_type)
+
+        unwritable = {
+            name: dtype
+            for name, dtype in obj.collect_schema().items()
+            if dtype in _UNWRITABLE_DTYPES
+        }
+        if unwritable:
+            raise UnwritableDtypeError(extension=_PARQUET_EXTENSION, columns=unwritable)
+
+        super().handle_output(context, obj)
+
+    @override
+    def dump_to_path(
+        self, context: "OutputContext", obj: pl.DataFrame | pl.LazyFrame, path: UPath
+    ) -> None:
+        """Writes the frame, then stats the path for `bytes_written`.
+
+        `get_metadata` never sees the path it was written to, so the size is taken here. It comes off the disk, so it reports the compression actually achieved rather than an in-memory estimate.
+        """
+        if isinstance(obj, pl.LazyFrame):
+            context.log.warning(
+                "Collecting a LazyFrame before the write. This manager supports polars DataFrame; "
+                "sinking lazily is planned work, tracked in issue #27."
+            )
+        frame = obj.collect() if isinstance(obj, pl.LazyFrame) else obj
+        with path.open("wb") as file:
+            frame.write_parquet(file)
+
+        context.add_output_metadata(
+            {
+                "bytes_written": MetadataValue.int(path.stat().st_size),
+                _STORAGE_KIND_KEY: MetadataValue.text("parquet"),
+            }
+        )
+
+    @override
+    def load_from_path(self, context: "InputContext", path: UPath) -> pl.DataFrame:
+        """Reads the file back eagerly, through the same filesystem the write went out on.
+
+        `context` is unused: parquet is self-describing, so reading a file needs nothing from the asset definition.
+        """
+        with path.open("rb") as file:
+            return pl.read_parquet(file)
+
+
+class DataframelyParquetIOManager(ConfigurableIOManagerFactory[_ParquetIOManager]):
+    """Stores polars frames as `.parquet` files under `base_dir`, locally or in cloud storage.
+
+    `base_dir` is a universal-pathlib path, so `s3://bucket/prefix`, `gs://...` and `az://...` are written the way a local directory is, on credentials from the ambient environment. A cloud scheme needs its fsspec filesystem installed alongside this package: `s3fs`, `gcsfs` or `adlfs`.
+
+    Every materialization carries `path`, `bytes_written` and `dagster/storage_kind`, and nothing else: no column schema, no data sample, no statistics pass. A dtype that parquet cannot represent raises `UnwritableDtypeError` before the write.
+
+    Polars `DataFrame` is the supported type. A `LazyFrame` output is collected before the write, with a warning in the run log, and a read always returns a `DataFrame`. Sinking and scanning lazily is planned work, tracked in issue #27.
+
+    Attributes:
+        base_dir: Directory or cloud URI the manager writes parquet files under.
+
+    Example:
+        >>> import dagster as dg
+        >>> import polars as pl
+        >>> import dagster_dataframely as dd
+        >>> @dg.asset
+        ... def orders() -> pl.DataFrame:
+        ...     return pl.DataFrame({"order_id": ["a", "b"]})
+        >>> defs = dg.Definitions(
+        ...     assets=[orders],
+        ...     resources={
+        ...         "io_manager": dd.DataframelyParquetIOManager(
+        ...             base_dir="s3://my-bucket/warehouse"
+        ...         )
+        ...     },
+        ... )
+    """
+
+    base_dir: str = Field(
+        description="Directory or cloud URI the manager writes parquet files under."
+    )
+
+    @override
+    def create_io_manager(self, context: "InitResourceContext") -> _ParquetIOManager:
+        """Builds the manager that does the writing.
+
+        `context` is unused: `base_dir` is the only source of the base path, so the `path` in the event log is always the one the user configured.
+        """
+        return _ParquetIOManager(base_dir=self.base_dir)

dagster_dataframely/metadata.py

@@ -0,0 +1,84 @@
+"""What the asset definition declares about its data, before it has ever run.
+
+The seam: the asset body owns what the data is, the IO manager owns where and how it landed. A schema is what the data is, so it lives here and the IO manager never emits it.
+"""
+
+import dagster as dg
+import dataframely as dy
+
+# `@public` upstream, but absent from `dagster` and with no `MetadataValue.object()` factory. Pinned by its own test (#16).
+from dagster._core.definitions.metadata.metadata_value import (
+    ObjectMetadataValue,
+)
+
+_COLUMN_SCHEMA_KEY = "dagster/column_schema"
+SCHEMA_CARRIER_KEY = "dagster_dataframely/schema"
+
+
+def _tags(column: dy.Column) -> dict[str, str] | None:
+    """Renders a column's free-form metadata as Dagster tags.
+
+    Values are stringified because `TableColumn.tags` is `Mapping[str, str]` and Dagster rejects anything else at definition time. That is a display rendering, not a cast: no data is touched, and refusing instead would mean a `metadata={"pii": False}` dataframely explicitly permits could not be attached to an asset at all.
+    """
+    if not column.metadata:
+        return None
+    return {key: str(value) for key, value in column.metadata.items()}
+
+
+def table_schema(schema: type[dy.Schema]) -> dg.TableSchema:
+    """Projects a schema onto Dagster's Columns tab.
+
+    Dtype, description, nullability, uniqueness and tags. Constraint pills and the table-level primary key arrive with the renderer (#20).
+
+    `unique` is read from the column's own flag and never derived from `primary_key`. dataframely keeps the two independent: a key member gets a composite `as_struct(...).is_unique()` rule and `column.unique` stays `False`, so deriving would claim a per-column uniqueness that nothing enforces.
+
+    Tags come from `Column.metadata`, which dataframely stores and never reads. It is the one dataframely attribute with no other home here, and free-form key/value annotation is exactly what Dagster's column tags are for.
+
+    Args:
+        schema: The schema to project.
+
+    Returns:
+        A table schema whose columns are in the schema's own order.
+    """
+    return dg.TableSchema(
+        columns=[
+            dg.TableColumn(
+                name=name,
+                type=str(column.dtype),
+                description=column.description,
+                constraints=dg.TableColumnConstraints(
+                    nullable=column.nullable, unique=column.unique
+                ),
+                tags=_tags(column),
+            )
+            for name, column in schema.columns().items()
+        ]
+    )
+
+
+def schema_metadata(
+    schema: type[dy.Schema],
+) -> dict[str, dg.TableSchema | ObjectMetadataValue]:
+    """Builds the definition metadata a schema-backed asset declares.
+
+    Two entries: the Columns tab, and the carrier that takes the live schema class to the IO manager on both the write and the read path. The carrier's label is passed explicitly because deriving it would yield the metaclass name, `SchemaMeta`.
+
+    Args:
+        schema: The schema being attached to the asset.
+
+    Returns:
+        A mapping to hand to `dg.AssetOut(metadata=...)`.
+
+    Example:
+        >>> import dagster as dg
+        >>> import dataframely as dy
+        >>> import dagster_dataframely as dd
+        >>> class Orders(dy.Schema):
+        ...     order_id = dy.String(primary_key=True)
+        >>> out = dg.AssetOut(metadata=dd.schema_metadata(Orders))
+    """
+    return {
+        _COLUMN_SCHEMA_KEY: table_schema(schema),
+        # A raw class here is deprecated upstream, so the carrier is explicit.
+        SCHEMA_CARRIER_KEY: ObjectMetadataValue(schema.__name__, instance=schema),
+    }

dagster_dataframely/naming.py

@@ -0,0 +1,97 @@
+"""The reserved namespace, the rule-name rewrite, and the two definition-time collision errors.
+
+`dy_` is hardcoded rather than configurable. A reserved namespace is not a preference; its whole value is being the same string in every project, so a knob would only let one project make its check names unrecognisable to the next.
+"""
+
+import inspect
+
+import dataframely as dy
+
+# Neither has a public equivalent. Pinned by their own tests (#16).
+from dataframely._rule import Rule, RuleFactory
+
+from dagster_dataframely.errors import CheckNameCollisionError, ReservedColumnError
+
+# Spelled out again wherever a name is built, never interpolated: one grep for `dy_rule__` finds every producer and consumer.
+RESERVED_PREFIX = "dy_"
+
+#: The gate check. Present at every granularity, always blocking.
+GATE_CHECK = "dy_schema__dtypes"
+
+
+def check_name(rule_name: str) -> str:
+    """Rewrites a dataframely rule name into an asset-check name.
+
+    `amount|min` becomes `dy_rule__amount__min`, the same string wherever the rule shows up.
+
+    The rewrite is forced, not chosen: every check spec becomes an op output named `<asset>_<check>`, and Dagster validates that against `^[A-Za-z0-9_]+$`, which `|` fails.
+
+    Args:
+        rule_name: The rule name dataframely reports, `|`-delimited for column rules.
+
+    Returns:
+        The asset-check name, inside the reserved namespace.
+
+    Example:
+        >>> check_name("amount|min")
+        'dy_rule__amount__min'
+        >>> check_name("paid_orders_have_amount")
+        'dy_rule__paid_orders_have_amount'
+    """
+    return f"dy_rule__{rule_name.replace('|', '__')}"
+
+
+def validation_rules(schema: type[dy.Schema]) -> dict[str, Rule]:
+    """Returns the schema's validation rules, keyed by rule name.
+
+    `with_cast=False` drops the `<column>|dtype` pseudo-rules, which would otherwise duplicate the gate at a different severity and without blocking.
+
+    Args:
+        schema: The schema to read rules from.
+
+    Returns:
+        Each rule keyed by the name dataframely gives it, `|`-delimited for column rules.
+    """
+    return schema._validation_rules(with_cast=False)  # noqa: SLF001
+
+
+def rule_description(schema: type[dy.Schema], rule_name: str) -> str | None:
+    """Returns a rule's docstring, or `None` for a rule that has no place to carry one.
+
+    A `@dy.rule()` leaves its `RuleFactory` on the class, so the decorated function and its docstring stay reachable by name long after the metaclass has built the `Rule`. Column rules are generated from column arguments and have no function at all, and their `|` means the lookup misses rather than needing a branch.
+
+    Args:
+        schema: The schema the rule belongs to.
+        rule_name: The rule name dataframely reports.
+
+    Returns:
+        The rule's docstring, dedented, or `None` if it has none.
+    """
+    factory = getattr(schema, rule_name, None)
+    if not isinstance(factory, RuleFactory):
+        return None
+    return inspect.getdoc(factory.validation_fn)
+
+
+def validate_namespace(schema: type[dy.Schema]) -> None:
+    """Raises the two errors Dagster would otherwise report opaquely, or not at all.
+
+    Args:
+        schema: The schema whose columns and rule names are being claimed.
+
+    Raises:
+        ReservedColumnError: A user column sits inside the reserved namespace.
+        CheckNameCollisionError: Two rules rewrite to the same asset-check name.
+    """
+    reserved: list[str] = [
+        column for column in schema.columns() if column.startswith(RESERVED_PREFIX)
+    ]
+    if reserved:
+        raise ReservedColumnError(schema.__name__, reserved, RESERVED_PREFIX)
+
+    seen: dict[str, str] = {}
+    for rule in validation_rules(schema):
+        name: str = check_name(rule)
+        if name in seen:
+            raise CheckNameCollisionError(schema.__name__, seen[name], rule, name)
+        seen[name] = rule

dagster_dataframely/runtime.py

@@ -0,0 +1,153 @@
+"""The state machine a schema-backed asset runs: gate, filter, then one of three outcomes.
+
+The asset's declared shape is the failure policy. There is no lenient/strict flag anywhere, so the failure behaviour is visible in the definition rather than in an argument's value, and it cannot disagree with what the asset actually declares.
+
+The two outcomes that need somewhere to put rejected rows arrive with the quarantine (#19).
+"""
+
+from collections.abc import Iterator
+
+import dagster as dg
+import dataframely as dy
+import polars as pl
+
+from dagster_dataframely.checks import _rule_results
+from dagster_dataframely.errors import SchemaGateError, ValidationAbortError
+from dagster_dataframely.naming import GATE_CHECK
+
+AssetYield = Iterator[dg.MaterializeResult[pl.DataFrame] | dg.AssetCheckResult]
+
+
+def _require_frame(frame: object, out_name: str) -> None:
+    """Rejects a transform output the gate cannot read.
+
+    The parameter's annotation is a promise Dagster cannot enforce, because it calls the transform dynamically. Left alone, a forgotten return annotation surfaces two frames down as `'NoneType' object has no attribute 'collect_schema'`.
+
+    Dagster's own error rather than the package's: this is a wiring mistake, not a data one, which is the line `_ParquetIOManager` already draws.
+    """
+    if isinstance(frame, (pl.DataFrame, pl.LazyFrame)):
+        return
+    wrong_type = (
+        f"'{out_name}' returned a {type(frame).__name__}. A schema-backed asset must "
+        f"return a polars DataFrame or LazyFrame, because the gate reads its columns "
+        f"and dtypes before anything is written. An asset that manages its own storage "
+        f"has no schema to validate, so write it as a plain `@dg.asset`."
+    )
+    raise dg.DagsterInvariantViolationError(wrong_type)
+
+
+def _gate_problems(
+    schema: type[dy.Schema], frame: pl.DataFrame | pl.LazyFrame
+) -> list[dict[str, str]]:
+    """Compares the frame's shape against the schema, naming every mismatch.
+
+    An explicit pre-check rather than a `try`/`except` around `filter`, which would behave differently depending on what the transform returned: `filter(cast=False)` raises at call time on a `DataFrame`, but on a `LazyFrame` it returns cleanly and the same error surfaces only on the eventual collect. The door promises either return type works, so the gate cannot be built on a difference between them.
+
+    Only public API, and none of it executes: `collect_schema()` resolves a `LazyFrame`'s shape without running it.
+
+    Returns:
+        One mapping of `column`, `expected` and `actual` per offending column, empty when the frame matches. The same list feeds the failing check's metadata and `SchemaGateError`, so the two cannot disagree.
+    """
+    actual = frame.collect_schema()
+    return [
+        {
+            "column": name,
+            "expected": str(column.dtype),
+            "actual": str(actual[name]) if name in actual else "<missing>",
+        }
+        for name, column in schema.columns().items()
+        if name not in actual or not column.validate_dtype(actual[name])
+    ]
+
+
+def _gate_failure(
+    problems: list[dict[str, str]], *, asset_key: dg.AssetKey
+) -> dg.AssetCheckResult:
+    """Builds the failing gate check, tabulating every offending column."""
+    return dg.AssetCheckResult(
+        check_name=GATE_CHECK,
+        asset_key=asset_key,
+        passed=False,
+        severity=dg.AssetCheckSeverity.ERROR,
+        metadata={
+            "dy_schema__errors": dg.MetadataValue.table(
+                [dg.TableRecord(problem) for problem in problems]
+            )
+        },
+    )
+
+
+def _check_results(
+    schema: type[dy.Schema], failure: dy.FailureInfo, *, asset_key: dg.AssetKey
+) -> list[dg.AssetCheckResult]:
+    """Builds every check result for a run that made it past the gate.
+
+    Severity is derived here, once, from whether the run rejected anything. That is what makes it a property of the run's outcome rather than of any one rule: no code path can hand two sibling checks different severities.
+    """
+    severity = (
+        dg.AssetCheckSeverity.ERROR if len(failure) else dg.AssetCheckSeverity.WARN
+    )
+    return [
+        dg.AssetCheckResult(check_name=GATE_CHECK, asset_key=asset_key, passed=True),
+        *_rule_results(
+            schema, failure.counts(), asset_key=asset_key, severity=severity
+        ),
+    ]
+
+
+def process(
+    schema: type[dy.Schema],
+    frame: pl.DataFrame | pl.LazyFrame,
+    *,
+    context: dg.AssetExecutionContext,
+    good_out: str,
+) -> AssetYield:
+    """Validates a transform's output and reports it to Dagster.
+
+    Two stages and three exits. The gate runs first, so a wrong-shaped frame never pays to be filtered. Then `Schema.filter(frame, cast=False)` splits the rows: it is the only validation call, because `validate()` carries per-rule detail as a string and this package needs structured counts.
+
+    Args:
+        schema: The schema the frame must satisfy.
+        frame: Whatever the transform returned.
+        context: The executing asset's context, for resolving the out to its key.
+        good_out: The output name the validated frame materializes under.
+
+    Yields:
+        A `MaterializeResult` carrying the good frame and every check result, or, on either failure path, the check results on their own.
+
+    Raises:
+        DagsterInvariantViolationError: The transform returned something that is not a polars frame.
+        SchemaGateError: The frame's columns or dtypes do not match the schema.
+        ValidationAbortError: Rows were rejected and no quarantine is declared.
+    """
+    _require_frame(frame, good_out)
+    good_key = context.asset_key_for_output(good_out)
+
+    # --- Stage 1: the schema gate ---
+    problems = _gate_problems(schema, frame)
+    if problems:
+        # Exit: pipeline defect. Nothing is filtered and nothing is written, so a wrong-shaped frame cannot corrupt the table.
+        yield _gate_failure(problems, asset_key=good_key)
+        raise SchemaGateError(schema.__name__, problems)
+
+    # --- Stage 2: the row filter ---
+    # Eager, not lazy: `filter` already collected, and `row_count` needs the length.
+    result, failure = schema.filter(frame, cast=False)
+    good = result.collect() if isinstance(result, pl.LazyFrame) else result
+    checks = _check_results(schema, failure, asset_key=good_key)
+    rejected = len(failure)
+
+    if rejected:
+        # Exit: data defect with no quarantine declared, so consent to partial data was never given.
+        # Both halves are discarded and the last-known-good table survives, but every rule still reports, so the failed run says what failed and by how much.
+        # The two quarantine exits land here with #19.
+        yield from checks
+        raise ValidationAbortError(schema.__name__, rejected, failure.counts())
+
+    # Exit: everything survived. The only path that materializes.
+    yield dg.MaterializeResult(
+        asset_key=good_key,
+        value=good,
+        metadata={"dagster/row_count": len(good)},
+        check_results=checks,
+    )

dagster_dataframely-0.0.1.dist-info/METADATA

@@ -0,0 +1,76 @@
+Metadata-Version: 2.4
+Name: dagster-dataframely
+Version: 0.0.1
+Summary: A Dataframely plugin for Dagster.
+Keywords: dagster,data,data-quality,dataframely,polars,validation
+Author: Ozan Ozbeker
+Author-email: Ozan Ozbeker <github@ozanozbeker.com>
+License-Expression: Apache-2.0
+License-File: LICENSE
+Classifier: Development Status :: 1 - Planning
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Database
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Typing :: Typed
+Requires-Dist: dagster>=1.13.16
+Requires-Dist: dataframely>=3.0.0
+Requires-Dist: polars>=1.43.2
+Requires-Dist: pydantic>=2
+Requires-Dist: universal-pathlib>=0.2.0
+Maintainer: Ozan Ozbeker
+Maintainer-email: Ozan Ozbeker <github@ozanozbeker.com>
+Requires-Python: >=3.12
+Project-URL: Homepage, https://github.com/ozanozbeker/dagster-dataframely
+Project-URL: Repository, https://github.com/ozanozbeker/dagster-dataframely
+Project-URL: Issues, https://github.com/ozanozbeker/dagster-dataframely/issues
+Project-URL: Changelog, https://github.com/ozanozbeker/dagster-dataframely/releases
+Description-Content-Type: text/markdown
+
+# dagster-dataframely
+
+Dataframely plugin for Dagster.
+
+> [!WARNING]
+> **Pre-release placeholder.**
+> This package is under active design and ships no functionality yet.
+> The name is reserved on PyPI while the design spec is finalized.
+> Do not depend on it.
+> Follow [issue #1](https://github.com/ozanozbeker/dagster-dataframely/issues/1) for progress.
+
+## What it will do
+
+[dataframely](https://github.com/Quantco/dataframely) declares schemas and validation rules for [polars](https://pola.rs) data frames.
+[Dagster](https://dagster.io) has first-class surfaces for data contracts: column schema metadata and asset checks.
+This package connects the two:
+
+- **Asset checks derived from your schema.**
+  Each dataframely rule becomes a Dagster asset check, so per-rule pass/fail history shows up in the catalog without hand-writing checks.
+- **Column schema metadata.**
+  Your schema populates Dagster's Columns tab automatically.
+- **Quarantine, opt-in.**
+  Declare a second output and failing rows are routed there with per-rule attribution instead of failing the run.
+- **Storage in the box.**
+  `DataframelyParquetIOManager` writes `.parquet` to a local directory or to `s3://`, `gs://` and `az://`, and it is the supported path.
+
+Dependencies are `dagster` and `dataframely` (plus `polars`) only.
+The IO manager imports `universal-pathlib` and `pydantic` directly, so both are declared too.
+Both already ship with `dagster`, so nothing new lands in your environment.
+
+## Installation
+
+```bash
+uv add dagster-dataframely
+```
+
+Requires Python 3.12+.
+
+## License
+
+[Apache-2.0](LICENSE)

dagster_dataframely-0.0.1.dist-info/RECORD

@@ -0,0 +1,13 @@
+dagster_dataframely/__init__.py,sha256=WLz0W4e18yJUmWE5jPrM50mKYK-U58dVTkMClIqVSxI,953
+dagster_dataframely/asset.py,sha256=MRZi1W1Zg0sc257kJ_C5hlmQKzvSK-quoldTnf4jBsI,10181
+dagster_dataframely/checks.py,sha256=VQi52wG0tY8V4pXdBM8yB8MP2m3wWnKTb7fRmmBUqE0,3255
+dagster_dataframely/errors.py,sha256=nkxWcf47OmEZPmh8AhZLgoNkdKcWIm4tBQCiyB0e_xM,7134
+dagster_dataframely/io_managers.py,sha256=U7Ct4A7ukpvnrGbUFojCsCHP6Jcfevja84gFgGfnbzM,5953
+dagster_dataframely/metadata.py,sha256=8uknv9GoIeGuM70TPW4oPEhONNWdPRkU_T3TDqHOrPI,3645
+dagster_dataframely/naming.py,sha256=HRZrdP64CBhxVYH_GwKbaGkDUq9_uQpkn2_1gD8U0CU,3852
+dagster_dataframely/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dagster_dataframely/runtime.py,sha256=3P2DSwngDmiPWrvsdR2Kj8GYLnRdFvEgCgb3nUhROs0,7044
+dagster_dataframely-0.0.1.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+dagster_dataframely-0.0.1.dist-info/WHEEL,sha256=EmLkUISDECbcUx3FMCYOqokNOJqNp2r0d4mJzjErvvs,80
+dagster_dataframely-0.0.1.dist-info/METADATA,sha256=JIrCBBrHv-TKypYnlHlh9e2zDuBenLQMbOlCYNsAEcU,3098
+dagster_dataframely-0.0.1.dist-info/RECORD,,

dagster_dataframely-0.0.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: uv 0.12.3
+Root-Is-Purelib: true
+Tag: py3-none-any

dagster_dataframely-0.0.1.dist-info/licenses/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.