datahub-bq-connector-session-patch 0.1.0__py3-none-any.whl

datahub_bq_connector_session_patch/__init__.py

@@ -0,0 +1,36 @@
+"""Recover unqualified BigQuery reads that DataHub silently discards.
+
+Full diagnosis, design and verification live in the DataVantage docs repo:
+``docs/datahub-unqualified-read-attribution.md``.
+
+Recipes use the ``datahub_bq_connector_session_patch`` source type, registered by this package's
+entry point. ``install()`` and ``install_on_aggregator()`` are for standalone scripts
+and tests that drive a pipeline or an aggregator directly.
+"""
+
+from datahub_bq_connector_session_patch.patch import armed, check_datahub_version, install, is_armed
+from datahub_bq_connector_session_patch.resolver import (
+    SessionPatchStats,
+    build_index,
+    install_on_aggregator,
+)
+
+__all__ = [
+    "SessionPatchStats",
+    "armed",
+    "build_index",
+    "check_datahub_version",
+    "install",
+    "install_on_aggregator",
+    "is_armed",
+]
+
+
+def __getattr__(name: str):
+    # Imported lazily: it pulls in datahub's bigquery extras, which a test or a script
+    # using only the resolver need not have.
+    if name == "BigQuerySessionPatchSource":
+        from datahub_bq_connector_session_patch.source import BigQuerySessionPatchSource
+
+        return BigQuerySessionPatchSource
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

datahub_bq_connector_session_patch/patch.py

@@ -0,0 +1,201 @@
+"""Compose the resolver onto ``BigQueryQueriesExtractor``.
+
+The extractor is constructed deep inside ``BigqueryV2Source.get_workunits_internal``,
+so there is no injection point -- the composition has to happen on the class. To keep a
+stock ``type: bigquery`` recipe stock even when this package is installed in the same
+interpreter, the patched ``__init__`` is inert unless it is explicitly armed. The
+``datahub_bq_connector_session_patch`` source arms it around its own run and nothing else.
+"""
+
+import contextlib
+import contextvars
+import inspect
+import logging
+from typing import Any, Collection, Iterator, Optional, Tuple
+
+from datahub_bq_connector_session_patch.resolver import SessionPatchStats, install_on_aggregator
+
+logger = logging.getLogger(__name__)
+
+#: Versions of acryl-datahub this patch has been tested against. It reaches into private
+#: internals (``BigQueryQueriesExtractor.__init__``, ``SqlParsingAggregator``), so a
+#: version outside this range is reported rather than trusted. Deliberately NOT a
+#: packaging dependency: this wheel installs into a DataHub executor's own interpreter
+#: beside its pinned acryl-datahub, and declaring it would let pip re-resolve that pin.
+MIN_ACRYL_DATAHUB = (1, 6)
+MAX_ACRYL_DATAHUB_EXCLUSIVE = (1, 8)
+
+_ARMED: contextvars.ContextVar = contextvars.ContextVar("datahub_bq_connector_session_patch_armed", default=False)
+
+
+@contextlib.contextmanager
+def armed() -> Iterator[None]:
+    """Arm the patch for the duration of the block."""
+    token = _ARMED.set(True)
+    try:
+        yield
+    finally:
+        _ARMED.reset(token)
+
+
+def is_armed() -> bool:
+    return _ARMED.get()
+
+
+def _version_tuple(raw: str) -> tuple:
+    parts = []
+    for chunk in raw.split(".")[:3]:
+        digits = "".join(c for c in chunk if c.isdigit())
+        if not digits:
+            break
+        parts.append(int(digits))
+    return tuple(parts)
+
+
+def check_datahub_version(report: Optional[Any] = None) -> Optional[str]:
+    """Warn if the installed acryl-datahub is outside the tested range.
+
+    Returns the message when out of range, else None. Never raises: a version check
+    failing closed would take down an ingestion run over its own metadata lookup.
+    """
+    try:
+        from importlib.metadata import version
+
+        raw = version("acryl-datahub")
+    except Exception:
+        return None
+
+    found = _version_tuple(raw)
+    if not found:
+        return None
+    if MIN_ACRYL_DATAHUB <= found[: len(MIN_ACRYL_DATAHUB)] and found < MAX_ACRYL_DATAHUB_EXCLUSIVE:
+        return None
+
+    lo = ".".join(str(n) for n in MIN_ACRYL_DATAHUB)
+    hi = ".".join(str(n) for n in MAX_ACRYL_DATAHUB_EXCLUSIVE)
+    message = (
+        f"datahub-bq-connector-session-patch is tested against acryl-datahub >={lo},<{hi}; found {raw}. "
+        "It patches private internals -- re-run the test suite before trusting this run."
+    )
+    if report is not None:
+        report.warning("datahub-bq-connector-session-patch: untested acryl-datahub version", context=message)
+    else:
+        logger.warning(message)
+    return message
+
+
+def describe_outcome(stats) -> Tuple[str, str, str]:
+    """(level, title, context) for the end-of-run report. Pure, so it can be tested.
+
+    The index is built lazily on the first classified upstream, so a run that saw no
+    queries at all leaves ``index_size`` at 0 as well. Reporting that as an empty
+    ``discovered_tables`` points the operator at the wrong subsystem.
+    """
+    queries_seen = stats.queries_passed + stats.queries_skipped + stats.queries_mixed
+    if queries_seen == 0:
+        return (
+            "warning",
+            "datahub-bq-connector-session-patch saw NO queries",
+            "the query log yielded nothing, so the resolution index was never built -- "
+            "check the audit window, the user filters and the region qualifiers, not "
+            "discovered_tables",
+        )
+    if stats.index_size == 0:
+        return (
+            "warning",
+            "datahub-bq-connector-session-patch built an EMPTY resolution index",
+            f"{queries_seen} queries seen but discovered_tables was empty -- "
+            "the patch recovered nothing",
+        )
+    if stats.rewritten == 0:
+        return (
+            "warning",
+            "datahub-bq-connector-session-patch rewrote nothing",
+            f"index={stats.index_size} names but no _SESSION upstreams matched",
+        )
+    return (
+        "info",
+        "datahub-bq-connector-session-patch recovered unqualified reads",
+        str(stats),
+    )
+
+
+def _live_discovered_tables(extractor, original_init, args: tuple, kwargs: dict) -> Collection[str]:
+    """The LIVE collection the source passed in, not ``__init__``'s snapshot.
+
+    ``BigqueryV2Source`` passes ``discovered_tables=self.bq_schema_extractor.table_refs``
+    -- a set that is still empty at construction and fills during the schema pass. The
+    extractor copies it into ``self.discovered_tables`` immediately, so reading that
+    attribute yields an empty index and a silently useless run.
+
+    Bound through the real signature rather than a hand-counted argument index, so a
+    positional call site or an upstream signature change cannot silently pick the wrong
+    parameter -- the failure mode this whole patch exists to avoid.
+    """
+    live = None
+    try:
+        bound = inspect.signature(original_init).bind(extractor, *args, **kwargs)
+        live = bound.arguments.get("discovered_tables")
+    except TypeError:
+        live = kwargs.get("discovered_tables")
+    if live is None:
+        live = extractor.discovered_tables or ()
+    return live
+
+
+def install(always_on: bool = False) -> None:
+    """Patch ``BigQueryQueriesExtractor`` so armed runs get the fix. Idempotent.
+
+    ``always_on`` arms the patch process-wide -- for standalone scripts that drive a
+    pipeline directly. Recipes should use the ``datahub_bq_connector_session_patch`` source instead,
+    which arms only its own run.
+    """
+    from datahub.ingestion.source.bigquery_v2 import queries_extractor as qe
+
+    if always_on:
+        _ARMED.set(True)
+
+    if getattr(qe.BigQueryQueriesExtractor, "_datahub_bq_connector_session_patch_installed", False):
+        return
+    original_init = qe.BigQueryQueriesExtractor.__init__
+
+    def __init__(self, *args, **kwargs):
+        original_init(self, *args, **kwargs)
+        if not is_armed():
+            return
+
+        check_datahub_version(self.structured_report)
+        live = _live_discovered_tables(self, original_init, args, kwargs)
+
+        def discovered() -> Collection[str]:
+            standardize = self.identifiers.standardize_identifier_case
+            return {standardize(name) for name in (live or ())}
+
+        stats: SessionPatchStats = install_on_aggregator(self.aggregator, discovered)
+        self.datahub_bq_connector_session_patch_stats = stats
+        logger.info("datahub-bq-connector-session-patch: armed, index builds on first query")
+
+        original_close = self.close
+
+        def close() -> None:
+            # Both historical failures of this patch were silent: the run completed and
+            # rewrote nothing. Make that loud instead -- but name the right subsystem.
+            try:
+                level, title, context = describe_outcome(stats)
+                getattr(self.structured_report, level)(title, context=context)
+                if level == "info":
+                    logger.info("datahub-bq-connector-session-patch: %s", stats)
+                    logger.info(
+                        "datahub-bq-connector-session-patch: per reader project\n%s",
+                        stats.per_project_table(),
+                    )
+            finally:
+                # The aggregator holds a file-backed sqlite connection. A failure while
+                # REPORTING must never be the reason it is left open -- least of all in
+                # a function whose whole job is making silent failures loud.
+                original_close()
+
+        self.close = close
+
+    qe.BigQueryQueriesExtractor.__init__ = __init__
+    qe.BigQueryQueriesExtractor._datahub_bq_connector_session_patch_installed = True

datahub_bq_connector_session_patch/resolver.py

@@ -0,0 +1,298 @@
+"""Resolve the ``_SESSION`` upstreams DataHub builds for unqualified BigQuery reads.
+
+``BigQueryQueriesExtractor._parse_audit_log_row`` sets ``default_schema="_SESSION"``
+(queries_extractor.py:542,558), so any table name a query does not qualify resolves to
+``{billing_project}._SESSION.{table}``. ``is_temp_table()`` then flags it, because the
+dataset starts with ``temp_table_dataset_prefix`` (default ``"_"``), and the aggregator
+drops it -- for usage (sql_parsing_aggregator.py:1022) and for the query entity (:1693).
+The net effect is that every unqualified read in the estate produces no usage, no query
+entity and no lineage, even though the parser saw the table name.
+
+This rewrites those upstreams back to the real object by matching the bare table name
+against the tables DataHub discovered during its schema pass. A name matching more than
+one discovered table is left untouched -- dropped exactly as before, never guessed.
+"""
+
+import dataclasses
+import logging
+import re
+from collections import defaultdict
+from typing import Callable, Collection, Dict, Optional, Set, Tuple, Union
+
+logger = logging.getLogger(__name__)
+
+_SESSION = "_session"
+
+#: Outcome of classifying one upstream URN. Returned alongside the resolved URN so the
+#: caller can count per-reference volume without re-running (and re-counting) the lookup.
+PASSTHROUGH = "passthrough"
+REWRITTEN = "rewritten"
+AMBIGUOUS = "ambiguous"
+UNKNOWN = "unknown"
+
+
+@dataclasses.dataclass
+class SessionPatchStats:
+    """Distinct-table counters, plus the per-reference volume behind each.
+
+    The distinct counters answer "how much of the catalog did we recover"; the
+    ``*_refs`` counters answer "how many reads did that attribute". Both are needed to
+    judge a run: a single hot table read 2,000 times is one rewrite and 2,000 references.
+    """
+
+    # Distinct. NOT distinct _SESSION URNs: those embed the reader (billing) project,
+    # so one table read from five projects would count five times. `rewritten` counts
+    # distinct resolved TARGET tables; `ambiguous`/`unknown` count distinct bare NAMES.
+    rewritten: int = 0
+    ambiguous: int = 0
+    unknown: int = 0
+    # Volume: upstream references seen across all queries. Counted only over
+    # ``parsed.upstreams``; the column_usage remap reuses the memoised result, so it
+    # can never inflate these.
+    rewritten_refs: int = 0
+    ambiguous_refs: int = 0
+    unknown_refs: int = 0
+    queries_passed: int = 0
+    queries_skipped: int = 0
+    queries_mixed: int = 0
+    index_size: int = 0
+    ambiguous_names: int = 0
+    #: reader (billing) project -> {"rewritten"|"ambiguous"|"unknown": reference count}.
+    #: Keyed by the project that RAN the query, not the one the table lives in: when the
+    #: sidecar's scope is widened, the question being asked is whose traffic benefits.
+    refs_by_reader_project: Dict[str, Dict[str, int]] = dataclasses.field(default_factory=dict)
+
+    def record_ref(self, project: Optional[str], outcome: str) -> None:
+        if project is None:
+            return
+        self.refs_by_reader_project.setdefault(project, {}).setdefault(outcome, 0)
+        self.refs_by_reader_project[project][outcome] += 1
+
+    def per_project_table(self) -> str:
+        """One line per reader project, worst recovery first. For run reports."""
+        if not self.refs_by_reader_project:
+            return "  (no _SESSION references seen)"
+        rows = []
+        for project, counts in self.refs_by_reader_project.items():
+            got = counts.get(REWRITTEN, 0)
+            amb = counts.get(AMBIGUOUS, 0)
+            unk = counts.get(UNKNOWN, 0)
+            total = got + amb + unk
+            pct = (100.0 * got / total) if total else 0.0
+            rows.append((pct, project, got, amb, unk, total))
+        rows.sort()
+        return "\n".join(
+            f"  {project:<34} recovered={got:<7} ambiguous={amb:<7} unknown={unk:<7} "
+            f"of {total:<7} ({pct:.1f}%)"
+            for pct, project, got, amb, unk, total in rows
+        )
+
+    def __str__(self) -> str:
+        return (
+            f"rewritten={self.rewritten} tables ({self.rewritten_refs} refs) "
+            f"ambiguous={self.ambiguous} names ({self.ambiguous_refs} refs) "
+            f"unknown={self.unknown} names ({self.unknown_refs} refs) "
+            f"queries_passed={self.queries_passed} queries_skipped={self.queries_skipped} "
+            f"queries_mixed={self.queries_mixed} "
+            f"(index={self.index_size} unique names, {self.ambiguous_names} ambiguous names)"
+        )
+
+
+_TABLE_REF_RE = re.compile(r"^projects/([^/]+)/datasets/([^/]+)/tables/(.+)$")
+
+
+def _normalise(ref: str) -> Optional[str]:
+    """Discovered tables arrive as ``projects/P/datasets/D/tables/T``; URNs use P.D.T."""
+    match = _TABLE_REF_RE.match(ref)
+    if match:
+        return ".".join(match.groups())
+    parts = ref.split(".")
+    return ".".join(parts[-3:]) if len(parts) >= 3 else None
+
+
+def build_index(discovered: Collection[str]) -> Tuple[Dict[str, str], Set[str]]:
+    """bare table name -> full name, only where the bare name is unambiguous.
+
+    Names matching several discovered tables go to the ambiguous set and are never
+    resolved. Narrowing the recipe's scope *removes* alternatives rather than resolving
+    between them, so the index must be built over a scope broad enough that real
+    collisions stay visible -- see the design doc, section 9.
+    """
+    candidates: Dict[str, Set[str]] = defaultdict(set)
+    for ref in discovered:
+        full = _normalise(ref)
+        if full:
+            candidates[full.split(".")[-1].lower()].add(full)
+    index = {name: next(iter(v)) for name, v in candidates.items() if len(v) == 1}
+    ambiguous = {name for name, v in candidates.items() if len(v) > 1}
+    return index, ambiguous
+
+
+def _split_urn(urn: str) -> Optional[Tuple[str, str, str]]:
+    try:
+        inner = urn[urn.index("(") + 1 : urn.rindex(")")]
+        platform, name, env = inner.split(",")
+        return platform, name, env
+    except Exception:
+        return None
+
+
+def _session_table(name: str) -> Optional[str]:
+    """Bare table name if this is a ``_SESSION``-qualified name, else None."""
+    parts = name.split(".")
+    if len(parts) < 3 or parts[-2].lower() != _SESSION:
+        return None
+    return parts[-1].lower()
+
+
+DiscoveredTables = Union[Collection[str], Callable[[], Collection[str]]]
+
+
+def install_on_aggregator(
+    aggregator,
+    discovered_tables: DiscoveredTables,
+    disjoint_only: bool = True,
+) -> SessionPatchStats:
+    """Wrap an aggregator so ``_SESSION`` upstreams resolve to real datasets.
+
+    With ``disjoint_only`` (the default) the aggregator sees ONLY the upstreams this
+    resolver actually recovered, and queries where nothing was recovered are dropped
+    entirely. That keeps a sidecar recipe strictly complementary to the main one: it
+    emits exactly what the main recipe structurally cannot, and nothing else, so the two
+    never write usage for the same (dataset, day) and widening the sidecar's scope is
+    harmless.
+
+    ``discovered_tables`` may be a collection or a zero-arg callable returning one.
+    The index is built lazily on first use: BigqueryV2Source hands the extractor a live
+    reference to ``bq_schema_extractor.table_refs``, which is still empty when the
+    extractor is constructed and only fills during the schema pass.
+    """
+    stats = SessionPatchStats()
+    state: Dict[str, object] = {}
+    # Distinct-ness is measured on what the counter CLAIMS to count, not on the cache
+    # key: the cache is keyed by the full _SESSION URN, which embeds the reader project.
+    seen_targets: Set[str] = set()
+    seen_ambiguous: Set[str] = set()
+    seen_unknown: Set[str] = set()
+    # Memoised so each distinct URN is classified -- and counted -- exactly once. The
+    # column_usage remap below resolves the same URNs a second time; without this the
+    # counters would report resolution calls rather than distinct tables.
+    cache: Dict[str, Tuple[str, str]] = {}
+
+    def get_index() -> Tuple[Dict[str, str], Set[str]]:
+        if "index" not in state:
+            tables = discovered_tables() if callable(discovered_tables) else discovered_tables
+            index, ambiguous = build_index(tables or ())
+            stats.index_size, stats.ambiguous_names = len(index), len(ambiguous)
+            state["index"] = (index, ambiguous)
+            logger.info("datahub-bq-connector-session-patch: resolution index built (%s)", stats)
+        return state["index"]  # type: ignore[return-value]
+
+    original = aggregator.add_preparsed_query
+
+    def classify(urn: str) -> Tuple[str, str, Optional[str]]:
+        """(resolved_urn, outcome, reader_project).
+
+        Counts DISTINCT outcomes on first sight only; per-reference volume is counted by
+        the caller, so the column_usage remap below cannot inflate either.
+        """
+        hit = cache.get(urn)
+        if hit is not None:
+            return hit
+
+        index, ambiguous = get_index()
+        result: Tuple[str, str, Optional[str]] = (urn, PASSTHROUGH, None)
+        split = _split_urn(urn)
+        if split is not None:
+            platform, name, env = split
+            table = _session_table(name)
+            if table is not None:
+                parts = name.split(".")
+                # `{billing_project}._SESSION.{table}` -- the project that ran the query.
+                reader = parts[-3] if len(parts) >= 3 else None
+                target = index.get(table)
+                if target is not None:
+                    prefix = parts[:-3]
+                    resolved = ".".join(prefix + [target])
+                    target_urn = f"urn:li:dataset:({platform},{resolved},{env})"
+                    seen_targets.add(target_urn)
+                    stats.rewritten = len(seen_targets)
+                    result = (target_urn, REWRITTEN, reader)
+                elif table in ambiguous:
+                    seen_ambiguous.add(table)
+                    stats.ambiguous = len(seen_ambiguous)
+                    logger.debug("datahub-bq-connector-session-patch: %r is ambiguous, leaving as temp", table)
+                    result = (urn, AMBIGUOUS, reader)
+                else:
+                    seen_unknown.add(table)
+                    stats.unknown = len(seen_unknown)
+                    result = (urn, UNKNOWN, reader)
+
+        cache[urn] = result
+        return result
+
+    def resolve(urn: str) -> str:
+        return classify(urn)[0]
+
+    def add_preparsed_query(parsed, *args, **kwargs):
+        recovered = set()
+        already_resolvable = set()
+        resolved = []
+        for upstream in parsed.upstreams:
+            new, outcome, reader = classify(upstream)
+            resolved.append(new)
+            if outcome == PASSTHROUGH:
+                # A real, already-qualified reference -- the main recipe resolves this
+                # one itself and will emit a query entity carrying it.
+                already_resolvable.add(new)
+            if outcome == REWRITTEN:
+                stats.rewritten_refs += 1
+                recovered.add(new)
+            elif outcome == AMBIGUOUS:
+                stats.ambiguous_refs += 1
+            elif outcome == UNKNOWN:
+                stats.unknown_refs += 1
+            if outcome != PASSTHROUGH:
+                stats.record_ref(reader, outcome)
+
+        if disjoint_only:
+            if not recovered:
+                # Every reference was already resolvable -- the main recipe has it.
+                stats.queries_skipped += 1
+                return
+            if already_resolvable:
+                # MIXED query: some references were already resolvable, some we
+                # recovered. Disjointness does NOT hold here. The query entity is keyed
+                # by get_query_fingerprint(sql), a pure function of the SQL text, so both
+                # pipelines emit the SAME urn:li:query:<fp> -- and querySubjects is a
+                # versioned aspect, not timeseries. Emitting our narrowed subject set
+                # would overwrite the main recipe's, and the entity would flip between
+                # the two on every scheduled run. Passing the union instead would
+                # double-book usage for the already-resolvable half. Neither is
+                # acceptable, so the query is left entirely to the main recipe.
+                stats.queries_mixed += 1
+                return
+            parsed.upstreams = sorted(recovered)
+        else:
+            parsed.upstreams = resolved
+            if parsed.downstream:
+                parsed.downstream = resolve(parsed.downstream)
+
+        if parsed.column_usage:
+            # Union rather than assign: two keys can collapse onto one URN (a self-join
+            # referencing the same table both qualified and unqualified), and dict
+            # comprehension would silently keep only the last column set.
+            remapped: Dict[str, Set[str]] = {}
+            for key, columns in parsed.column_usage.items():
+                remapped.setdefault(resolve(key), set()).update(columns or ())
+            parsed.column_usage = (
+                {k: v for k, v in remapped.items() if k in recovered}
+                if disjoint_only
+                else remapped
+            )
+
+        stats.queries_passed += 1
+        return original(parsed, *args, **kwargs)
+
+    aggregator.add_preparsed_query = add_preparsed_query
+    return stats

datahub_bq_connector_session_patch/source.py

@@ -0,0 +1,100 @@
+"""``datahub_bq_connector_session_patch`` -- the sidecar source type.
+
+A ``BigqueryV2Source`` that arms the resolver for its own run and nothing else. Point a
+narrow sidecar recipe at ``type: datahub_bq_connector_session_patch``; the production recipe keeps
+``type: bigquery`` and is unaffected even though both interpreters have this installed.
+
+It also refuses to start on the two misconfigurations that are actively destructive
+rather than merely wrong -- see ``_check_sidecar_safety``.
+"""
+
+import logging
+from typing import Iterable, List
+
+from datahub.ingestion.api.common import PipelineContext
+from datahub.ingestion.api.decorators import (
+    SupportStatus,
+    config_class,
+    platform_name,
+    support_status,
+)
+from datahub.ingestion.api.workunit import MetadataWorkUnit
+from datahub.ingestion.source.bigquery_v2.bigquery import BigqueryV2Source
+from datahub.ingestion.source.bigquery_v2.bigquery_config import BigQueryV2Config
+
+from datahub_bq_connector_session_patch.patch import armed, install
+
+logger = logging.getLogger(__name__)
+
+
+class SidecarMisconfigured(ValueError):
+    """The recipe would damage the catalog. Raised before any metadata is read."""
+
+
+def _check_sidecar_safety(config: BigQueryV2Config) -> None:
+    """Refuse the two settings that make this sidecar destructive.
+
+    Both were reproduced against a live catalog; neither is a style preference:
+
+    * ``remove_stale_metadata`` -- the sidecar writes a ``status`` aspect on every
+      dataset it produces usage for, which enrols them in *its* stale-removal
+      checkpoint. Its URN set is usage-driven, so any dataset not read in the window
+      falls out of the checkpoint and gets soft-deleted -- datasets the main recipe
+      owns. This is normal operation, not an edge case.
+    * ``include_table_lineage`` -- a mis-resolved write would create a wrong edge into a
+      real table, which is worse than the missing usage this package exists to fix.
+    """
+    problems: List[str] = []
+
+    stateful = config.stateful_ingestion
+    # remove_stale_metadata defaults to True, so only the combination bites: stale
+    # removal cannot run at all when stateful ingestion is off, and refusing that
+    # combination would block every file-sink measurement run.
+    if (
+        stateful is not None
+        and getattr(stateful, "enabled", False)
+        and getattr(stateful, "remove_stale_metadata", False)
+    ):
+        problems.append(
+            "stateful_ingestion.remove_stale_metadata must be false. This sidecar's URN "
+            "set is usage-driven, so stale removal soft-deletes datasets the main "
+            "recipe owns, on every run."
+        )
+
+    if config.include_table_lineage:
+        problems.append(
+            "include_table_lineage must be false. Resolved names are used for usage "
+            "attribution only; emitting lineage from a resolved name risks a wrong edge "
+            "into a real table."
+        )
+
+    if problems:
+        raise SidecarMisconfigured(
+            "datahub_bq_connector_session_patch refuses to run with this recipe:\n  - "
+            + "\n  - ".join(problems)
+        )
+
+
+@platform_name("BigQuery")
+@config_class(BigQueryV2Config)
+@support_status(SupportStatus.BETA)
+class BigQuerySessionPatchSource(BigqueryV2Source):
+    """BigQuery source that recovers reads whose SQL did not qualify its table names."""
+
+    def __init__(self, ctx: PipelineContext, config: BigQueryV2Config):
+        _check_sidecar_safety(config)
+        super().__init__(ctx, config)
+
+    @classmethod
+    def create(cls, config_dict: dict, ctx: PipelineContext) -> "BigQuerySessionPatchSource":
+        config = BigQueryV2Config.model_validate(config_dict)
+        return cls(ctx, config)
+
+    def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
+        # install() patches the class; armed() is what makes the patch do anything, and
+        # it covers only this generator's execution. The extractor is constructed inside
+        # super().get_workunits_internal(), so both have to be in place before we
+        # delegate.
+        install()
+        with armed():
+            yield from super().get_workunits_internal()

datahub_bq_connector_session_patch-0.1.0.dist-info/METADATA

@@ -0,0 +1,287 @@
+Metadata-Version: 2.5
+Name: datahub-bq-connector-session-patch
+Version: 0.1.0
+Summary: DataHub BigQuery sidecar source: attribute reads whose SQL does not qualify its table names
+Project-URL: Homepage, https://github.com/gutro/dv-datahub/tree/master/datahub-bq-connector-session-patch
+Project-URL: Repository, https://github.com/gutro/dv-datahub
+Project-URL: Issues, https://github.com/gutro/dv-datahub/issues
+Author: DataVantage Platform
+License-Expression: Apache-2.0
+License-File: LICENSE
+Keywords: bigquery,datahub,ingestion,lineage,usage
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Database
+Requires-Python: >=3.9
+Provides-Extra: test
+Requires-Dist: acryl-datahub[bigquery]<1.8,>=1.6; extra == 'test'
+Requires-Dist: pytest>=7; extra == 'test'
+Requires-Dist: pyyaml; extra == 'test'
+Description-Content-Type: text/markdown
+
+# datahub-bq-connector-session-patch
+
+A DataHub BigQuery **sidecar source** that attributes reads whose SQL did not qualify
+its table names — the ones the stock connector sees, misfiles and silently discards.
+
+Full diagnosis, design, verification and limits:
+`datavantage/docs/datahub-unqualified-read-attribution.md`.
+
+## The problem
+
+A client with a BigQuery *default dataset* set can write `SELECT ... FROM Bets_Details`.
+BigQuery fills in the blank at run time, but never records it: `INFORMATION_SCHEMA.JOBS`
+carries the SQL text and the billing project, not the job's default dataset.
+
+DataHub's connector substitutes `_SESSION` for the missing dataset
+(`queries_extractor.py:542,558`), on the assumption that a bare name is usually a
+temporary table. Its own temp-table rule then flags anything whose dataset starts with
+`_` (`:309`), and the aggregator drops it — for usage
+(`sql_parsing_aggregator.py:1022`) and for the query entity (`:1693`).
+
+So the connector *sees* the table name, guesses the wrong container, and discards the
+result. Nothing is reported: the sibling branch of `is_temp_table` records what it drops,
+this one returns `True` in silence. Measured here: one client, 2,866 jobs in 7 days,
+100% unqualified, 0 usage rows.
+
+**A `0` in DataHub usage is therefore ambiguous** — "nobody reads this", or "read
+constantly, with unqualified SQL". Decommissioning decisions depend on telling those apart.
+
+## What this does
+
+For upstreams that landed in `_SESSION`, it resolves the bare table name against the
+tables DataHub discovered during its own schema pass. A name matching more than one
+discovered table is **left untouched** — dropped exactly as before, never guessed.
+
+It ships as a distinct source type, so the production recipe stays stock:
+
+```yaml
+source:
+  type: datahub_bq_connector_session_patch
+```
+
+## Installing
+
+Per-recipe, via **Extra Pip Libraries** (DataHub UI) or `extra_pip_requirements`. That
+field is scoped to one recipe, which is what keeps the production recipe unaffected.
+
+```
+datahub-bq-connector-session-patch
+```
+
+The wheel declares **no dependencies** — it imports only `datahub.*`, which any
+environment running a recipe already has. Declaring `acryl-datahub` would let pip
+re-resolve the executor's own pinned copy while installing this. The tested range
+(`>=1.6,<1.8`) is enforced at runtime instead: an out-of-range executor raises a
+structured warning into the ingestion report rather than failing the run.
+
+## The sidecar recipe
+
+```yaml
+pipeline_name: bq-session-attribution        # its OWN name — never the main recipe's
+source:
+  type: datahub_bq_connector_session_patch
+  config:
+    project_id_pattern:
+      allow:
+        - '^dv-prod-eu-w1(-.+)?-data$'
+        - '^dv-ext-prod-eu-w1(-.+)?-data$'
+        - '^dv-prod-eu-w1(-.+)?-comp\d+$'
+    include_tables: false          # lightweight discovery still fills table_refs
+    include_views: false
+    include_table_lineage: false   # refused if true — see below
+    include_usage_statistics: true
+    use_queries_v2: true
+    include_queries: true
+    include_query_usage_statistics: true
+    region_qualifiers: ['region-europe-west1']
+    start_time: '-7 days'
+    enable_stateful_time_window: true
+    stateful_ingestion:
+      enabled: true                # the time-window watermark
+      remove_stale_metadata: false # refused if true — see below
+    env: PROD
+```
+
+### Two settings this source refuses to start with
+
+Both were reproduced against a live catalog. They are not style preferences, so they are
+enforced in code rather than left to the YAML:
+
+| setting | why it is refused |
+|---|---|
+| `stateful_ingestion.remove_stale_metadata: true` | The sidecar writes a `status` aspect on every dataset it produces usage for, enrolling them in *its* stale-removal checkpoint. Its URN set is **usage-driven**, so any dataset not read that day falls out of the checkpoint and is soft-deleted — datasets the main recipe owns. Normal operation, not an edge case. |
+| `include_table_lineage: true` | Resolved names drive usage attribution only. A mis-resolved write would create a **wrong edge into a real table**, which is worse than the missing usage this package exists to fix. |
+
+`remove_stale_metadata` defaults to `true`, so it must be set explicitly. Only the
+combination bites — with `stateful_ingestion.enabled: false` there is no checkpoint and
+nothing is refused.
+
+### Scope it to the readers, not to minimise ambiguity
+
+`dataset_pattern` should be **wide**. "Ambiguous" means the name matches more than one
+entry in `discovered_tables` *within the recipe's scope*, so narrowing scope removes the
+alternatives rather than resolving between them. If a reader's default dataset is
+`prod_spain_*` but the index holds only `prod_malta_*`, the name resolves **confidently
+and wrongly**.
+
+Ambiguity fails safe. A too-narrow index fails silently, in the wrong direction.
+
+Widening is otherwise harmless: `disjoint_only` means the sidecar emits only URNs it
+actually recovered, so it stays quiet wherever SQL is already qualified and never
+double-books usage with the main recipe.
+
+## Reading the run report
+
+```
+rewritten=41 tables (2792 refs) ambiguous=18 names (63 refs) unknown=7 names (12 refs)
+queries_passed=954 queries_skipped=3118 (index=1631 unique names, 412 ambiguous names)
+```
+
+- **distinct counters** — how much of the catalog was recovered. `rewritten` counts
+  distinct resolved **target tables**; `ambiguous` and `unknown` count distinct bare
+  **names**. Deliberately not distinct `_SESSION` URNs: those embed the reader project,
+  so one table read from five projects would report as five.
+- **`*_refs`** — how many reads that attributed. One hot table read 2,000 times is one
+  rewrite and 2,000 references.
+- **`queries_skipped`** — queries whose SQL was already fully qualified. The main recipe
+  has those; the sidecar correctly stays out of the way.
+- **`queries_mixed`** — queries carrying *both* an already-resolvable reference and a
+  recovered one. Also left to the main recipe, for a subtler reason, below.
+
+### Why mixed queries are skipped
+
+`disjoint_only` keeps the two pipelines from double-booking **usage**, which is a
+timeseries aspect. It does not, by itself, protect the **query entity**.
+
+Query entities are keyed by `get_query_fingerprint(sql, platform, fast=True)` — a pure
+function of the SQL text, so both pipelines derive the *same* `urn:li:query:<fp>`. But
+`querySubjects` is a versioned aspect, not timeseries. If the sidecar emitted its
+narrowed subject set for a query the main recipe also sees, whichever ran last would
+win, and the entity would flip between the two subject sets on every scheduled run.
+
+Passing the union instead would fix the subjects and double-book usage for the
+already-resolvable half. Neither is acceptable, so a mixed query is left entirely to the
+main recipe and counted. Measured on `dv-ext-prod-eu-w1-data` over 7 days:
+`queries_mixed=0` — the guard costs nothing there, because that client's SQL is
+uniformly unqualified.
+
+A per-reader-project breakdown is logged alongside it, so a project that recovers nothing
+shows up at 0.0% rather than vanishing.
+
+### Two failures that look like success
+
+Both were hit during development. Each leaves the run green and the output plausible:
+
+1. **`discovered_tables` is empty at construction.** The source passes a *live reference*
+   to `bq_schema_extractor.table_refs`, which only fills during the schema pass. The
+   index is built lazily on first use, never in `__init__`.
+2. **Refs are `projects/P/datasets/D/tables/T`, not `P.D.T`.** Naive dotted parsing
+   matches nothing.
+
+Because of these the source raises a structured **warning** if the index came out empty
+or nothing was rewritten. **Never trust a negative result without checking the index
+size.**
+
+## Limits
+
+Unique-name resolution only. Where a bare name matches several catalogued objects it is
+dropped. How much that costs depends entirely on the estate:
+
+| population | ambiguous names |
+|---|---|
+| views in `dv-ext-prod-eu-w1-data` | **0.0%** (162/162 unique) |
+| views in `dv-prod-eu-w1-data` | **54.4%** |
+| tables referenced estate-wide | 53.0% |
+
+The external-facing views carry licence/brand suffixes and are unique by construction.
+The internal estate's per-market layout duplicates names across `prod_malta_*`,
+`prod_spain_*`, `prod_italy_*`…, and the cross-market `common_` layer duplicates a
+further 39 names against them — so roughly half of any bare reads against it stay
+unrecovered.
+
+Resolving those needs **leaf-set matching**, not name matching: the colliding
+common-vs-market views have identical names but cleanly distinct leaf sets. That is
+explicitly out of scope — see §9.1 of the design doc for what it would take.
+
+## Measuring a scope before shipping it
+
+```bash
+export BQ_SESSION_PATCH_BILLING_PROJECT=dv-ext-prod-eu-w1-data
+python tools/measure_scope.py eu baseline --days 7
+python tools/measure_scope.py eu patched  --days 7
+python tools/diff_runs.py runs/mcps_eu_baseline_7d.json runs/mcps_eu_patched_7d.json
+```
+
+Scope flags: `--projects` for explicit ids, `--project-pattern` to override the region's
+`project_id_pattern.allow` regexes, `--days` for the window.
+
+**The billing project must be explicit** — `--billing`, or
+`$BQ_SESSION_PATCH_BILLING_PROJECT`, or inferred when `--projects` names exactly one.
+DataHub passes it straight to `bigquery.Client(...)`, and when it is `None` the client
+inherits whatever `google.auth.default()` resolves: `GOOGLE_CLOUD_PROJECT` first, then the
+ADC file's quota project, then the active gcloud config. A stray env var will therefore
+bill every `INFORMATION_SCHEMA` job to an unrelated project and fail as
+`bigquery.jobs.create` denied against a project you never named. The script refuses to
+start rather than inherit one, and prints both the billing and the ambient project so a
+mismatch is visible.
+
+`measure_scope.py` writes MCPs to a file sink (nothing reaches GMS) and prints the
+per-project recovery table; `diff_runs.py` reports datasets gaining usage, query entities
+gained and readers newly attributed, and fails if any `schemaMetadata`,
+`upstreamLineage` or `datasetProperties` aspect leaked into the output.
+
+## Trying it against a local DataHub
+
+`recipes/local/` holds a two-step pair for a quickstart instance, scoped to one project:
+
+```bash
+make venv && source .venv/bin/activate
+datahub ingest -c recipes/local/01-catalog.yml       # stock `bigquery` — the catalog
+datahub ingest -c recipes/local/02-usage-sidecar.yml # this package — the recovered usage
+```
+
+Step 1 runs with usage **on**, exactly as production does, and still misses the
+unqualified reads — that is the premise. Step 2 shows what it missed. Measured against
+`dv-ext-prod-eu-w1-data` over 7 days:
+
+| | step 1 (catalog) | step 2 (sidecar) |
+|---|---|---|
+| datasets with a usage aspect | 5 | **+59** |
+| query entities | 170 | **+61** |
+| aspects emitted | schema, lineage, properties, usage, queries | usage and queries only |
+
+Both use ADC, so the VPN must be up — VPC-SC blocks ADC while the `bq` CLI keeps
+working, which makes the failure look like a permissions problem.
+
+They run **stateless** (`stateful_ingestion.enabled: false`), so re-running re-emits the
+same days. Timeseries aspects append rather than upsert, so repeated local runs
+accumulate duplicate usage documents for a day. Harmless locally; in a real deployment
+keep the stateful time window, which is what makes each day emit exactly once.
+
+## Developing
+
+```bash
+make venv      # .venv on python 3.11, package installed editable + test extras
+make test      # 49 tests against a real acryl-datahub
+make gate      # the acceptance gate: must FAIL on stock, PASS patched
+make release   # test, gate, bump, rebuild, verify  -> dist/
+make publish
+```
+
+`make venv` exists for interactive work and for the `tools/` scripts, which need real
+BigQuery credentials and so cannot run in a throwaway environment. `make test` and
+`make gate` deliberately **do not** use it — they build their own environment per run, so
+a stale or hand-modified `.venv` can never make them pass.
+
+`make gate` is the one that matters on an acryl-datahub bump. This package patches
+private internals, so `make test` alone would stay green even if upstream fixed the
+defect out from under it — at which point this package is dead weight and should be
+retired, not shipped. `gate` fails loudly in that case.
+
+## Upstream
+
+The defect is unfixed on `datahub-project/datahub` master as of 2026-09-14. The ask
+there is modest: report the `_SESSION` discard, or offer opt-in resolution against
+discovered tables. See `UPSTREAM-ISSUE.md`.

datahub_bq_connector_session_patch-0.1.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+datahub_bq_connector_session_patch/__init__.py,sha256=U_kaflaxg_0rAwthDzHJDyrNL2W867klJyEUHtPhEzY,1247
+datahub_bq_connector_session_patch/patch.py,sha256=GwkNiMIbACZM8HJrJHVWRCqKcu7uTRhyeDkDhCNXtqQ,8128
+datahub_bq_connector_session_patch/resolver.py,sha256=szUyW622C4NGZhi-nIEcRwhxsfpGfxw9o5aG21mk8Og,13419
+datahub_bq_connector_session_patch/source.py,sha256=16lsU8UDdOoC9qzbv69xP1RG4ZsxF0RDlOOTVXI822Q,4168
+datahub_bq_connector_session_patch-0.1.0.dist-info/METADATA,sha256=6dNNhK5hKxpN9trR0RX5y9-jfa8PPDlk3isoTqtUMhQ,13628
+datahub_bq_connector_session_patch-0.1.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+datahub_bq_connector_session_patch-0.1.0.dist-info/entry_points.txt,sha256=HGFQhMcd2ToOmoOBYO4Q3TEu_KsFvDxG98J8M4sDNQc,141
+datahub_bq_connector_session_patch-0.1.0.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+datahub_bq_connector_session_patch-0.1.0.dist-info/RECORD,,

datahub_bq_connector_session_patch-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.0
+Root-Is-Purelib: true
+Tag: py3-none-any

datahub_bq_connector_session_patch-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[datahub.ingestion.source.plugins]
+datahub_bq_connector_session_patch = datahub_bq_connector_session_patch.source:BigQuerySessionPatchSource

datahub_bq_connector_session_patch-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.