mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/sources/catalog/coverage.py

@@ -0,0 +1,137 @@
+"""What a catalogue generation claims to cover, and how much it was drawn from.
+
+A converged sweep of a provider is one immutable corpus.  Publishing all of it is one honest
+thing to do; publishing a stated part of it is another.  Publishing a stated part of it *while
+looking exactly like all of it* is the thing this module exists to make impossible.
+
+The release format used to have no way to say which of the two a generation was.  A generation
+that authored 200,000 of 743,000 records read, member for member, exactly like a generation whose
+provider published 200,000 records -- same head, same descriptors, same entry count.  Nothing in
+the bytes distinguished a deliberate selection from an exhaustive catalogue, and nothing
+distinguished either from a run that had quietly stopped early.
+
+So every document along the harvest -> author -> classify -> publish -> release chain carries one
+coverage statement, and it carries it as a *required* member.  A required member is the point: an
+optional one would let a bounded catalogue claim exhaustiveness by saying nothing, which is the
+failure this module is written against.
+
+Four members, and no fifth:
+
+``rule``
+    ``"exhaustive"``, or the name of one registered selection rule.  A rule is a *total order
+    over the whole corpus*, so the records it selects are a property of the corpus rather than of
+    how far authoring happened to get.  That is the difference between a selection and a
+    truncation, and it is why a bounded generation can be complete.
+
+``bound``
+    ``None`` under ``"exhaustive"``; otherwise how many records the rule was asked for.  It is
+    what the operator requested, not what they got.
+
+``corpus_records``
+    The converged sweep's unique record count -- the size of the thing the rule chose from.
+    Without it a bound means nothing: 200,000 of 200,000 and 200,000 of 743,000 are different
+    catalogues that would otherwise state identical numbers.
+
+``selected_records``
+    How many records the rule actually selected, which equals ``min(bound, corpus_records)``.
+
+**Exhaustiveness is deliberately not a member.**  A reader answers it with
+``selected_records == corpus_records`` and nothing else.  A ``kind`` or ``exhaustive`` flag would
+be a second source of truth that could disagree with the counts beside it, and a reader would
+have no way to know which one to believe.  A comparison cannot disagree with itself.
+
+Note what a downstream reader can and cannot establish.  The arithmetic above is re-derived at
+every boundary that carries a coverage statement, so a fabricated one is refused.  That the
+selected records really are the ones the named rule ranks highest is established only by the
+authoring stage, which is the only stage that ever sees the records the rule excluded; every
+stage after it holds the selection and not the corpus.  This module states the claim and closes
+its arithmetic.  It does not pretend to re-derive an ordering over records nobody downstream has.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+#: The coverage a generation states when it authored every record its converged sweep found.
+#: It is a rule name in the same member as every other rule so that a reader answers "which rule
+#: chose these records" in one place, and never has to infer exhaustiveness from an absent field.
+COVERAGE_EXHAUSTIVE_RULE = "exhaustive"
+
+#: The registered bounded selection rules, by name.
+#:
+#: ``last_harvested_date_desc`` orders the corpus by the provider's ``$.last_harvested_date``
+#: observation, descending, ties broken by ascending provider record identifier, and takes the
+#: first ``bound`` of them.  It is the sweep's own walk order -- the harvest paginates
+#: ``sort=last_harvested_date`` -- so "the N most recently harvested datasets" is a catalogue
+#: that can be described to a reader in one sentence, and re-derived by anyone holding the
+#: staging generation.  A record whose harvest date the rule cannot read is ranked below every
+#: record it can; see ``authoring_shards`` for exactly what "cannot read" means.
+SELECTION_RULE_LAST_HARVESTED_DATE_DESC = "last_harvested_date_desc"
+SELECTION_RULES = frozenset({SELECTION_RULE_LAST_HARVESTED_DATE_DESC})
+
+#: Exactly what one coverage statement says, and nothing else.
+COVERAGE_MEMBERS = frozenset({"rule", "bound", "corpus_records", "selected_records"})
+
+
+def coverage_is_valid(coverage: Any) -> bool:
+    """Whether one coverage statement is exactly its contract, with self-consistent values.
+
+    Every document that carries coverage answers this one question, so that adding a member to
+    the contract is one edit rather than six that have to agree.  Each caller still raises its
+    own typed refusal, because the boundary the statement failed at is what an operator needs.
+
+    The final clause is what makes a coverage statement hard to fabricate: ``selected_records``
+    is not free, it is ``min(bound, corpus_records)``.  A generation cannot claim to have drawn
+    200,000 records from a corpus of 200,000 under a bound of 1,000, and it cannot claim a
+    corpus larger than the one its staging coordinate names, because the stage that writes the
+    statement takes ``corpus_records`` from the terminal pass map rather than from its caller.
+    """
+
+    if (
+        not isinstance(coverage, dict)
+        or set(coverage) != COVERAGE_MEMBERS
+        # The rule is tested for being a string before it is tested for membership: canonical
+        # JSON admits a list or an object here, and both are unhashable, so asking a frozenset
+        # about one raises TypeError rather than answering. Every caller of this predicate has
+        # its own typed refusal to raise, and a crash is not one of them.
+        or not isinstance(coverage["rule"], str)
+        or type(coverage["corpus_records"]) is not int
+        or type(coverage["selected_records"]) is not int
+        or coverage["corpus_records"] < 0
+        or not 0 <= coverage["selected_records"] <= coverage["corpus_records"]
+    ):
+        return False
+    if coverage["rule"] == COVERAGE_EXHAUSTIVE_RULE:
+        return (
+            coverage["bound"] is None and coverage["selected_records"] == coverage["corpus_records"]
+        )
+    return (
+        coverage["rule"] in SELECTION_RULES
+        and type(coverage["bound"]) is int
+        and coverage["bound"] >= 1
+        and coverage["selected_records"] == min(coverage["bound"], coverage["corpus_records"])
+    )
+
+
+def covers_whole_corpus(coverage: Any) -> bool:
+    """Whether this generation holds every record its converged sweep found.
+
+    This is the one question a consumer asks of a coverage statement, and it is a comparison
+    rather than a stored flag on purpose.  An invalid statement is not exhaustive; it is not
+    anything, and saying so here keeps every caller from having to check validity first.
+    """
+
+    return coverage_is_valid(coverage) and (
+        coverage["selected_records"] == coverage["corpus_records"]
+    )
+
+
+def exhaustive_coverage(record_count: int) -> dict[str, Any]:
+    """The coverage of a generation that authored every record of a corpus this size."""
+
+    return {
+        "rule": COVERAGE_EXHAUSTIVE_RULE,
+        "bound": None,
+        "corpus_records": record_count,
+        "selected_records": record_count,
+    }