tabalyst 0.4.0__tar.gz → 0.4.1__tar.gz

{tabalyst-0.4.0/src/tabalyst.egg-info → tabalyst-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tabalyst
-Version: 0.4.0
+Version: 0.4.1
 Summary: Open-source toolkit for understanding and working with unfamiliar data.
 Author: Gregory Borelli
 License-Expression: MIT

{tabalyst-0.4.0 → tabalyst-0.4.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "tabalyst"
-version = "0.4.0"
+version = "0.4.1"
 description = "Open-source toolkit for understanding and working with unfamiliar data."
 readme = "README.md"
 requires-python = ">=3.11"

{tabalyst-0.4.0 → tabalyst-0.4.1}/src/tabalyst/models.py

@@ -378,7 +378,7 @@ class DatasetProfile(ResultModel):
 
 class ReportProfile(ResultModel):
     format_version: Literal["0.1.0a"] = "0.1.0a"
-    format_revision: Literal[6] = 6
+    format_revision: Literal[7] = 7
     generated_at: datetime
     processing_seconds: FiniteFloat
     source: SourceInfo

{tabalyst-0.4.0 → tabalyst-0.4.1}/src/tabalyst/scanner/config.py

@@ -52,6 +52,9 @@ MAX_PREVIEW_RECORDS = 1_000
 MAX_PATTERNS = 200
 MAX_PATTERN_LENGTH = 1_000
 MAX_PATTERN_INPUT_LENGTH = 10_000
+# A streamed field keeps its warm-up values: the warm-up is bounded.
+MAX_WARMUP_VALUES = 1_000_000
+MAX_PROBE_INTERVAL = 1_000_000
 
 MissingCategory = Literal["absent", "null", "empty", "blank", "marker"]
 ScalarType = Literal["string", "integer", "number", "boolean"]
@@ -176,6 +179,9 @@ class TypeSettings(_Settings):
 
 class DetectionSettings(_Settings):
     minimum_share: float = Field(default=0.95, gt=0, le=1)
+    # Adaptive detection (design 13): 0 keeps detection exhaustive.
+    warmup_values: int = Field(default=10_000, ge=0, le=MAX_WARMUP_VALUES)
+    probe_interval: int = Field(default=100, ge=0, le=MAX_PROBE_INTERVAL)
 
 
 NumberConvention = Literal["dot", "comma"]

{tabalyst-0.4.0 → tabalyst-0.4.1}/src/tabalyst/scanner/detectors/base.py

@@ -69,9 +69,15 @@ class DetectorAccumulator:
     ``add`` receives every tested value with its classification (``None`` when
     not matched) and its count. In both execution modes it sees the same
     distinct values, first seen first, with the same total counts.
+
+    A class whose ``add`` does nothing for unmatched values (``None``) sets
+    ``ignores_unmatched`` in its own body: the engine then skips those calls.
+    The flag applies only to the class that defines ``add``, so a subclass
+    that overrides ``add`` receives every value unless it sets it again.
     """
 
     __slots__ = ()
+    ignores_unmatched: ClassVar[bool] = True
 
     def add(
         self, value: str, classification: Classification | None, count: int
@@ -135,6 +141,7 @@ class AmbiguityAccumulator(DetectorAccumulator):
     """
 
     __slots__ = ("candidates", "count", "evidence", "resolution")
+    ignores_unmatched = True
 
     def __init__(self, readings: list[str], resolution: dict[str, str] | None) -> None:
         self.count = 0

{tabalyst-0.4.0 → tabalyst-0.4.1}/src/tabalyst/scanner/detectors/boolean.py

@@ -51,6 +51,7 @@ class BooleanDetector(Detector):
 
 class BooleanAccumulator(DetectorAccumulator):
     __slots__ = ("false", "true")
+    ignores_unmatched = True
 
     def __init__(self) -> None:
         self.true = 0

{tabalyst-0.4.0 → tabalyst-0.4.1}/src/tabalyst/scanner/detectors/currency.py

@@ -113,6 +113,7 @@ class CurrencyDetector(Detector):
 
 class CurrencyAccumulator(AmountAccumulator):
     __slots__ = ("currencies",)
+    ignores_unmatched = True
 
     def __init__(self, reader: AmountReader) -> None:
         super().__init__(reader)

{tabalyst-0.4.0 → tabalyst-0.4.1}/src/tabalyst/scanner/detectors/email.py

@@ -72,6 +72,7 @@ class EmailDetector(Detector):
 
 class EmailAccumulator(DetectorAccumulator):
     __slots__ = ("domains",)
+    ignores_unmatched = True
 
     def __init__(self, domains: NameCounts) -> None:
         self.domains = domains

{tabalyst-0.4.0 → tabalyst-0.4.1}/src/tabalyst/scanner/detectors/ip.py

@@ -114,6 +114,7 @@ class IpAddressDetector(Detector):
 
 class IpAddressAccumulator(DetectorAccumulator):
     __slots__ = ("versions",)
+    ignores_unmatched = True
 
     def __init__(self, versions: tuple[str, ...]) -> None:
         self.versions = dict.fromkeys(sorted(versions), 0)

{tabalyst-0.4.0 → tabalyst-0.4.1}/src/tabalyst/scanner/detectors/phone.py

@@ -126,6 +126,7 @@ class PhoneDetector(Detector):
 
 class PhoneAccumulator(DetectorAccumulator):
     __slots__ = ("regions",)
+    ignores_unmatched = True
 
     def __init__(self, regions: tuple[str, ...]) -> None:
         self.regions = dict.fromkeys(sorted(regions), 0)

{tabalyst-0.4.0 → tabalyst-0.4.1}/src/tabalyst/scanner/detectors/postal.py

@@ -85,6 +85,7 @@ class PostalCodeDetector(Detector):
 
 class PostalCodeAccumulator(DetectorAccumulator):
     __slots__ = ("regions",)
+    ignores_unmatched = True
 
     def __init__(self, regions: tuple[str, ...]) -> None:
         self.regions = dict.fromkeys(sorted(regions), 0)

{tabalyst-0.4.0 → tabalyst-0.4.1}/src/tabalyst/scanner/detectors/quantity.py

@@ -83,6 +83,7 @@ class QuantityDetector(Detector):
 
 class QuantityAccumulator(AmountAccumulator):
     __slots__ = ("units",)
+    ignores_unmatched = True
 
     def __init__(self, reader: AmountReader, units: NameCounts) -> None:
         super().__init__(reader)

{tabalyst-0.4.0 → tabalyst-0.4.1}/src/tabalyst/scanner/detectors/registry.py

@@ -37,6 +37,7 @@ from tabalyst.scanner.detectors.uuid import UuidDetector
 from tabalyst.scanner.exposure import SHOW, ExposureGate
 from tabalyst.scanner.measures import UNCONVERTIBLE
 from tabalyst.scanner.models import (
+    Adaptive,
     Coverage,
     DetectorComplete,
     DetectorDisabled,
@@ -70,10 +71,15 @@ BUILT_INS: tuple[type[Detector], ...] = (
 _NATIVE_SCALARS = ("string", "integer", "number", "boolean")
 _NO_VALUES = NotApplicable(reason="no_values")
 _STATES = ("matched", "ambiguous", "invalid", "not_matched")
+# Distinct shapes memoized by a detector set (``DetectorSet._fits``).
+SHAPE_CACHE_SIZE = 4_096
 
 # ``report_failure(detector_id, error)`` records a ``detector_failed``
 # diagnostic and returns its index.
 FailureReporter = Callable[[str, str], int]
+# ``report_probe(detector_id, count)`` records a ``detector_skipped_reacted``
+# warning for ``count`` probed occurrences and returns its index.
+ProbeReporter = Callable[[str, int], int]
 
 
 class _NotEligible:
@@ -88,6 +94,22 @@ class _NotEligible:
 NOT_ELIGIBLE = _NotEligible()
 
 
+class _Skipped:
+    """Adaptive detection did not test the value (design 13)."""
+
+    __slots__ = ()
+
+    def __repr__(self) -> str:
+        return "SKIPPED"
+
+
+SKIPPED = _Skipped()
+# The plan of a skipped detector, for the native types it accepts.
+_SKIP = ("skip",)
+# Per native type, one plan per active detector (``DetectorSet._plans``).
+Plans = dict[str, tuple[object, ...]]
+
+
 class DetectorRegistry:
     """Detector classes by ``id``, in registration order."""
 
@@ -121,6 +143,15 @@ def default_registry() -> DetectorRegistry:
     return DetectorRegistry(BUILT_INS)
 
 
+def _ignores_unmatched(accumulator: object) -> bool:
+    """Whether the class that defines ``accumulator.add`` declares, in its own
+    body, that ``add`` does nothing for unmatched values."""
+    for cls in type(accumulator).__mro__:
+        if "add" in cls.__dict__:
+            return cls.__dict__.get("ignores_unmatched", False) is True
+    return False
+
+
 class DetectorTally:
     """Coverage, formats and evidence of one detector on one field."""
 
@@ -138,6 +169,10 @@ class DetectorTally:
         "max_length",
         "not_matched",
         "not_tested",
+        "skipped",
+        "skipped_after",
+        "unmatched_examples",
+        "unmatched_to_accumulator",
     )
 
     def __init__(self, detector: Detector, max_examples: int, max_length: int) -> None:
@@ -150,22 +185,49 @@ class DetectorTally:
         self.invalid = 0
         self.not_matched = 0
         self.not_tested = 0
+        # Adaptive detection: values not tested, and the warm-up size once
+        # the detector is skipped for the field.
+        self.skipped = 0
+        self.skipped_after: int | None = None
         self.formats: dict[str, int] = {}
         self.examples: dict[str, list[str]] = {state: [] for state in _STATES}
         self.error: str | None = None
+        self.unmatched_examples = self.examples["not_matched"]
+        self.unmatched_to_accumulator = True
         try:
             self.accumulator = detector.accumulator()
         except Exception as exc:  # noqa: BLE001 - detector code is isolated (CA19)
             self.error = type(exc).__name__
+        else:
+            self.unmatched_to_accumulator = not _ignores_unmatched(self.accumulator)
 
     def add(self, value: str, classification: object, count: int) -> None:
+        if classification is None:  # the most frequent case
+            if self.error is not None:
+                return
+            self.eligible += count
+            self.not_matched += count
+            examples = self.unmatched_examples
+            if (
+                len(examples) < self.max_examples
+                and len(value) <= self.max_length
+                and value not in examples
+            ):
+                examples.append(value)
+            if self.unmatched_to_accumulator:
+                try:
+                    self.accumulator.add(value, None, count)
+                except Exception as exc:  # noqa: BLE001 - detector code is isolated (CA19)
+                    self.error = type(exc).__name__
+            return
         if classification is NOT_ELIGIBLE or self.error is not None:
             return
         self.eligible += count
-        if classification is None:  # the most frequent case
-            state = "not_matched"
-            self.not_matched += count
-        elif classification is NOT_TESTED:
+        if classification is SKIPPED:
+            self.not_tested += count
+            self.skipped += count
+            return
+        if classification is NOT_TESTED:
             self.not_tested += count
             return
         elif type(classification) is DetectorFailure:
@@ -197,12 +259,23 @@ class DetectorTally:
         except Exception as exc:  # noqa: BLE001 - detector code is isolated (CA19)
             self.error = type(exc).__name__
 
+    def reacted(self) -> bool:
+        """A value matched, was ambiguous or invalid, or the detector failed."""
+        return bool(
+            self.matched or self.ambiguous or self.invalid or self.error is not None
+        )
+
     def exposes_sensitive_values(self) -> bool:
         """A sensitive detector matched a value of the field, or failed on it,
         which cannot prove that nothing matched (design 12.8)."""
         return self.detector.sensitive and (self.matched > 0 or self.error is not None)
 
-    def result(self, report_failure: FailureReporter, gate: ExposureGate = SHOW):
+    def result(
+        self,
+        report_failure: FailureReporter,
+        gate: ExposureGate = SHOW,
+        report_probe: ProbeReporter | None = None,
+    ):
         detector = self.detector
         identity = {"id": detector.id, "version": detector.version}
         if self.error is None and self.eligible:
@@ -229,6 +302,19 @@ class DetectorTally:
             moved = (examples["matched"] + examples["not_matched"])[: self.max_examples]
             examples = {**examples, "matched": [], "not_matched": moved}
         tested = self.eligible - self.not_tested
+        adaptive = None
+        if self.skipped_after is not None:
+            # The warm-up saw no reaction: every reaction comes from a probe.
+            reactions = matched + self.ambiguous + self.invalid
+            adaptive = Adaptive(
+                skipped_after=self.skipped_after,
+                not_tested=self.skipped,
+                diagnostic=(
+                    report_probe(detector.id, reactions)
+                    if reactions and report_probe is not None
+                    else None
+                ),
+            )
         return DetectorComplete(
             **identity,
             coverage=Coverage(
@@ -252,6 +338,7 @@ class DetectorTally:
                 **{state: gate.examples(values) for state, values in examples.items()}
             ),
             details=details,
+            adaptive=adaptive,
         )
 
 
@@ -263,16 +350,20 @@ class DetectorSet:
     """The detectors of one scan, in registry order."""
 
     __slots__ = (
+        "_adaptive",
         "_date",
         "_exposure",
         "_idle",
         "_number",
         "_plans",
+        "_shape_cache",
         "active",
         "entries",
         "max_examples",
         "max_length",
         "minimum_share",
+        "probe_interval",
+        "warmup_values",
     )
 
     def __init__(self, registry: DetectorRegistry, config: ScanConfig) -> None:
@@ -300,6 +391,13 @@ class DetectorSet:
         self.max_examples = config.limits.max_evidence_examples
         self.max_length = config.limits.max_stored_value_length
         self.minimum_share = exact_share(config.detection.minimum_share)
+        self.warmup_values = config.detection.warmup_values
+        self.probe_interval = config.detection.probe_interval
+        # The number and date detectors give every value its technical type
+        # family (design 9.7): adaptive detection never skips them.
+        self._adaptive = tuple(
+            i not in (self._number, self._date) for i in range(len(active))
+        )
         self._exposure = ExposureGate(config.exposure.sensitive_values)
         # Native types that no active detector accepts need no work.
         self._idle = {
@@ -318,30 +416,37 @@ class DetectorSet:
             )
             for native_type in _NATIVE_SCALARS
         }
+        self._shape_cache: dict[str, tuple[bool, ...]] = {}
 
     def versions(self) -> dict[str, int]:
         return {detector.id: detector.version for detector in self.active}
 
-    def classify(self, native_type: str, text: str) -> tuple[object, ...]:
+    def classify(
+        self, native_type: str, text: str, plans: Plans | None = None
+    ) -> tuple[object, ...]:
         """One entry per active detector: a classification, ``None`` (not
-        matched), ``NOT_TESTED``, ``NOT_ELIGIBLE`` or a ``DetectorFailure``."""
+        matched), ``NOT_TESTED``, ``SKIPPED``, ``NOT_ELIGIBLE`` or a
+        ``DetectorFailure``. ``plans`` skips detectors (``skipping``)."""
         idle = self._idle.get(native_type)
         if idle is not None:
             return idle
         results: list[object] = []
-        value_shape = None
-        for plan in self._plans[native_type]:
+        fits = None
+        for i, plan in enumerate((plans or self._plans)[native_type]):
             if plan is None:
                 results.append(NOT_ELIGIBLE)
                 continue
+            if plan is _SKIP:
+                results.append(SKIPPED)
+                continue
             classify, cap, pattern = plan
             if cap is not None and len(text) > cap:
                 results.append(NOT_TESTED)
                 continue
             if pattern is not None:
-                if value_shape is None:
-                    value_shape = shape(text)
-                if pattern.fullmatch(value_shape) is None:
+                if fits is None:
+                    fits = self._fits(shape(text))
+                if not fits[i]:
                     results.append(None)
                     continue
             try:
@@ -354,11 +459,52 @@ class DetectorSet:
             results.append(found)
         return tuple(results)
 
-    def analyze(self, text: str) -> tuple[tuple[object, ...], object, bool, str]:
+    def _fits(self, value_shape: str) -> tuple[bool, ...]:
+        """Whether ``value_shape`` fits the shapes of each active detector.
+
+        Shapes repeat across values far more than values do: the matches are
+        memoized per shape, in a bounded cache.
+        """
+        cache = self._shape_cache
+        fits = cache.get(value_shape)
+        if fits is None:
+            if len(cache) >= SHAPE_CACHE_SIZE:
+                cache.clear()
+            fits = cache[value_shape] = tuple(
+                detector.shapes is None
+                or detector.shapes.fullmatch(value_shape) is not None
+                for detector in self.active
+            )
+        return fits
+
+    def skipping(self, tallies: list[DetectorTally]) -> Plans | None:
+        """Plans that skip, for the rest of a field, the adaptive detectors
+        that reacted to none of its warm-up values; ``None`` when every
+        detector reacted (design 13). Skipped tallies record the warm-up."""
+        skip = [
+            adaptive and not tally.reacted()
+            for adaptive, tally in zip(self._adaptive, tallies)
+        ]
+        if not any(skip):
+            return None
+        for tally, skipped in zip(tallies, skip):
+            if skipped:
+                tally.skipped_after = self.warmup_values
+        return {
+            native_type: tuple(
+                _SKIP if skipped and plan is not None else plan
+                for plan, skipped in zip(plans, skip)
+            )
+            for native_type, plans in self._plans.items()
+        }
+
+    def analyze(
+        self, text: str, plans: Plans | None = None
+    ) -> tuple[tuple[object, ...], object, bool, str]:
         """Classifications of an analytical string, the number the number
         detector reads without ambiguity, whether it has a decimal form, and
         the technical type family."""
-        classifications = self.classify("string", text)
+        classifications = self.classify("string", text, plans)
         number, decimal_form = None, False
         if self._number is not None:
             found = classifications[self._number]
@@ -389,6 +535,7 @@ class DetectorSet:
         tallies: list[DetectorTally] | None,
         report_failure: FailureReporter,
         gate: ExposureGate = SHOW,
+        report_probe: ProbeReporter | None = None,
     ) -> list:
         """Every registered detector in order; ``tallies`` is ``None`` for a
         field without analyzable values."""
@@ -409,7 +556,9 @@ class DetectorSet:
                     )
                 )
             else:
-                results.append(tallies[position].result(report_failure, gate))
+                results.append(
+                    tallies[position].result(report_failure, gate, report_probe)
+                )
             position += 1
         return results
 

{tabalyst-0.4.0 → tabalyst-0.4.1}/src/tabalyst/scanner/detectors/temporal.py

@@ -347,6 +347,7 @@ class DateAccumulator(AmbiguityAccumulator):
     9.6): ambiguous values are counted, never compared."""
 
     __slots__ = ("kinds", "unresolved")
+    ignores_unmatched = True
 
     def __init__(self, readings: list[str], resolution: dict[str, str] | None) -> None:
         super().__init__(readings, resolution)

{tabalyst-0.4.0 → tabalyst-0.4.1}/src/tabalyst/scanner/detectors/url.py

@@ -130,6 +130,7 @@ class UrlDetector(Detector):
 
 class UrlAccumulator(DetectorAccumulator):
     __slots__ = ("hosts",)
+    ignores_unmatched = True
 
     def __init__(self, hosts: NameCounts) -> None:
         self.hosts = hosts

{tabalyst-0.4.0 → tabalyst-0.4.1}/src/tabalyst/scanner/detectors/uuid.py

@@ -83,6 +83,7 @@ class UuidDetector(Detector):
 
 class UuidAccumulator(DetectorAccumulator):
     __slots__ = ("versions",)
+    ignores_unmatched = True
 
     def __init__(self) -> None:
         self.versions: Counter[str] = Counter()

{tabalyst-0.4.0 → tabalyst-0.4.1}/src/tabalyst/scanner/measures.py

@@ -37,7 +37,7 @@ from tabalyst.scanner.models import (
 from tabalyst.scanner.normalization import ANALYTICAL_STAGE, STAGES, Normalizer
 
 if TYPE_CHECKING:  # detectors import this module
-    from tabalyst.scanner.detectors.registry import DetectorSet
+    from tabalyst.scanner.detectors.registry import DetectorSet, Plans
 
 # Exact decimal arithmetic: any rounding or exponent outside the range makes
 # the numeric envelope ``limited`` with reason ``precision``.
@@ -151,11 +151,14 @@ def decimal_or_unconvertible(text: str) -> Decimal | _Unconvertible:
         return UNCONVERTIBLE
 
 
-def string_facts(raw: str, normalizer: Normalizer, detectors: DetectorSet) -> Facts:
-    """Facts of a content string; detectors see its analytical value."""
+def string_facts(
+    raw: str, normalizer: Normalizer, detectors: DetectorSet, plans: Plans | None = None
+) -> Facts:
+    """Facts of a content string; detectors see its analytical value, and
+    ``plans`` may skip some of them (design 13)."""
     forms, changes = normalizer.run(raw)
     text = forms[ANALYTICAL_STAGE]
-    classifications, number, decimal_form, family = detectors.analyze(text)
+    classifications, number, decimal_form, family = detectors.analyze(text, plans)
     return (
         "string",
         text,
@@ -177,11 +180,13 @@ def canonical_text(native_type: str, value: object) -> str:
     return str(value)
 
 
-def scalar_facts(native_type: str, value: object, detectors: DetectorSet) -> Facts:
+def scalar_facts(
+    native_type: str, value: object, detectors: DetectorSet, plans: Plans | None = None
+) -> Facts:
     """Facts of an integer, number (``Decimal``) or boolean; detectors that
     accept its native type see its canonical text."""
     text = canonical_text(native_type, value)
-    classifications = detectors.classify(native_type, text)
+    classifications = detectors.classify(native_type, text, plans)
     if native_type == "boolean":
         return (native_type, text, 0, -1, None, False, 0, None, classifications, "boolean")
     return (

{tabalyst-0.4.0 → tabalyst-0.4.1}/src/tabalyst/scanner/models.py

@@ -18,7 +18,7 @@ T = TypeVar("T")
 
 FORMAT = "tabalyst.scan"
 FORMAT_VERSION = "0.1.0a"
-FORMAT_REVISION = 2
+FORMAT_REVISION = 3
 
 
 class ScanModel(BaseModel):
@@ -341,12 +341,23 @@ class _Detector(ScanModel):
     version: int
 
 
+class Adaptive(ScanModel):
+    """Design 13: the detector reacted to none of the first ``skipped_after``
+    distinct values of the field, so later values other than probes were not
+    tested. ``diagnostic`` points to the warning raised when a probe reacted."""
+
+    skipped_after: int
+    not_tested: int
+    diagnostic: int | None
+
+
 class DetectorComplete(_Detector):
     status: Literal["complete"] = "complete"
     coverage: Coverage
     formats: list[FormatCount]
     evidence: Evidence
     details: dict[str, Any]
+    adaptive: Adaptive | None
 
 
 class DetectorNotApplicable(_Detector):