typemonkey 1.0.0__py3-none-any.whl

typemonkey/__init__.py

@@ -0,0 +1,49 @@
+"""typemonkey — column type inference and type-aware cleaning.
+
+Public API (import ``from typemonkey import X``):
+
+* :func:`infer_type` — profile a column, returning a :class:`ColumnProfile`.
+* :func:`clean_column` — clean a column to an inferred or given type.
+* :func:`clean_numeric` / :func:`clean_boolean` — type-specific cleaners.
+* :func:`normalize_nulls` / :func:`is_null` — null handling.
+* :func:`detect_number_locale` — US vs European number format detection.
+* :func:`looks_like_preserve_string` — zip/phone/zero-padded-ID detection.
+* :class:`ColumnProfile`, :class:`CleanResult`, :class:`TypeName` — result types.
+* :data:`DEFAULT_NULLS`, :data:`TRUE_VALUES`, :data:`FALSE_VALUES` — vocabularies.
+
+See LIMITATIONS.md for deliberate design tradeoffs and SKILL.md for an
+LLM-oriented quick reference.
+"""
+
+from __future__ import annotations
+
+from .booleans import FALSE_VALUES, TRUE_VALUES, clean_boolean, parse_boolean
+from .clean import clean_column
+from .infer import infer_type
+from .locale import detect_number_locale
+from .models import CleanResult, ColumnProfile, TypeName
+from .nulls import DEFAULT_NULLS, is_null, normalize_nulls
+from .numbers import clean_numeric, parse_number
+from .preserve import looks_like_preserve_string
+
+__version__ = "1.0.0"
+
+__all__ = [
+    "infer_type",
+    "clean_column",
+    "clean_numeric",
+    "clean_boolean",
+    "parse_number",
+    "parse_boolean",
+    "normalize_nulls",
+    "is_null",
+    "detect_number_locale",
+    "looks_like_preserve_string",
+    "ColumnProfile",
+    "CleanResult",
+    "TypeName",
+    "DEFAULT_NULLS",
+    "TRUE_VALUES",
+    "FALSE_VALUES",
+    "__version__",
+]

typemonkey/booleans.py

@@ -0,0 +1,124 @@
+"""Boolean vocabulary and normalisation.
+
+This module exists because "true" arrives as ``yes``, ``Y``, ``1``, ``on``,
+``T``, ``true``, ``TRUE`` and a dozen other spellings, and inference needs one
+authoritative mapping from token to ``bool``.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .models import CleanResult, TypeName
+from .nulls import is_null
+
+# Tokens that mean True / False. Matching is case-insensitive and
+# whitespace-trimmed, so only one casing of each spelling is listed.
+TRUE_VALUES: frozenset[str] = frozenset(
+    {"true", "t", "yes", "y", "1", "on", "enabled", "active"}
+)
+FALSE_VALUES: frozenset[str] = frozenset(
+    {"false", "f", "no", "n", "0", "off", "disabled", "inactive"}
+)
+# The numeric pair is recognised as boolean but is *also* valid integer data.
+# Inference uses this set to avoid mislabelling a 0/1 integer column as boolean.
+NUMERIC_BOOLEANS: frozenset[str] = frozenset({"0", "1"})
+
+
+@dataclass
+class ParsedBoolean:
+    """A parsed boolean token.
+
+    Attributes:
+        value: The resulting ``bool``.
+        numeric: ``True`` when the source token was ``"0"`` or ``"1"`` — the
+            ambiguous case that is equally valid integer data.
+    """
+
+    value: bool
+    numeric: bool
+
+
+def parse_boolean(
+    token: object,
+    *,
+    true_values=None,
+    false_values=None,
+) -> ParsedBoolean:
+    """Parse one boolean token, raising :class:`ValueError` if unrecognised.
+
+    Args:
+        token: The value to parse. Non-strings are stringified; the result is
+            trimmed and lower-cased before lookup.
+        true_values: Override truthy vocabulary (case-insensitive). Defaults
+            to :data:`TRUE_VALUES`.
+        false_values: Override falsy vocabulary. Defaults to
+            :data:`FALSE_VALUES`.
+
+    Returns:
+        A :class:`ParsedBoolean`.
+
+    Raises:
+        ValueError: If the token is in neither vocabulary.
+    """
+    trues = TRUE_VALUES if true_values is None else frozenset(v.lower() for v in true_values)
+    falses = FALSE_VALUES if false_values is None else frozenset(v.lower() for v in false_values)
+    key = str(token).strip().lower()
+    if key in trues:
+        return ParsedBoolean(value=True, numeric=key in NUMERIC_BOOLEANS)
+    if key in falses:
+        return ParsedBoolean(value=False, numeric=key in NUMERIC_BOOLEANS)
+    raise ValueError(f"{token!r} is not a recognised boolean")
+
+
+def is_boolean(token: object, **kwargs) -> bool:
+    """Return ``True`` when :func:`parse_boolean` would succeed on ``token``."""
+    try:
+        parse_boolean(token, **kwargs)
+        return True
+    except ValueError:
+        return False
+
+
+def clean_boolean(
+    values,
+    *,
+    true_values=None,
+    false_values=None,
+    null_values=None,
+) -> CleanResult:
+    """Clean a column of messy boolean strings into Python ``bool`` values.
+
+    Args:
+        values: The column to clean.
+        true_values: Override truthy vocabulary (see :func:`parse_boolean`).
+        false_values: Override falsy vocabulary.
+        null_values: Null spellings (see :func:`typemonkey.nulls.is_null`);
+            recognised nulls become ``None`` and are not counted as failures.
+
+    Returns:
+        A :class:`CleanResult` whose ``values`` align 1:1 with ``values`` and
+        whose ``failures`` records ``(index, original)`` for non-null tokens
+        that matched neither vocabulary.
+    """
+    out: list[object] = []
+    failures: list[tuple[int, str]] = []
+    null_count = 0
+    for i, raw in enumerate(values):
+        if is_null(raw, null_values):
+            out.append(None)
+            null_count += 1
+            continue
+        try:
+            parsed = parse_boolean(raw, true_values=true_values, false_values=false_values)
+        except ValueError:
+            out.append(None)
+            failures.append((i, str(raw)))
+            continue
+        out.append(parsed.value)
+    return CleanResult(
+        values=out,
+        target_type=TypeName.BOOLEAN,
+        null_count=null_count,
+        failures=failures,
+    )

typemonkey/clean.py

@@ -0,0 +1,162 @@
+"""``clean_column`` — infer (or accept) a target type, then clean to it.
+
+This module exists as the one-call convenience entry point: hand it a column
+and it figures out the type and returns cleaned values, or pass ``target_type``
+to force the conversion. It dispatches to the type-specific cleaners in
+:mod:`typemonkey.numbers` and :mod:`typemonkey.booleans`, and to datemonkey for
+dates.
+"""
+
+from __future__ import annotations
+
+import cleanmonkey
+from datemonkey import parse_dates
+
+from .booleans import clean_boolean
+from .infer import infer_type
+from .locale import detect_number_locale
+from .models import CleanResult, TypeName
+from .nulls import is_null
+from .numbers import clean_numeric
+
+_NUMERIC = {TypeName.INTEGER, TypeName.FLOAT, TypeName.CURRENCY, TypeName.PERCENTAGE}
+
+
+def clean_column(
+    values,
+    *,
+    target_type: TypeName | str | None = None,
+    null_values=None,
+    locale: str | None = None,
+    locale_preference: str | None = None,
+    true_values=None,
+    false_values=None,
+    percent_as_fraction: bool = True,
+    integers: bool = True,
+) -> CleanResult:
+    """Clean a column to ``target_type``, inferring the type when not given.
+
+    ``values`` is materialised once up front, so a one-shot iterable
+    (generator) is safe: inference and cleaning see the same data, and counts
+    stay consistent.
+
+    Args:
+        values: The column to clean. Consumed exactly once.
+        target_type: A :class:`TypeName` (or its string value) to force. When
+            ``None`` the type is inferred via :func:`typemonkey.infer.infer_type`
+            and that result — including its detected ``locale`` — drives the
+            cleaning.
+        null_values: Null spellings (see :func:`typemonkey.nulls.is_null`).
+        locale: Number locale for numeric targets. When ``None`` it is taken
+            from inference (auto-detected from the data), or detected directly
+            for a forced numeric ``target_type``.
+        locale_preference: ``"us"``/``"eu"`` hint for date parsing.
+        true_values: Override truthy boolean vocabulary.
+        false_values: Override falsy boolean vocabulary.
+        percent_as_fraction: Scale percents by 1/100 (see
+            :func:`typemonkey.numbers.clean_numeric`).
+        integers: Return whole numbers as ``int`` (see
+            :func:`typemonkey.numbers.clean_numeric`).
+
+    Returns:
+        A :class:`CleanResult` whose ``values`` align 1:1 with ``values`` and
+        whose ``target_type`` echoes the inferred or forced type. For ``STRING``
+        targets every non-null value is cleanmonkey-normalised and never a
+        failure; for ``NULL`` every value is ``None`` and ``null_count`` equals
+        the input length.
+    """
+    vals = list(values)  # materialise once — safe for generators
+
+    inferred_locale: str | None = None
+    if target_type is None:
+        profile = infer_type(
+            vals,
+            null_values=null_values,
+            locale=locale,
+            locale_preference=locale_preference,
+            true_values=true_values,
+            false_values=false_values,
+        )
+        target = profile.type
+        inferred_locale = profile.locale
+    else:
+        target = TypeName(target_type)
+
+    if target in _NUMERIC:
+        # Honour an explicit locale, else the one inference detected, else
+        # detect it directly (forced numeric target with no inference step).
+        effective_locale = locale or inferred_locale or detect_number_locale(vals)
+        return clean_numeric(
+            vals,
+            locale=effective_locale,
+            null_values=null_values,
+            percent_as_fraction=percent_as_fraction,
+            integers=integers,
+            target_type=target,
+        )
+    if target is TypeName.BOOLEAN:
+        return clean_boolean(
+            vals,
+            true_values=true_values,
+            false_values=false_values,
+            null_values=null_values,
+        )
+    if target is TypeName.DATE:
+        return _clean_dates(vals, null_values, locale_preference)
+    if target is TypeName.NULL:
+        return CleanResult(
+            values=[None] * len(vals),
+            target_type=TypeName.NULL,
+            null_count=len(vals),
+        )
+    return _clean_strings(vals, null_values)
+
+
+def _clean_dates(values, null_values, locale_preference) -> CleanResult:
+    """Parse a date column via datemonkey, preserving null/failure distinction."""
+    vals = list(values)
+    null_mask = [is_null(v, null_values) for v in vals]
+    # datemonkey treats None as a null/failure; feed it None for our nulls so
+    # the row indices line up.
+    feed = [None if null_mask[i] else vals[i] for i in range(len(vals))]
+    batch = parse_dates(feed, locale_preference=locale_preference)
+    # datemonkey returns an *empty* ``results`` list when no value in the batch
+    # parses, and otherwise one result per row tagged with ``row_index``. Map by
+    # ``row_index`` and walk our own indices so output stays 1:1 with the input
+    # even when zero values parsed (every non-null then becomes a failure).
+    by_index = {r.row_index: r for r in batch.results}
+    out: list[object] = []
+    failures: list[tuple[int, str]] = []
+    for i in range(len(vals)):
+        if null_mask[i]:
+            out.append(None)
+            continue
+        result = by_index.get(i)
+        if result is not None and result.parsed is not None:
+            out.append(result.parsed)
+        else:
+            out.append(None)
+            failures.append((i, str(vals[i])))
+    return CleanResult(
+        values=out,
+        target_type=TypeName.DATE,
+        null_count=sum(null_mask),
+        failures=failures,
+    )
+
+
+def _clean_strings(values, null_values) -> CleanResult:
+    """Normalise a string column with cleanmonkey; nulls become ``None``."""
+    out: list[object] = []
+    null_count = 0
+    for v in values:
+        if is_null(v, null_values):
+            out.append(None)
+            null_count += 1
+        else:
+            out.append(cleanmonkey.clean(str(v)))
+    return CleanResult(
+        values=out,
+        target_type=TypeName.STRING,
+        null_count=null_count,
+    )

typemonkey/cli.py

@@ -0,0 +1,100 @@
+"""Thin CLI wrapper around the typemonkey library.
+
+This module exists only to parse arguments, read input columns, and format
+output; all logic lives in the library. Commands:
+
+* ``typemonkey profile`` — infer a column's type and print a JSON report.
+* ``typemonkey clean`` — clean a column to an inferred or given type.
+
+Input is read one value per line from a file argument or stdin.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from typing import Sequence
+
+from .clean import clean_column
+from .infer import infer_type
+from .models import TypeName, _jsonable
+
+
+def _read_values(path: str | None) -> list[str]:
+    """Read one value per line from ``path`` (or stdin when ``None``/``"-"``)."""
+    if path in (None, "-"):
+        text = sys.stdin.read()
+    else:
+        with open(path, "r", encoding="utf-8") as fh:
+            text = fh.read()
+    # Strip only the trailing newline of the file, then split on line breaks so
+    # genuinely empty lines survive as empty-string values.
+    text = text.rstrip("\n")
+    return text.split("\n") if text != "" else []
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="typemonkey",
+        description="Infer column types and clean values to those types.",
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    p_profile = sub.add_parser("profile", help="Infer a column's type (JSON report).")
+    p_profile.add_argument("file", nargs="?", default="-", help="Input file, or - for stdin.")
+    p_profile.add_argument("--threshold", type=float, default=None, help="Conformance threshold (0-1).")
+    p_profile.add_argument("--locale", choices=["us", "eu"], default=None, help="Force number locale.")
+
+    p_clean = sub.add_parser("clean", help="Clean a column to a type (one value per line).")
+    p_clean.add_argument("file", nargs="?", default="-", help="Input file, or - for stdin.")
+    p_clean.add_argument(
+        "--type",
+        choices=[t.value for t in TypeName],
+        default=None,
+        help="Force target type instead of inferring.",
+    )
+    p_clean.add_argument("--locale", choices=["us", "eu"], default=None, help="Force number locale.")
+    p_clean.add_argument(
+        "--keep-percent",
+        action="store_true",
+        help="Keep percents as whole numbers (12%% -> 12) instead of fractions.",
+    )
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """CLI entry point. Returns a process exit code."""
+    parser = _build_parser()
+    args = parser.parse_args(argv)
+    values = _read_values(args.file)
+
+    if args.command == "profile":
+        kwargs = {}
+        if args.threshold is not None:
+            kwargs["threshold"] = args.threshold
+        if args.locale is not None:
+            kwargs["locale"] = args.locale
+        profile = infer_type(values, **kwargs)
+        json.dump(profile.to_dict(), sys.stdout, indent=2, sort_keys=True)
+        sys.stdout.write("\n")
+        return 0
+
+    if args.command == "clean":
+        result = clean_column(
+            values,
+            target_type=args.type,
+            locale=args.locale,
+            percent_as_fraction=not args.keep_percent,
+        )
+        for value in result.values:
+            jv = _jsonable(value)
+            sys.stdout.write("" if jv is None else str(jv))
+            sys.stdout.write("\n")
+        return 1 if result.failures else 0
+
+    return 2  # pragma: no cover - argparse enforces a valid command
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())

typemonkey/detectors/__init__.py

@@ -0,0 +1,16 @@
+"""Pluggable per-column type detectors.
+
+Each detector is a single exported function taking a list of non-null values
+and returning a :class:`Detection`. They are intentionally independent so a
+caller (or :mod:`typemonkey.infer`) can run a subset, and so new strategies
+can be added without touching the orchestrator.
+"""
+
+from __future__ import annotations
+
+from .base import Detection
+from .boolean import detect_boolean
+from .date import detect_date
+from .numeric import detect_numeric
+
+__all__ = ["Detection", "detect_boolean", "detect_date", "detect_numeric"]

typemonkey/detectors/base.py

@@ -0,0 +1,28 @@
+"""Shared result type for detectors."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class Detection:
+    """The outcome of running one detector over a column's non-null values.
+
+    Attributes:
+        match_count: Number of non-null values the detector recognised.
+        sample_size: Number of non-null values examined.
+        confidence: ``match_count / sample_size`` in ``[0.0, 1.0]``; ``0.0``
+            for an empty sample.
+        detail: Detector-specific extras (e.g. fine kind counts for numeric,
+            detected date format for date).
+    """
+
+    match_count: int
+    sample_size: int
+    detail: dict = field(default_factory=dict)
+
+    @property
+    def confidence(self) -> float:
+        """Fraction of the sample the detector matched (0.0 if empty)."""
+        return self.match_count / self.sample_size if self.sample_size else 0.0

typemonkey/detectors/boolean.py

@@ -0,0 +1,39 @@
+"""Boolean detector."""
+
+from __future__ import annotations
+
+from ..booleans import NUMERIC_BOOLEANS, is_boolean
+from .base import Detection
+
+
+def detect_boolean(values, *, true_values=None, false_values=None) -> Detection:
+    """Detect whether a column of non-null values is boolean.
+
+    Args:
+        values: Non-null values to test.
+        true_values: Override truthy vocabulary (see
+            :func:`typemonkey.booleans.parse_boolean`).
+        false_values: Override falsy vocabulary.
+
+    Returns:
+        A :class:`Detection`. ``detail["numeric_only"]`` is ``True`` when every
+        matched value was ``"0"``/``"1"`` — the ambiguous case that is equally
+        valid integer data, which the orchestrator uses to avoid labelling a
+        0/1 integer column as boolean. ``detail["distinct"]`` holds the count
+        of distinct normalised tokens matched.
+    """
+    matched = 0
+    distinct: set[str] = set()
+    numeric_only = True
+    for v in values:
+        if is_boolean(v, true_values=true_values, false_values=false_values):
+            matched += 1
+            key = str(v).strip().lower()
+            distinct.add(key)
+            if key not in NUMERIC_BOOLEANS:
+                numeric_only = False
+    return Detection(
+        match_count=matched,
+        sample_size=len(values),
+        detail={"numeric_only": numeric_only, "distinct": len(distinct)},
+    )

typemonkey/detectors/date.py

@@ -0,0 +1,53 @@
+"""Date detector — delegates to datemonkey.
+
+typemonkey does not reimplement date parsing. This detector is a thin adapter
+around :func:`datemonkey.detect_format` that reports its batch confidence in
+typemonkey's :class:`Detection` shape.
+"""
+
+from __future__ import annotations
+
+from datemonkey import Confidence, detect_format
+
+from .base import Detection
+
+# Map datemonkey's categorical confidence onto a fraction of the sample we are
+# willing to treat as matched. FAILED means no date format fit at all.
+_CONFIDENCE_WEIGHT = {
+    Confidence.HIGH: 1.0,
+    Confidence.MEDIUM: 0.75,
+    Confidence.LOW: 0.4,
+    Confidence.FAILED: 0.0,
+}
+
+
+def detect_date(values, *, locale_preference: str | None = None) -> Detection:
+    """Detect whether a column of non-null values is a date column.
+
+    Args:
+        values: Non-null values to test.
+        locale_preference: ``"us"`` or ``"eu"`` hint forwarded to datemonkey to
+            resolve DD/MM vs MM/DD ambiguity.
+
+    Returns:
+        A :class:`Detection` whose ``match_count`` is datemonkey's
+        ``match_count`` and whose ``detail`` carries the detected ``format``
+        (strftime pattern or ``None``), the categorical ``confidence`` value,
+        and any ``ambiguities`` datemonkey reported.
+    """
+    if not values:
+        return Detection(
+            0, 0, {"format": None, "confidence": "failed", "weight": 0.0, "ambiguities": []}
+        )
+    result = detect_format(values, locale_preference=locale_preference)
+    fmt = result.format.pattern if result.format is not None else None
+    return Detection(
+        match_count=result.match_count,
+        sample_size=result.sample_size,
+        detail={
+            "format": fmt,
+            "confidence": result.confidence.value,
+            "weight": _CONFIDENCE_WEIGHT.get(result.confidence, 0.0),
+            "ambiguities": [a.value for a in result.ambiguities],
+        },
+    )

typemonkey/detectors/numeric.py

@@ -0,0 +1,46 @@
+"""Numeric detector — classifies each value into a fine numeric kind."""
+
+from __future__ import annotations
+
+from collections import Counter
+
+from ..numbers import parse_number
+from .base import Detection
+
+
+def detect_numeric(values, *, locale: str = "us") -> Detection:
+    """Detect numeric values and bucket each into a fine kind.
+
+    Every value is run through :func:`typemonkey.numbers.parse_number`. Each
+    success is tallied into exactly one of ``currency``, ``percentage``,
+    ``integer``, or ``float`` (currency and percentage take precedence over
+    the plain int/float distinction when their marker is present).
+
+    Args:
+        values: Non-null values to test.
+        locale: Number locale for parsing (``"us"`` or ``"eu"``).
+
+    Returns:
+        A :class:`Detection` whose ``match_count`` is the count of values that
+        parsed as numbers and whose ``detail["kinds"]`` is a
+        :class:`collections.Counter` over the fine kinds.
+    """
+    kinds: Counter[str] = Counter()
+    for v in values:
+        try:
+            p = parse_number(v, locale)
+        except ValueError:
+            continue
+        if p.had_currency:
+            kinds["currency"] += 1
+        elif p.had_percent:
+            kinds["percentage"] += 1
+        elif p.is_integer:
+            kinds["integer"] += 1
+        else:
+            kinds["float"] += 1
+    return Detection(
+        match_count=sum(kinds.values()),
+        sample_size=len(values),
+        detail={"kinds": kinds, "locale": locale},
+    )