cleanmonkey 0.1.0__py3-none-any.whl

cleanmonkey/__init__.py

@@ -0,0 +1,7 @@
+"""cleanmonkey — one-call text cleanup for invisible characters, smart quotes, and whitespace."""
+
+from cleanmonkey.core import MAX_DEPTH, clean, clean_column, clean_dict, inspect
+from cleanmonkey.profiles import PROFILES, Profile
+
+__version__ = "0.1.0"
+__all__ = ["MAX_DEPTH", "clean", "clean_column", "clean_dict", "inspect", "Profile", "PROFILES"]

cleanmonkey/__main__.py

@@ -0,0 +1,5 @@
+"""Allow running cleanmonkey as ``python -m cleanmonkey``."""
+
+from cleanmonkey.cli import _main_with_broken_pipe_handling
+
+_main_with_broken_pipe_handling()

cleanmonkey/cli.py

@@ -0,0 +1,400 @@
+"""CLI entry point for cleanmonkey."""
+
+from __future__ import annotations
+
+import argparse
+import contextlib
+import json
+import os
+import shutil
+import stat
+import sys
+import tempfile
+from typing import Any, TextIO
+
+from cleanmonkey.core import clean, inspect
+from cleanmonkey.profiles import PROFILES
+
+
+def _fsync_directory(dir_path: str) -> None:
+    """Best-effort fsync of a directory for rename durability on POSIX.
+
+    Silently ignores errors when directory fsync is unsupported (e.g. Windows,
+    certain filesystems).  Emits a warning to stderr for unexpected I/O errors
+    that may indicate a real durability problem.
+    """
+    import errno
+    # Errors that indicate "not supported here" rather than a real failure.
+    # EACCES/EBADF are NOT included: they may indicate real permission issues
+    # on the directory that should surface as warnings.
+    _UNSUPPORTED_ERRNOS = {
+        errno.ENOTSUP, errno.EOPNOTSUPP, errno.ENOSYS, errno.EINVAL,
+    }
+    try:
+        fd = os.open(dir_path, os.O_RDONLY)
+        try:
+            os.fsync(fd)
+        finally:
+            os.close(fd)
+    except OSError as exc:
+        if exc.errno not in _UNSUPPORTED_ERRNOS:
+            print(
+                f"cleanmonkey: warning: directory fsync failed for "
+                f"{dir_path!r}: {exc}",
+                file=sys.stderr,
+            )
+        # Never fatal — durability is best-effort.
+
+
+def _open_streams(
+    file_path: str | None,
+    output_path: str | None,
+) -> contextlib.AbstractContextManager[tuple[TextIO, TextIO, str | None]]:
+    """Open input/output streams with explicit ownership.
+
+    When a path is provided (and is not ``"-"``), the file is opened and will
+    be closed when the context manager exits.  ``stdin``/``stdout`` are
+    *borrowed* – never closed by this function.
+
+    When *output_path* refers to a real file, writes go to a temporary file in
+    the same directory.  The caller receives the temp path as the third element
+    of the yielded tuple so it can be atomically renamed to the final
+    destination **after** processing succeeds.  On failure, the temp file is
+    removed and the original destination is untouched.
+    """
+
+    @contextlib.contextmanager
+    def _ctx():
+        in_stream: TextIO | None = None
+        out_stream: TextIO | None = None
+        tmp_path: str | None = None
+        # Resolve symlinks so os.replace() writes through to the real
+        # destination instead of replacing the symlink itself.
+        resolved_output = (
+            os.path.realpath(output_path)
+            if output_path is not None and output_path != "-"
+            else output_path
+        )
+        success = False
+        try:
+            if file_path is None or file_path == "-":
+                in_stream = sys.stdin
+            else:
+                in_stream = open(file_path, "r", encoding="utf-8", newline="")
+
+            if resolved_output is None or resolved_output == "-":
+                out_stream = sys.stdout
+            else:
+                # Write to a temp file in the same directory so os.replace()
+                # is atomic on the same filesystem.
+                out_dir = os.path.dirname(os.path.abspath(resolved_output))
+                fd, tmp_path = tempfile.mkstemp(
+                    dir=out_dir, prefix=".cleanmonkey_", suffix=".tmp",
+                )
+                out_stream = open(fd, "w", encoding="utf-8", newline="")
+
+            yield in_stream, out_stream, tmp_path
+            success = True
+        finally:
+            # Only close streams we opened (not stdin/stdout).
+            if in_stream is not None and in_stream is not sys.stdin:
+                in_stream.close()
+            finalize_error: OSError | None = None
+            if out_stream is not None and out_stream is not sys.stdout:
+                try:
+                    # Flush to OS and fsync for durability before atomic rename.
+                    if tmp_path is not None and success:
+                        out_stream.flush()
+                        os.fsync(out_stream.fileno())
+                except OSError as exc:
+                    # Flush/fsync failed — treat as unsuccessful so we clean up.
+                    success = False
+                    finalize_error = exc
+                finally:
+                    out_stream.close()
+            # Clean up temp file on failure; promote on success.
+            if tmp_path is not None:
+                if success and resolved_output is not None:
+                    try:
+                        # Preserve original file metadata if destination exists.
+                        dest_existed = os.path.exists(resolved_output)
+                        if dest_existed:
+                            try:
+                                shutil.copystat(resolved_output, tmp_path)
+                            except OSError:
+                                # copystat failed (e.g. unsupported metadata).
+                                # Fall back to preserving at least the file mode
+                                # so os.replace() doesn't leave mkstemp's 0600.
+                                try:
+                                    orig_mode = os.stat(resolved_output).st_mode
+                                    os.chmod(tmp_path, stat.S_IMODE(orig_mode))
+                                except OSError as perm_exc:
+                                    print(
+                                        f"cleanmonkey: warning: could not preserve "
+                                        f"permissions for {resolved_output!r}: {perm_exc}",
+                                        file=sys.stderr,
+                                    )
+                                else:
+                                    print(
+                                        f"cleanmonkey: warning: metadata preservation "
+                                        f"partially failed for {resolved_output!r}; "
+                                        f"file mode preserved",
+                                        file=sys.stderr,
+                                    )
+                        else:
+                            # mkstemp creates files with mode 0600; apply
+                            # umask-derived default so new files behave like
+                            # a normal open() would (typically 0644).
+                            # Apply to temp file BEFORE replace so that a
+                            # chmod failure cannot leave a replaced destination.
+                            umask = os.umask(0)
+                            os.umask(umask)
+                            os.chmod(tmp_path, 0o666 & ~umask)
+                        os.replace(tmp_path, resolved_output)
+                        _fsync_directory(os.path.dirname(os.path.abspath(resolved_output)))
+                    except OSError:
+                        # replace or dir fsync failed — clean up temp file.
+                        try:
+                            os.unlink(tmp_path)
+                        except OSError:
+                            pass
+                        raise
+                else:
+                    try:
+                        os.unlink(tmp_path)
+                    except OSError:
+                        pass
+            # Re-raise flush/fsync error after cleanup so callers know it failed.
+            if finalize_error is not None:
+                raise finalize_error
+
+    return _ctx()
+
+
+def _non_negative_int(value: str) -> int:
+    """Argparse type for non-negative integers."""
+    try:
+        n = int(value)
+    except ValueError:
+        raise argparse.ArgumentTypeError(f"invalid int value: {value!r}")
+    if n < 0:
+        raise argparse.ArgumentTypeError(f"must be non-negative, got {n}")
+    return n
+
+
+def main(argv: list[str] | None = None) -> None:
+    parser = argparse.ArgumentParser(
+        prog="cleanmonkey",
+        description="Clean invisible characters, smart quotes, and whitespace from text.",
+    )
+    parser.add_argument(
+        "file",
+        nargs="?",
+        default=None,
+        help="Input file (default: stdin, use '-' for stdin)",
+    )
+    parser.add_argument(
+        "-o", "--output",
+        default=None,
+        help="Output file (default: stdout, use '-' for stdout)",
+    )
+    parser.add_argument(
+        "-p", "--profile",
+        choices=sorted(PROFILES),
+        default="default",
+        help="Cleaning profile (default: default)",
+    )
+    parser.add_argument(
+        "--inspect",
+        action="store_true",
+        dest="inspect_mode",
+        help="Inspect mode: report problematic characters instead of cleaning. "
+             "Reports character-map replacements only; structural changes like "
+             "space collapsing and per-line stripping are not reported.",
+    )
+    parser.add_argument(
+        "--no-smart-quotes", action="store_true", help="Disable smart quote normalization",
+    )
+    parser.add_argument(
+        "--no-dashes", action="store_true", help="Disable dash normalization",
+    )
+    parser.add_argument(
+        "--fullwidth", action="store_true",
+        help="Enable fullwidth ASCII letter and digit normalization "
+             "(e.g. \uff21\u2192A, \uff10\u21920; fullwidth punctuation is not covered)",
+    )
+    parser.add_argument(
+        "--no-line-endings", action="store_true",
+        help="Disable line-ending normalization (also disables CR reporting in inspect mode)",
+    )
+    parser.add_argument(
+        "--no-strip", action="store_true",
+        help="Disable stripping of leading/trailing whitespace per line",
+    )
+    parser.add_argument(
+        "--no-collapse-spaces", action="store_true",
+        help="Disable collapsing of multiple spaces into one",
+    )
+    parser.add_argument(
+        "--json", action="store_true",
+        help="Output inspect results as JSON (implies --inspect)",
+    )
+    parser.add_argument(
+        "--max-positions", type=_non_negative_int, default=None, metavar="N",
+        help="Limit position lists in inspect output to at most N entries "
+             "(count is always accurate). Useful for large files.",
+    )
+    parser.add_argument(
+        "--stream", action="store_true",
+        help="Process input line-by-line instead of loading it all into memory. "
+             "Suitable for very large files. Ignored in inspect mode.",
+    )
+    parser.add_argument(
+        "--version", action="version", version=f"%(prog)s {_get_version()}",
+    )
+
+    args = parser.parse_args(argv)
+
+    # --json implies --inspect.
+    if args.json:
+        args.inspect_mode = True
+
+    # Warn if --stream is used with --inspect (inspect needs full text).
+    if args.stream and args.inspect_mode:
+        print(
+            "cleanmonkey: warning: --stream is ignored in inspect mode",
+            file=sys.stderr,
+        )
+
+    # Guard against same input/output path (would truncate source before read).
+    if (
+        args.file is not None
+        and args.file != "-"
+        and args.output is not None
+        and args.output != "-"
+    ):
+        try:
+            if os.path.samefile(args.file, args.output):
+                parser.error("input and output paths refer to the same file; this would cause data loss")
+        except FileNotFoundError:
+            # Output file doesn't exist yet – that's fine, no collision.
+            pass
+        except OSError as exc:
+            parser.error(
+                f"cannot compare input {args.file!r} and output {args.output!r}: {exc}"
+            )
+
+    try:
+        _run(parser, args)
+    except (OSError, ValueError) as exc:
+        parser.error(str(exc))
+
+
+def _run(parser: argparse.ArgumentParser, args: argparse.Namespace) -> None:
+    """Execute the main logic, allowing OSError to propagate to the caller."""
+    with _open_streams(args.file, args.output) as (in_stream, out_stream, _tmp):
+        # Build clean overrides (shared by buffered and streaming paths).
+        overrides: dict[str, bool] = {}
+        if args.no_smart_quotes:
+            overrides["smart_quotes"] = False
+        if args.no_dashes:
+            overrides["dashes"] = False
+        if args.fullwidth:
+            overrides["fullwidth"] = True
+        if args.no_line_endings:
+            overrides["line_endings"] = False
+        if args.no_strip:
+            overrides["strip"] = False
+        if args.no_collapse_spaces:
+            overrides["collapse_spaces"] = False
+
+        # --stream: line-by-line processing for clean mode (not inspect).
+        if args.stream and not args.inspect_mode:
+            try:
+                for line in in_stream:
+                    out_stream.write(clean(line, profile=args.profile, **overrides))
+            except UnicodeDecodeError:
+                parser.error(f"cannot decode {args.file or '<stdin>'!r}: file is not valid UTF-8")
+            return
+
+        try:
+            text = in_stream.read()
+        except UnicodeDecodeError:
+            parser.error(f"cannot decode {args.file or '<stdin>'!r}: file is not valid UTF-8")
+
+        if args.inspect_mode:
+            inspect_kw: dict[str, Any] = {"profile": args.profile}
+            if args.fullwidth:
+                inspect_kw["fullwidth"] = True
+            if args.no_line_endings:
+                inspect_kw["line_endings"] = False
+            if args.max_positions is not None:
+                inspect_kw["max_positions"] = args.max_positions
+            findings = inspect(text, **inspect_kw)
+            if args.json:
+                json.dump(
+                    [
+                        {
+                            "char": info.char,
+                            "codepoint": info.codepoint,
+                            "name": info.name,
+                            "category": info.category,
+                            "count": info.count,
+                            "positions": info.positions,
+                        }
+                        for info in findings
+                    ],
+                    out_stream,
+                    ensure_ascii=False,
+                )
+                out_stream.write("\n")
+                return
+            if not findings:
+                print("No problematic characters found.", file=out_stream)
+                return
+            for info in findings:
+                # When max_positions is set, inspect() already truncated;
+                # otherwise apply a default cap of 10 for text readability.
+                if args.max_positions is not None:
+                    shown = info.positions
+                else:
+                    shown = info.positions[:10]
+                truncated = len(shown) < info.count
+                print(
+                    f"{info.codepoint} {info.name} (count: {info.count}, "
+                    f"positions: {shown}{'...' if truncated else ''})",
+                    file=out_stream,
+                )
+            return
+
+        result = clean(text, profile=args.profile, **overrides)
+        out_stream.write(result)
+
+
+def _get_version() -> str:
+    from cleanmonkey import __version__
+    return __version__
+
+
+def _main_with_broken_pipe_handling() -> None:
+    """Entry point that handles BrokenPipeError for pipeline-friendly behavior."""
+    try:
+        main()
+    except BrokenPipeError:
+        # Suppress noisy traceback when downstream consumer closes early
+        # (e.g., `cleanmonkey file.txt | head -n1`).
+        # Flush stderr and restore default SIGPIPE behavior for clean exit.
+        try:
+            sys.stdout.close()
+        except BrokenPipeError:
+            pass
+        try:
+            sys.stderr.close()
+        except BrokenPipeError:
+            pass
+        # Exit with the conventional signal code for SIGPIPE (128 + 13 = 141)
+        sys.exit(141)
+
+
+if __name__ == "__main__":
+    _main_with_broken_pipe_handling()

cleanmonkey/core.py

@@ -0,0 +1,449 @@
+"""Core cleaning functions."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, replace
+from typing import Any
+
+_BOOL_OVERRIDE_NAMES_CLEAN = (
+    "smart_quotes", "dashes", "ellipsis", "invisible", "whitespace",
+    "control", "fullwidth", "line_endings", "collapse_spaces", "strip",
+)
+_BOOL_OVERRIDE_NAMES_INSPECT = ("fullwidth", "line_endings")
+
+
+def _validate_bool_overrides(overrides: dict[str, Any], allowed: tuple[str, ...], func_name: str) -> None:
+    """Raise TypeError if any override value is not None or bool, or if unknown keys are present."""
+    unknown = set(overrides) - set(allowed)
+    if unknown:
+        raise TypeError(
+            f"{func_name}() got unexpected keyword argument(s): {', '.join(sorted(unknown))}"
+        )
+    for name in allowed:
+        val = overrides.get(name)
+        if val is not None and not isinstance(val, bool):
+            raise TypeError(
+                f"{func_name}() override {name!r} must be bool or None, got {type(val).__name__}"
+            )
+
+from cleanmonkey.maps import (
+    CONTROL,
+    DASHES,
+    ELLIPSIS,
+    FULLWIDTH,
+    INVISIBLE,
+    SMART_QUOTES,
+    WHITESPACE,
+)
+from cleanmonkey.profiles import PROFILES, Profile
+
+
+def _validate_profile_kwarg(kwargs: dict[str, Any], func_name: str) -> None:
+    """Validate the 'profile' kwarg type and name if present, matching clean()'s contract."""
+    if "profile" in kwargs:
+        p = kwargs["profile"]
+        if isinstance(p, str):
+            if p not in PROFILES:
+                raise ValueError(
+                    f"Unknown profile {p!r}. Available: {', '.join(sorted(PROFILES))}"
+                )
+        elif not isinstance(p, Profile):
+            raise TypeError(
+                f"{func_name}() profile must be str or Profile, got {type(p).__name__}"
+            )
+
+_MULTI_SPACE = re.compile(r" {2,}")
+
+#: Maximum nesting depth for recursive cleaners (clean_dict, clean_column).
+#: Kept well below half of Python's default recursion limit (1000) since each
+#: nesting level consumes multiple Python frames.
+MAX_DEPTH: int = 200
+
+
+def _build_table(profile: Profile) -> dict[int, str | int | None]:
+    """Build a str.translate table from a profile."""
+    merged: dict[str, str] = {}
+    if profile.invisible:
+        merged.update(INVISIBLE)
+    if profile.whitespace:
+        merged.update(WHITESPACE)
+    if profile.control:
+        merged.update(CONTROL)
+    if profile.smart_quotes:
+        merged.update(SMART_QUOTES)
+    if profile.dashes:
+        merged.update(DASHES)
+    if profile.ellipsis:
+        merged.update(ELLIPSIS)
+    if profile.fullwidth:
+        merged.update(FULLWIDTH)
+    return str.maketrans({k: v for k, v in merged.items()})
+
+
+# Cache tables for default profiles
+_TABLE_CACHE: dict[str, dict[int, str | int | None]] = {}
+
+
+def _get_table(profile: Profile, profile_name: str | None = None) -> dict[int, str | int | None]:
+    if profile_name and profile_name in _TABLE_CACHE:
+        return _TABLE_CACHE[profile_name]
+    table = _build_table(profile)
+    if profile_name:
+        _TABLE_CACHE[profile_name] = table
+    return table
+
+
+def clean(
+    text: str,
+    *,
+    profile: str | Profile = "default",
+    smart_quotes: bool | None = None,
+    dashes: bool | None = None,
+    ellipsis: bool | None = None,
+    invisible: bool | None = None,
+    whitespace: bool | None = None,
+    control: bool | None = None,
+    fullwidth: bool | None = None,
+    line_endings: bool | None = None,
+    collapse_spaces: bool | None = None,
+    strip: bool | None = None,
+) -> str:
+    """Clean text with sensible defaults.
+
+    Parameters
+    ----------
+    text : str
+        The text to clean.
+    profile : str or Profile
+        Named profile or a Profile instance. Default is "default".
+    smart_quotes, dashes, ellipsis, invisible, whitespace, control,
+    fullwidth, line_endings, collapse_spaces, strip :
+        Override individual profile settings. None means use profile value.
+
+    Returns
+    -------
+    str
+        Cleaned text.
+
+    Notes
+    -----
+    When ``strip=True`` (the default profile), leading and trailing spaces
+    and tabs are removed from **each line individually**, not just the
+    overall string.  This will destroy meaningful indentation (e.g. Python,
+    YAML, Markdown code blocks).  Pass ``strip=False`` for
+    indentation-sensitive content.
+    """
+    if not isinstance(text, str):
+        raise TypeError(f"clean() expects str, got {type(text).__name__}")
+
+    # Validate override types before any other processing
+    _validate_bool_overrides(
+        {"smart_quotes": smart_quotes, "dashes": dashes, "ellipsis": ellipsis,
+         "invisible": invisible, "whitespace": whitespace, "control": control,
+         "fullwidth": fullwidth, "line_endings": line_endings,
+         "collapse_spaces": collapse_spaces, "strip": strip},
+        _BOOL_OVERRIDE_NAMES_CLEAN, "clean",
+    )
+
+    # Resolve profile (validate before early return so invalid profiles always raise)
+    if isinstance(profile, str):
+        profile_name: str | None = profile
+        if profile not in PROFILES:
+            raise ValueError(f"Unknown profile {profile!r}. Available: {', '.join(sorted(PROFILES))}")
+        base = PROFILES[profile]
+    elif isinstance(profile, Profile):
+        profile_name = None
+        base = profile
+    else:
+        raise TypeError(f"profile must be str or Profile, got {type(profile).__name__}")
+
+    if not text:
+        return text
+
+    # Apply overrides
+    overrides = {
+        k: v
+        for k, v in {
+            "smart_quotes": smart_quotes,
+            "dashes": dashes,
+            "ellipsis": ellipsis,
+            "invisible": invisible,
+            "whitespace": whitespace,
+            "control": control,
+            "fullwidth": fullwidth,
+            "line_endings": line_endings,
+            "collapse_spaces": collapse_spaces,
+            "strip": strip,
+        }.items()
+        if v is not None
+    }
+
+    if overrides:
+        p = replace(base, **overrides)
+        profile_name = None  # don't cache custom combos
+    else:
+        p = base
+
+    # Translate characters
+    table = _get_table(p, profile_name)
+    result = text.translate(table)
+
+    # Normalize line endings
+    if p.line_endings:
+        result = result.replace("\r\n", "\n").replace("\r", "\n")
+
+    # Collapse multiple spaces
+    if p.collapse_spaces:
+        result = _MULTI_SPACE.sub(" ", result)
+
+    # Strip
+    if p.strip:
+        result = "\n".join(line.strip(" \t") for line in result.split("\n"))
+
+    return result
+
+
+def _clean_value(
+    v: Any, *, keys: bool = False, _seen: set[int] | None = None, _depth: int = 0, **kwargs: Any,
+) -> Any:
+    """Recursively clean a value of any type."""
+    if isinstance(v, str):
+        return clean(v, **kwargs)
+    if isinstance(v, dict):
+        return clean_dict(v, keys=keys, _seen=_seen, _depth=_depth, **kwargs)
+    if isinstance(v, list):
+        return clean_column(v, keys=keys, _seen=_seen, _depth=_depth, **kwargs)
+    if isinstance(v, (tuple, set, frozenset)):
+        if _depth >= MAX_DEPTH:
+            raise ValueError(f"Maximum nesting depth ({MAX_DEPTH}) exceeded")
+        obj_id = id(v)
+        if _seen is None:
+            _seen = set()
+        if obj_id in _seen:
+            raise ValueError("Circular reference detected in input structure")
+        _seen.add(obj_id)
+        try:
+            cleaned_items = [
+                _clean_value(item, keys=keys, _seen=_seen, _depth=_depth + 1, **kwargs)
+                for item in v
+            ]
+            result = type(v)(cleaned_items)
+            if isinstance(v, (set, frozenset)) and len(result) != len(v):
+                raise ValueError(
+                    "Set member collision: cleaning produced duplicate members"
+                )
+            return result
+        finally:
+            _seen.discard(obj_id)
+    return v
+
+
+def clean_column(
+    values: list[Any], *, keys: bool = False, _seen: set[int] | None = None,
+    _depth: int = 0, **kwargs: Any,
+) -> list[Any]:
+    """Clean a list of values. Recursively traverses nested dicts and lists."""
+    if _depth == 0:
+        if not isinstance(values, list):
+            raise TypeError(f"clean_column() expects list, got {type(values).__name__}")
+        if not isinstance(keys, bool):
+            raise TypeError(f"clean_column() keys must be bool, got {type(keys).__name__}")
+        # Validate profile type if provided (same contract as clean()).
+        _validate_profile_kwarg(kwargs, "clean_column")
+    # Validate overrides upfront so invalid types always raise, regardless of data shape.
+    if _depth == 0:
+        # Exclude 'profile' from bool validation — it's a valid passthrough to clean().
+        bool_kwargs = {k: v for k, v in kwargs.items() if k != "profile"}
+        _validate_bool_overrides(bool_kwargs, _BOOL_OVERRIDE_NAMES_CLEAN, "clean_column")
+    if _depth >= MAX_DEPTH:
+        raise ValueError(f"Maximum nesting depth ({MAX_DEPTH}) exceeded")
+    if _seen is None:
+        _seen = set()
+    obj_id = id(values)
+    if obj_id in _seen:
+        raise ValueError("Circular reference detected in input structure")
+    _seen.add(obj_id)
+    try:
+        return [_clean_value(v, keys=keys, _seen=_seen, _depth=_depth + 1, **kwargs) for v in values]
+    finally:
+        _seen.discard(obj_id)
+
+
+def clean_dict(
+    d: dict[Any, Any], *, keys: bool = False, _seen: set[int] | None = None,
+    _depth: int = 0, **kwargs: Any,
+) -> dict[Any, Any]:
+    """Recursively clean string values in a dict.
+
+    Parameters
+    ----------
+    d : dict
+        Dictionary to clean.
+    keys : bool
+        If True, also clean dictionary keys (only str keys are cleaned).
+    **kwargs :
+        Passed to clean().
+    """
+    if _depth == 0:
+        if not isinstance(d, dict):
+            raise TypeError(f"clean_dict() expects dict, got {type(d).__name__}")
+        if not isinstance(keys, bool):
+            raise TypeError(f"clean_dict() keys must be bool, got {type(keys).__name__}")
+        # Validate profile type if provided (same contract as clean()).
+        _validate_profile_kwarg(kwargs, "clean_dict")
+    # Validate overrides upfront so invalid types always raise, regardless of data shape.
+    if _depth == 0:
+        bool_kwargs = {k: v for k, v in kwargs.items() if k != "profile"}
+        _validate_bool_overrides(bool_kwargs, _BOOL_OVERRIDE_NAMES_CLEAN, "clean_dict")
+    if _depth >= MAX_DEPTH:
+        raise ValueError(f"Maximum nesting depth ({MAX_DEPTH}) exceeded")
+    if _seen is None:
+        _seen = set()
+    obj_id = id(d)
+    if obj_id in _seen:
+        raise ValueError("Circular reference detected in input structure")
+    _seen.add(obj_id)
+    try:
+        out: dict[Any, Any] = {}
+        for k, v in d.items():
+            new_key = clean(k, **kwargs) if keys and isinstance(k, str) else k
+            if new_key in out:
+                raise ValueError(
+                    f"Key collision: {k!r} normalizes to {new_key!r} which already exists"
+                )
+            out[new_key] = _clean_value(v, keys=keys, _seen=_seen, _depth=_depth + 1, **kwargs)
+        return out
+    finally:
+        _seen.discard(obj_id)
+
+
+@dataclass(slots=True)
+class CharInfo:
+    """Information about a character found during inspection."""
+    char: str
+    codepoint: str
+    name: str
+    category: str
+    positions: list[int]
+    count: int
+
+
+def inspect(
+    text: str,
+    *,
+    profile: str | Profile = "default",
+    fullwidth: bool | None = None,
+    line_endings: bool | None = None,
+    max_positions: int | None = None,
+) -> list[CharInfo]:
+    """Inspect text for non-standard characters.
+
+    Parameters
+    ----------
+    text : str
+        The text to inspect.
+    profile : str or Profile
+        Named profile or a Profile instance.  Only character categories
+        enabled in the profile are flagged.  Default is ``"default"``.
+    fullwidth : bool or None
+        Override the profile's fullwidth setting.
+    line_endings : bool or None
+        Override the profile's line_endings setting.
+    max_positions : int or None
+        If set, limit the ``positions`` list in each :class:`CharInfo` to at
+        most this many entries.  ``count`` always reflects the true total.
+        Useful for bounding memory on very large inputs.
+
+    Returns a list of CharInfo objects describing each problematic character
+    found, sorted by first position.
+
+    Notes
+    -----
+    Positions are **character indices** (``enumerate`` over the Python
+    string), not byte offsets.  For multibyte UTF-8 characters such as
+    emoji, the character index will be smaller than the byte offset.  If
+    byte offsets are needed, convert via
+    ``len(text[:pos].encode('utf-8'))``.
+    """
+    if not isinstance(text, str):
+        raise TypeError(f"inspect() expects str, got {type(text).__name__}")
+
+    # Validate max_positions
+    if max_positions is not None:
+        if not isinstance(max_positions, int) or isinstance(max_positions, bool):
+            raise TypeError(
+                f"inspect() max_positions must be int or None, got {type(max_positions).__name__}"
+            )
+        if max_positions < 0:
+            raise ValueError(
+                f"inspect() max_positions must be non-negative, got {max_positions}"
+            )
+
+    # Validate override types
+    _validate_bool_overrides(
+        {"fullwidth": fullwidth, "line_endings": line_endings},
+        _BOOL_OVERRIDE_NAMES_INSPECT, "inspect",
+    )
+
+    # Resolve profile
+    if isinstance(profile, str):
+        if profile not in PROFILES:
+            raise ValueError(f"Unknown profile {profile!r}. Available: {', '.join(sorted(PROFILES))}")
+        p = PROFILES[profile]
+    elif isinstance(profile, Profile):
+        p = profile
+    else:
+        raise TypeError(f"profile must be str or Profile, got {type(profile).__name__}")
+
+    # Apply overrides
+    use_fullwidth = fullwidth if fullwidth is not None else p.fullwidth
+    use_line_endings = line_endings if line_endings is not None else p.line_endings
+
+    import unicodedata
+
+    # Collect only chars that the resolved profile would change
+    target_chars: set[str] = set()
+    if p.invisible:
+        target_chars.update(INVISIBLE.keys())
+    if p.whitespace:
+        target_chars.update(WHITESPACE.keys())
+    if p.control:
+        target_chars.update(CONTROL.keys())
+    if p.smart_quotes:
+        target_chars.update(SMART_QUOTES.keys())
+    if p.dashes:
+        target_chars.update(DASHES.keys())
+    if p.ellipsis:
+        target_chars.update(ELLIPSIS.keys())
+    if use_fullwidth:
+        target_chars.update(FULLWIDTH.keys())
+    if use_line_endings:
+        target_chars.add("\r")
+
+    found: dict[str, list[int]] = {}
+    counts: dict[str, int] = {}
+    first_pos: dict[str, int] = {}
+    for i, ch in enumerate(text):
+        if ch in target_chars:
+            counts[ch] = counts.get(ch, 0) + 1
+            if ch not in first_pos:
+                first_pos[ch] = i
+                found[ch] = []
+            # Only store positions up to max_positions to bound memory.
+            if max_positions is None or len(found[ch]) < max_positions:
+                found[ch].append(i)
+
+    results: list[CharInfo] = []
+    for ch in sorted(found, key=lambda c: first_pos[c]):
+        positions = found[ch]
+        total = counts[ch]
+        results.append(CharInfo(
+            char=ch,
+            codepoint=f"U+{ord(ch):04X}",
+            name=unicodedata.name(ch, f"UNKNOWN-{ord(ch):04X}"),
+            category=unicodedata.category(ch),
+            positions=positions,
+            count=total,
+        ))
+    return results

cleanmonkey/maps.py

@@ -0,0 +1,106 @@
+"""Character replacement maps used by cleanmonkey."""
+
+# Smart / curly quotes → ASCII equivalents
+SMART_QUOTES: dict[str, str] = {
+    "\u2018": "'",   # left single
+    "\u2019": "'",   # right single
+    "\u201a": "'",   # single low-9
+    "\u201b": "'",   # single high-reversed-9
+    "\u201c": '"',   # left double
+    "\u201d": '"',   # right double
+    "\u201e": '"',   # double low-9
+    "\u201f": '"',   # double high-reversed-9
+    "\u2039": "'",   # single left-pointing angle
+    "\u203a": "'",   # single right-pointing angle
+    "\u00ab": '"',   # left-pointing double angle
+    "\u00bb": '"',   # right-pointing double angle
+}
+
+# Dash-like characters → ASCII hyphen-minus
+DASHES: dict[str, str] = {
+    "\u2010": "-",   # hyphen
+    "\u2011": "-",   # non-breaking hyphen
+    "\u2012": "-",   # figure dash
+    "\u2013": "-",   # en dash
+    "\u2014": "-",   # em dash
+    "\u2015": "-",   # horizontal bar
+    "\u2212": "-",   # minus sign
+    "\ufe58": "-",   # small em dash
+    "\ufe63": "-",   # small hyphen-minus
+    "\uff0d": "-",   # fullwidth hyphen-minus
+}
+
+# Ellipsis
+ELLIPSIS: dict[str, str] = {
+    "\u2026": "...",  # horizontal ellipsis
+}
+
+# Zero-width and invisible characters → removed
+INVISIBLE: dict[str, str] = {
+    "\u200b": "",    # zero-width space
+    "\u200c": "",    # zero-width non-joiner
+    "\u200d": "",    # zero-width joiner
+    "\u200e": "",    # left-to-right mark
+    "\u200f": "",    # right-to-left mark
+    "\u2060": "",    # word joiner
+    "\u2061": "",    # function application
+    "\u2062": "",    # invisible times
+    "\u2063": "",    # invisible separator
+    "\u2064": "",    # invisible plus
+    "\ufeff": "",    # BOM / zero-width no-break space
+    "\u00ad": "",    # soft hyphen
+    "\u034f": "",    # combining grapheme joiner
+    "\u061c": "",    # Arabic letter mark
+    "\u180e": "",    # Mongolian vowel separator
+}
+
+# Whitespace-like characters → ASCII space
+WHITESPACE: dict[str, str] = {
+    "\u00a0": " ",   # non-breaking space
+    "\u1680": " ",   # ogham space mark
+    "\u2000": " ",   # en quad
+    "\u2001": " ",   # em quad
+    "\u2002": " ",   # en space
+    "\u2003": " ",   # em space
+    "\u2004": " ",   # three-per-em space
+    "\u2005": " ",   # four-per-em space
+    "\u2006": " ",   # six-per-em space
+    "\u2007": " ",   # figure space
+    "\u2008": " ",   # punctuation space
+    "\u2009": " ",   # thin space
+    "\u200a": " ",   # hair space
+    "\u202f": " ",   # narrow no-break space
+    "\u205f": " ",   # medium mathematical space
+    "\u3000": " ",   # ideographic space
+}
+
+# Control characters (C0/C1) to remove, excluding \t \n \r which are handled separately
+CONTROL: dict[str, str] = {
+    "\x00": "",  # null
+    "\x01": "",  "\x02": "",  "\x03": "",  "\x04": "",
+    "\x05": "",  "\x06": "",  "\x07": "",  "\x08": "",
+    "\x0b": "",  # vertical tab
+    "\x0c": "",  # form feed
+    "\x0e": "",  "\x0f": "",
+    "\x10": "",  "\x11": "",  "\x12": "",  "\x13": "",
+    "\x14": "",  "\x15": "",  "\x16": "",  "\x17": "",
+    "\x18": "",  "\x19": "",  "\x1a": "",  "\x1b": "",
+    "\x1c": "",  "\x1d": "",  "\x1e": "",  "\x1f": "",
+    "\x7f": "",  # DEL
+    # C1 control characters (U+0080–U+009F)
+    **{chr(i): "" for i in range(0x80, 0xA0)},
+}
+
+# Fullwidth ASCII digits → normal digits
+FULLWIDTH_DIGITS: dict[str, str] = {
+    chr(0xFF10 + i): str(i) for i in range(10)
+}
+
+# Fullwidth ASCII letters → normal letters
+FULLWIDTH_LETTERS: dict[str, str] = {
+    **{chr(0xFF21 + i): chr(0x41 + i) for i in range(26)},  # A-Z
+    **{chr(0xFF41 + i): chr(0x61 + i) for i in range(26)},  # a-z
+}
+
+# Combined fullwidth map
+FULLWIDTH: dict[str, str] = {**FULLWIDTH_DIGITS, **FULLWIDTH_LETTERS}

cleanmonkey/profiles.py

@@ -0,0 +1,57 @@
+"""Preset cleaning profiles for common use cases."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True, slots=True)
+class Profile:
+    """Configuration for which normalizations to apply."""
+
+    smart_quotes: bool = True
+    dashes: bool = True
+    ellipsis: bool = True
+    invisible: bool = True
+    whitespace: bool = True
+    control: bool = True
+    fullwidth: bool = False
+    line_endings: bool = True        # normalize \r\n and \r to \n
+    collapse_spaces: bool = True     # multiple spaces → single space
+    strip: bool = True               # strip leading/trailing whitespace per line
+
+
+# Named profiles
+PROFILES: dict[str, Profile] = {
+    "default": Profile(),
+    "csv": Profile(
+        fullwidth=True,
+    ),
+    "sql": Profile(
+        fullwidth=True,
+    ),
+    "display": Profile(
+        smart_quotes=False,
+        dashes=False,
+        ellipsis=False,
+        fullwidth=False,
+    ),
+    "minimal": Profile(
+        smart_quotes=False,
+        dashes=False,
+        ellipsis=False,
+        invisible=True,
+        whitespace=False,
+        control=False,
+        fullwidth=False,
+        line_endings=False,
+        collapse_spaces=False,
+        strip=False,
+    ),
+    # "aggressive" enables every available normalization, including fullwidth.
+    # It is intentionally equivalent to the default profile with fullwidth=True,
+    # providing a semantic alias for pipelines that want maximum cleaning.
+    "aggressive": Profile(
+        fullwidth=True,
+    ),
+}

cleanmonkey-0.1.0.dist-info/METADATA

@@ -0,0 +1,152 @@
+Metadata-Version: 2.4
+Name: cleanmonkey
+Version: 0.1.0
+Summary: One-call text cleanup: invisible characters, smart quotes, whitespace normalization.
+Author-email: RexBytes <pythonic@rexbytes.com>
+License: MIT
+Project-URL: Homepage, https://github.com/RexBytes/cleanmonkey
+Project-URL: Repository, https://github.com/RexBytes/cleanmonkey
+Project-URL: Issues, https://github.com/RexBytes/cleanmonkey/issues
+Keywords: text,cleanup,whitespace,unicode,normalize
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Text Processing
+Classifier: Topic :: Text Processing :: Filters
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file
+
+# cleanmonkey
+
+One-call text cleanup for invisible characters, smart quotes, and whitespace normalization.
+
+## Install
+
+```bash
+pip install cleanmonkey
+```
+
+## Quick Start
+
+```python
+from cleanmonkey import clean
+
+# Sensible defaults handle the common garbage
+clean("hello\u00a0world\u2019s \u2014 test")
+# → "hello world's - test"
+
+# Idempotent — safe to call twice
+clean(clean(text)) == clean(text)
+```
+
+## What It Cleans (by default)
+
+| Category | Examples | Result |
+|---|---|---|
+| Non-breaking spaces | `\u00a0`, `\u2007`, `\u202f` | Regular space |
+| Zero-width chars | `\u200b`, `\u200c`, `\u200d`, `\ufeff` | Removed |
+| Smart quotes | `\u2018` `\u2019` `\u201c` `\u201d` | `'` and `"` |
+| Dashes | `\u2013` (en), `\u2014` (em) | `-` |
+| Ellipsis | `\u2026` | `...` |
+| Control chars | null, form feed, vertical tab | Removed |
+| Line endings | `\r\n`, `\r` | `\n` |
+| Multiple spaces | `"hello   world"` | `"hello world"` |
+| Leading/trailing | `"  hello  "` | `"hello"` |
+
+## Granular Control
+
+Override any default:
+
+```python
+clean(text, smart_quotes=False)       # keep curly quotes
+clean(text, dashes=False)             # keep em/en dashes
+clean(text, fullwidth=True)           # also normalize fullwidth digits/letters
+clean(text, collapse_spaces=False)    # keep multiple spaces
+clean(text, strip=False)              # keep leading/trailing whitespace
+```
+
+## Profiles
+
+```python
+clean(text, profile="default")     # all normalizations (the default)
+clean(text, profile="csv")         # default + fullwidth normalization
+clean(text, profile="sql")         # default + fullwidth normalization
+clean(text, profile="display")     # keep smart quotes & dashes; still clean invisible, control, whitespace, line endings
+clean(text, profile="minimal")     # invisible chars only, no collapsing or stripping
+clean(text, profile="aggressive")  # everything including fullwidth
+```
+
+## Batch Helpers
+
+```python
+from cleanmonkey import clean_column, clean_dict
+
+# Clean a list (non-strings pass through)
+clean_column(["hello\u00a0world", 42, None])
+# → ["hello world", 42, None]
+
+# Recursively clean dict values
+clean_dict({"name": "John\u00a0Doe", "nested": {"val": "test\u200b"}})
+# → {"name": "John Doe", "nested": {"val": "test"}}
+
+# Also clean keys
+clean_dict({"key\u00a0name": "val"}, keys=True)
+# → {"key name": "val"}
+```
+
+## Inspect
+
+Find out what's lurking in your text:
+
+```python
+from cleanmonkey import inspect
+
+for info in inspect("hello\u00a0world\u200b"):
+    print(f"{info.codepoint} {info.name} count={info.count} at {info.positions}")
+# U+00A0 NO-BREAK SPACE count=1 at [5]
+# U+200B ZERO WIDTH SPACE count=1 at [11]
+```
+
+## CLI
+
+```bash
+# Clean a file
+cleanmonkey input.txt -o output.txt
+
+# Pipe through stdin
+cat dirty.csv | cleanmonkey > clean.csv
+
+# Use a profile
+cleanmonkey --profile csv input.txt
+
+# Inspect mode — report what's in a file
+cleanmonkey --inspect input.txt
+
+# Machine-readable JSON inspect output
+cleanmonkey --json input.txt
+
+# Selective overrides
+cleanmonkey --no-smart-quotes --fullwidth input.txt
+
+# Preserve whitespace structure
+cleanmonkey --no-strip --no-collapse-spaces input.txt
+
+# Preserve line endings (CR/CRLF)
+cleanmonkey --no-line-endings input.txt
+```
+
+## Built for LLMs
+
+cleanmonkey is designed to work well as a tool for large language models. Invisible character cleanup is a constant source of silent bugs in LLM-driven data pipelines — non-breaking spaces break splits, zero-width characters corrupt comparisons, and smart quotes fail exact matches. Without cleanmonkey, LLMs end up generating repetitive `.replace()` chains that miss edge cases and waste tokens. A single `clean()` call handles all of it with a structured, idempotent result — no multi-step prompting or character-by-character debugging required. Fewer tokens in, clean data out.
+
+## License
+
+MIT

cleanmonkey-0.1.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+cleanmonkey/__init__.py,sha256=gHz9hgjVMCryQXmiycnVzdWzWdDvOCKoSnMSSRlbNN4,353
+cleanmonkey/__main__.py,sha256=jFCbwZHhmw8k8JnQsSE14dtyiZpBDkb8T7mjWBRaAg4,158
+cleanmonkey/cli.py,sha256=wPXbCwCrT1yZx8kMCvauBFT2ioZTczTItLuwr35-w7Y,15788
+cleanmonkey/core.py,sha256=vO03OpJqz7TzjJoEgOYJLQcwlT3gMLM_kHgASw36KYY,15624
+cleanmonkey/maps.py,sha256=AS5kxrtZtcxYLOFES38jG34gFqB96V5fXRNwHtlkU98,3708
+cleanmonkey/profiles.py,sha256=hG6zWAwAFhP0Pxm_vpc4HRKKfk_ipRsDv2AFuJeZ1YU,1534
+cleanmonkey/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+cleanmonkey-0.1.0.dist-info/licenses/LICENSE,sha256=srNahN_Cxejm5SlFsCghF2Mml1gXgqlnuqWlDt7F1ck,1065
+cleanmonkey-0.1.0.dist-info/METADATA,sha256=tHxlu1XJxo_kQLYJal9F2iTwyRCBSHgeYk6VRQD3kYQ,4914
+cleanmonkey-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+cleanmonkey-0.1.0.dist-info/entry_points.txt,sha256=ePX3uiSQ0P3GDiO4yAci3iILpk-FqD6QCvNkXFjJmyg,80
+cleanmonkey-0.1.0.dist-info/top_level.txt,sha256=q7GGSdV6NFD8-QSz1Vowde7NLAO5MPoaMvpnpmhrQWI,12
+cleanmonkey-0.1.0.dist-info/RECORD,,

cleanmonkey-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

cleanmonkey-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+cleanmonkey = cleanmonkey.cli:_main_with_broken_pipe_handling

cleanmonkey-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 RexBytes
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

cleanmonkey-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+cleanmonkey