fix-text 0.1.0__tar.gz

fix_text-0.1.0/.copier-answers.yml

@@ -0,0 +1,8 @@
+# Changes here will be overwritten by Copier. Do not edit manually.
+_commit: v0.2.20
+_src_path: gh:jlevy/simple-modern-uv
+package_author_name: Dean Li
+package_description: Detect and normalize suspicious Unicode characters in text files.
+package_github_org: deantvv
+package_module: fix_text
+package_name: fix-text

fix_text-0.1.0/.gitignore

@@ -0,0 +1,10 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv

fix_text-0.1.0/.python-version

@@ -0,0 +1 @@
+3.12

fix_text-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Dean
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

fix_text-0.1.0/PKG-INFO

@@ -0,0 +1,136 @@
+Metadata-Version: 2.4
+Name: fix-text
+Version: 0.1.0
+Summary: Detect and normalize suspicious Unicode characters in text files.
+Project-URL: Homepage, https://github.com/deantvv/fix-text
+Project-URL: Repository, https://github.com/deantvv/fix-text
+Project-URL: Issues, https://github.com/deantvv/fix-text/issues
+Project-URL: Changelog, https://github.com/deantvv/fix-text/releases
+Author: Dean Li
+License: MIT
+License-File: LICENSE
+Keywords: cleanup,cli,normalization,text,unicode
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Text Processing
+Classifier: Topic :: Utilities
+Requires-Python: <4,>=3.12
+Requires-Dist: orjson>=3.10.0
+Requires-Dist: pyyaml>=6.0.0
+Description-Content-Type: text/markdown
+
+# fix-text
+
+`fix-text` is a small Python CLI for detecting and cleaning suspicious Unicode
+characters in text files, including fullwidth spaces, non-breaking spaces, zero-width
+characters, BOM markers, and optional control characters.
+
+## What it fixes
+
+- Replaces fullwidth and non-standard spacing characters with a normal ASCII space
+- Removes zero-width characters and byte order marks
+- Optionally removes unsupported control characters with `--include-controls`
+- Validates cleaned `.json` files before rewriting them
+- Validates cleaned `.yaml` and `.yml` files before rewriting them
+- Scans individual files or entire directory trees
+
+## Install for development
+
+This project is managed with `uv`.
+
+```bash
+uv sync
+```
+
+Install the test dependency group:
+
+```bash
+uv sync --group dev
+```
+
+Run the CLI without installing globally:
+
+```bash
+uv run fix-text --help
+```
+
+Run the test suite:
+
+```bash
+uv run --group dev pytest
+```
+
+## Usage
+
+Scan a file or directory:
+
+```bash
+uv run fix-text path/to/file.txt path/to/dir
+```
+
+Rewrite files in place:
+
+```bash
+uv run fix-text --apply path/to/file.txt
+```
+
+Include control-character cleanup:
+
+```bash
+uv run fix-text --apply --include-controls logs/
+```
+
+Treat additional suffixes as text:
+
+```bash
+uv run fix-text --ext .log --ext .cfg sample/
+```
+
+For `.json` files, `fix-text` validates the cleaned output with `orjson` before writing.
+For `.yaml` and `.yml`, it validates the cleaned output with `PyYAML`.
+If the cleaned content would still be invalid JSON or YAML, the file is left unchanged.
+
+## Example output
+
+```text
+notes.txt
+  3:14 U+3000 IDEOGRAPHIC SPACE '\u3000' -> replace with space
+  8:2 U+200B ZERO WIDTH SPACE '\u200b' -> delete
+
+Found 2 issue(s). Re-run with --apply to rewrite files.
+```
+
+## PyPI release plan
+
+1. Replace the placeholder GitHub URLs in `pyproject.toml`.
+2. Create a PyPI account and API token.
+3. Build distributions with `uv build`.
+4. Publish with `uv publish`.
+5. Tag the release in git so the source and package versions stay aligned.
+
+Recommended pre-release checks:
+
+- `uv run fix-text --help`
+- `uv run fix-text README.md`
+- `uv run --group dev pytest`
+- `uv build --out-dir dist`
+
+Build into the repo-local `dist/` directory:
+
+```bash
+./scripts/release.sh build
+```
+
+Publish the repo-local artifacts:
+
+```bash
+./scripts/release.sh publish
+```
+
+## License
+
+MIT

fix_text-0.1.0/README.md

@@ -0,0 +1,111 @@
+# fix-text
+
+`fix-text` is a small Python CLI for detecting and cleaning suspicious Unicode
+characters in text files, including fullwidth spaces, non-breaking spaces, zero-width
+characters, BOM markers, and optional control characters.
+
+## What it fixes
+
+- Replaces fullwidth and non-standard spacing characters with a normal ASCII space
+- Removes zero-width characters and byte order marks
+- Optionally removes unsupported control characters with `--include-controls`
+- Validates cleaned `.json` files before rewriting them
+- Validates cleaned `.yaml` and `.yml` files before rewriting them
+- Scans individual files or entire directory trees
+
+## Install for development
+
+This project is managed with `uv`.
+
+```bash
+uv sync
+```
+
+Install the test dependency group:
+
+```bash
+uv sync --group dev
+```
+
+Run the CLI without installing globally:
+
+```bash
+uv run fix-text --help
+```
+
+Run the test suite:
+
+```bash
+uv run --group dev pytest
+```
+
+## Usage
+
+Scan a file or directory:
+
+```bash
+uv run fix-text path/to/file.txt path/to/dir
+```
+
+Rewrite files in place:
+
+```bash
+uv run fix-text --apply path/to/file.txt
+```
+
+Include control-character cleanup:
+
+```bash
+uv run fix-text --apply --include-controls logs/
+```
+
+Treat additional suffixes as text:
+
+```bash
+uv run fix-text --ext .log --ext .cfg sample/
+```
+
+For `.json` files, `fix-text` validates the cleaned output with `orjson` before writing.
+For `.yaml` and `.yml`, it validates the cleaned output with `PyYAML`.
+If the cleaned content would still be invalid JSON or YAML, the file is left unchanged.
+
+## Example output
+
+```text
+notes.txt
+  3:14 U+3000 IDEOGRAPHIC SPACE '\u3000' -> replace with space
+  8:2 U+200B ZERO WIDTH SPACE '\u200b' -> delete
+
+Found 2 issue(s). Re-run with --apply to rewrite files.
+```
+
+## PyPI release plan
+
+1. Replace the placeholder GitHub URLs in `pyproject.toml`.
+2. Create a PyPI account and API token.
+3. Build distributions with `uv build`.
+4. Publish with `uv publish`.
+5. Tag the release in git so the source and package versions stay aligned.
+
+Recommended pre-release checks:
+
+- `uv run fix-text --help`
+- `uv run fix-text README.md`
+- `uv run --group dev pytest`
+- `uv build --out-dir dist`
+
+Build into the repo-local `dist/` directory:
+
+```bash
+./scripts/release.sh build
+```
+
+Publish the repo-local artifacts:
+
+```bash
+./scripts/release.sh publish
+```
+
+## License
+
+MIT

fix_text-0.1.0/pyproject.toml

@@ -0,0 +1,46 @@
+[project]
+name = "fix-text"
+version = "0.1.0"
+description = "Detect and normalize suspicious Unicode characters in text files."
+readme = "README.md"
+requires-python = ">=3.12,<4"
+dependencies = [
+    "orjson>=3.10.0",
+    "PyYAML>=6.0.0",
+]
+license = { text = "MIT" }
+authors = [
+    { name = "Dean Li" },
+]
+keywords = ["cli", "text", "unicode", "normalization", "cleanup"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Text Processing",
+    "Topic :: Utilities",
+]
+
+[project.urls]
+Homepage = "https://github.com/deantvv/fix-text"
+Repository = "https://github.com/deantvv/fix-text"
+Issues = "https://github.com/deantvv/fix-text/issues"
+Changelog = "https://github.com/deantvv/fix-text/releases"
+
+[project.scripts]
+fix-text = "fix_text.cli:main"
+
+[build-system]
+requires = ["hatchling>=1.27.0"]
+build-backend = "hatchling.build"
+
+[dependency-groups]
+dev = [
+    "pytest>=8.0.0",
+]
+
+[tool.uv]
+package = true

fix_text-0.1.0/src/fix_text/__init__.py

@@ -0,0 +1,3 @@
+__all__ = ["__version__"]
+
+__version__ = "0.1.0"

fix_text-0.1.0/src/fix_text/__main__.py

@@ -0,0 +1,5 @@
+from fix_text.cli import main
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

fix_text-0.1.0/src/fix_text/cli.py

@@ -0,0 +1,272 @@
+from __future__ import annotations
+
+import argparse
+import sys
+import unicodedata
+from collections.abc import Iterable
+from dataclasses import dataclass
+from pathlib import Path
+
+import orjson
+import yaml
+
+from fix_text import __version__
+
+
+REPLACEMENTS: dict[str, str] = {
+    "\u00a0": " ",   # no-break space
+    "\u1680": " ",   # ogham space mark
+    "\u2000": " ",
+    "\u2001": " ",
+    "\u2002": " ",
+    "\u2003": " ",
+    "\u2004": " ",
+    "\u2005": " ",
+    "\u2006": " ",
+    "\u2007": " ",
+    "\u2008": " ",
+    "\u2009": " ",
+    "\u200a": " ",
+    "\u202f": " ",
+    "\u205f": " ",
+    "\u3000": " ",   # fullwidth space
+    "\ufeff": "",    # BOM / zero-width no-break space
+    "\u200b": "",    # zero-width space
+    "\u200c": "",    # zero-width non-joiner
+    "\u200d": "",    # zero-width joiner
+    "\u2060": "",    # word joiner
+    "\u00ad": "",    # soft hyphen
+}
+
+CONTROL_EXCEPTIONS = {"\n", "\r", "\t"}
+TEXT_SUFFIXES = {
+    ".csv",
+    ".html",
+    ".ini",
+    ".js",
+    ".json",
+    ".md",
+    ".py",
+    ".rst",
+    ".sql",
+    ".svg",
+    ".toml",
+    ".ts",
+    ".tsx",
+    ".txt",
+    ".xml",
+    ".yaml",
+    ".yml",
+}
+
+
+@dataclass(slots=True)
+class Issue:
+    line: int
+    column: int
+    char: str
+    action: str
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="fix-text",
+        description=(
+            "Detect and normalize suspicious Unicode characters in text files. "
+            "Use --apply to rewrite files in place."
+        ),
+    )
+    parser.add_argument("paths", nargs="+", help="File or directory paths to scan.")
+    parser.add_argument(
+        "--apply",
+        action="store_true",
+        help="Rewrite files in place. Without this flag the tool only reports issues.",
+    )
+    parser.add_argument(
+        "--include-controls",
+        action="store_true",
+        help="Remove unsupported control characters as well as mapped Unicode spaces.",
+    )
+    parser.add_argument(
+        "--ext",
+        action="append",
+        default=[],
+        metavar="SUFFIX",
+        help="Extra file suffix to treat as text, for example --ext .log",
+    )
+    parser.add_argument(
+        "--encoding",
+        default="utf-8",
+        help="Text encoding to use when reading and writing files. Default: utf-8",
+    )
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=f"%(prog)s {__version__}",
+    )
+    return parser
+
+
+def iter_target_files(paths: Iterable[str], extra_suffixes: Iterable[str]) -> Iterable[Path]:
+    suffixes = TEXT_SUFFIXES | {normalize_suffix(value) for value in extra_suffixes}
+    seen: set[Path] = set()
+
+    for raw_path in paths:
+        path = Path(raw_path)
+        if not path.exists():
+            print(f"warning: path does not exist: {path}", file=sys.stderr)
+            continue
+
+        candidates = [path] if path.is_file() else sorted(p for p in path.rglob("*") if p.is_file())
+        for candidate in candidates:
+            resolved = candidate.resolve()
+            if resolved in seen:
+                continue
+            seen.add(resolved)
+            if candidate.suffix.lower() in suffixes:
+                yield candidate
+
+
+def normalize_suffix(value: str) -> str:
+    return value if value.startswith(".") else f".{value}"
+
+
+def find_issues(text: str, include_controls: bool) -> list[Issue]:
+    issues: list[Issue] = []
+    line = 1
+    column = 1
+
+    for char in text:
+        action = replacement_for(char, include_controls)
+        if action is not None:
+            issues.append(Issue(line=line, column=column, char=char, action=describe_action(char, action)))
+
+        if char == "\n":
+            line += 1
+            column = 1
+        else:
+            column += 1
+
+    return issues
+
+
+def replacement_for(char: str, include_controls: bool) -> str | None:
+    replacement = REPLACEMENTS.get(char)
+    if replacement is not None:
+        return replacement
+
+    if include_controls and unicodedata.category(char) == "Cc" and char not in CONTROL_EXCEPTIONS:
+        return ""
+
+    return None
+
+
+def describe_action(char: str, replacement: str) -> str:
+    if replacement == "":
+        return "delete"
+    if replacement == " ":
+        return "replace with space"
+    return f"replace with {replacement!r}"
+
+
+def sanitize_text(text: str, include_controls: bool) -> str:
+    cleaned: list[str] = []
+    for char in text:
+        replacement = replacement_for(char, include_controls)
+        cleaned.append(char if replacement is None else replacement)
+    return "".join(cleaned)
+
+
+def format_char(char: str) -> str:
+    codepoint = f"U+{ord(char):04X}"
+    name = unicodedata.name(char, "UNKNOWN")
+    printable = repr(char)
+    return f"{codepoint} {name} {printable}"
+
+
+def validate_json_text(text: str) -> bool:
+    try:
+        orjson.loads(text)
+    except orjson.JSONDecodeError:
+        return False
+    return True
+
+
+def validate_yaml_text(text: str) -> bool:
+    try:
+        yaml.safe_load(text)
+    except yaml.YAMLError:
+        return False
+    return True
+
+
+def validate_cleaned_text(path: Path, text: str) -> tuple[bool, str | None]:
+    suffix = path.suffix.lower()
+    if suffix == ".json":
+        return validate_json_text(text), "JSON"
+    if suffix in {".yaml", ".yml"}:
+        return validate_yaml_text(text), "YAML"
+    return True, None
+
+
+def process_file(path: Path, encoding: str, apply: bool, include_controls: bool) -> tuple[int, bool]:
+    try:
+        original = path.read_text(encoding=encoding)
+    except UnicodeDecodeError:
+        print(f"skipped binary or non-{encoding} file: {path}", file=sys.stderr)
+        return 0, False
+
+    issues = find_issues(original, include_controls=include_controls)
+    if not issues:
+        return 0, False
+
+    print(path)
+    for issue in issues:
+        print(f"  {issue.line}:{issue.column} {format_char(issue.char)} -> {issue.action}")
+
+    changed = False
+    if apply:
+        sanitized = sanitize_text(original, include_controls=include_controls)
+        if sanitized != original:
+            is_valid, file_type = validate_cleaned_text(path, sanitized)
+            if not is_valid and file_type is not None:
+                print(f"  skipped rewrite: cleaned content is not valid {file_type}", file=sys.stderr)
+                return len(issues), False
+            path.write_text(sanitized, encoding=encoding)
+            changed = True
+            print("  rewritten")
+
+    return len(issues), changed
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+
+    total_issues = 0
+    changed_files = 0
+
+    for path in iter_target_files(args.paths, args.ext):
+        issue_count, changed = process_file(
+            path,
+            encoding=args.encoding,
+            apply=args.apply,
+            include_controls=args.include_controls,
+        )
+        total_issues += issue_count
+        changed_files += int(changed)
+
+    if total_issues == 0:
+        print("No suspicious characters found.")
+        return 0
+
+    if args.apply:
+        print(f"\nFixed {total_issues} issue(s) across {changed_files} file(s).")
+    else:
+        print(f"\nFound {total_issues} issue(s). Re-run with --apply to rewrite files.")
+
+    return 1 if not args.apply else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

fix_text-0.1.0/tests/test_cli.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from fix_text.cli import (
+    main,
+    process_file,
+    sanitize_text,
+    validate_json_text,
+    validate_yaml_text,
+)
+
+
+def test_sanitize_text_replaces_and_deletes_mapped_characters() -> None:
+    assert sanitize_text("A\u3000B\u200bC", include_controls=False) == "A BC"
+
+
+def test_validate_json_text_accepts_valid_json() -> None:
+    assert validate_json_text('{"name":"value"}')
+
+
+def test_validate_json_text_rejects_invalid_json() -> None:
+    assert not validate_json_text('{"name":"value",}')
+
+
+def test_validate_yaml_text_accepts_valid_yaml() -> None:
+    assert validate_yaml_text("name: value\nitems:\n  - one\n")
+
+
+def test_validate_yaml_text_rejects_invalid_yaml() -> None:
+    assert not validate_yaml_text("items: [one\n")
+
+
+def test_process_file_rewrites_valid_json(tmp_path: Path, capsys) -> None:
+    path = tmp_path / "sample.json"
+    path.write_text('{"name":"A\u3000B","note":"x\u200by"}\n', encoding="utf-8")
+
+    issues, changed = process_file(path, encoding="utf-8", apply=True, include_controls=False)
+
+    assert issues == 2
+    assert changed is True
+    assert path.read_text(encoding="utf-8") == '{"name":"A B","note":"xy"}\n'
+    captured = capsys.readouterr()
+    assert "rewritten" in captured.out
+
+
+def test_process_file_skips_invalid_json_rewrite(tmp_path: Path, capsys) -> None:
+    path = tmp_path / "invalid.json"
+    original = '{"name":"A\u3000B",}\n'
+    path.write_text(original, encoding="utf-8")
+
+    issues, changed = process_file(path, encoding="utf-8", apply=True, include_controls=False)
+
+    assert issues == 1
+    assert changed is False
+    assert path.read_text(encoding="utf-8") == original
+    captured = capsys.readouterr()
+    assert "skipped rewrite: cleaned content is not valid JSON" in captured.err
+
+
+def test_process_file_rewrites_valid_yaml(tmp_path: Path, capsys) -> None:
+    path = tmp_path / "sample.yaml"
+    path.write_text("title: A\u3000B\nnote: x\u200by\n", encoding="utf-8")
+
+    issues, changed = process_file(path, encoding="utf-8", apply=True, include_controls=False)
+
+    assert issues == 2
+    assert changed is True
+    assert path.read_text(encoding="utf-8") == "title: A B\nnote: xy\n"
+    captured = capsys.readouterr()
+    assert "rewritten" in captured.out
+
+
+def test_process_file_skips_invalid_yml_rewrite(tmp_path: Path, capsys) -> None:
+    path = tmp_path / "invalid.yml"
+    original = "items: [one\u200b\n"
+    path.write_text(original, encoding="utf-8")
+
+    issues, changed = process_file(path, encoding="utf-8", apply=True, include_controls=False)
+
+    assert issues == 1
+    assert changed is False
+    assert path.read_text(encoding="utf-8") == original
+    captured = capsys.readouterr()
+    assert "skipped rewrite: cleaned content is not valid YAML" in captured.err
+
+
+def test_main_reports_findings_without_apply(tmp_path: Path, capsys) -> None:
+    path = tmp_path / "notes.txt"
+    path.write_text("hello\u3000world\n", encoding="utf-8")
+
+    exit_code = main([str(path)])
+
+    assert exit_code == 1
+    captured = capsys.readouterr()
+    assert "U+3000 IDEOGRAPHIC SPACE" in captured.out
+    assert "Re-run with --apply" in captured.out