slopdam 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
slopdam/__init__.py ADDED
@@ -0,0 +1,10 @@
1
+ # SPDX-License-Identifier: MIT
2
+ from .rules import LEVELS, PROFILES, RULES, Finding, Rule, Sanitizer
3
+ from .stdlib import PlainFilter, PlainFormatter
4
+
5
+ __all__ = ["LEVELS", "PROFILES", "RULES", "Finding", "PlainFilter", "PlainFormatter", "Rule", "Sanitizer", "clean"]
6
+ __version__ = "0.2.0"
7
+
8
+
9
+ def clean(text: str, profile: str = "reader", level: str = "basic") -> str:
10
+ return Sanitizer(profile, level).clean(text)
slopdam/cli.py ADDED
@@ -0,0 +1,204 @@
1
+ # SPDX-License-Identifier: MIT
2
+ """Command line entry point.
3
+
4
+ slopdam check [PATH ...] report rule hits, exit 1 if any
5
+ slopdam fix [PATH ...] rewrite string literals in Python files (--diff to preview)
6
+ slopdam clean read stdin, write cleaned text to stdout
7
+
8
+ Settings come from slopdam.toml or [tool.slopdam] in pyproject.toml (see config.py);
9
+ command line options win over the file. Exit codes: 0 clean, 1 findings, 2 usage or config error.
10
+ """
11
+ from __future__ import annotations
12
+
13
+ import argparse
14
+ import difflib
15
+ import json
16
+ import os
17
+ import re
18
+ import sys
19
+ from typing import Iterator
20
+
21
+ from .config import SCOPES, Config, ConfigError, config_from_env, load_config
22
+ from .fixer import fix_source
23
+ from .rules import DASH_INLINE, LEVELS, PROFILES, Sanitizer
24
+
25
+ SOURCE_EXTS = {".py", ".js", ".mjs", ".cjs", ".ts", ".tsx", ".jsx", ".java", ".kt", ".go", ".rb", ".php", ".rs", ".swift"}
26
+ SKIP_DIRS = {".git", "node_modules", ".venv", "venv", "dist", "build", "__pycache__", ".tox", "target"}
27
+
28
+ # Lines that look like a log/print call: logger.info(, logging.warning(, console.log(, print(, println(, slog.Info(, Log.d(
29
+ LOG_CALL = re.compile(r"(?:\blog\w*|\bconsole|\bprint\w*|\bfmt\.(?:Print|Sprint|Fprint)\w*|\bslog)\s*[.(]", re.I)
30
+ # slopdam: ignore -> skip the whole line
31
+ # slopdam: ignore[dash,emoji] -> skip only those rule ids / prefixes
32
+ COMMENT_PREFIXES = ("#", "//", "/*", "*", "--") # comment-only lines are not log calls
33
+ IGNORE = re.compile(r"slopdam:\s*ignore(?:\[([\w.,\s-]+)\])?")
34
+
35
+
36
+ def iter_files(paths: list, cfg: Config) -> Iterator[str]:
37
+ for path in paths:
38
+ if os.path.isfile(path):
39
+ yield path
40
+ continue
41
+ for root, dirs, files in os.walk(path):
42
+ dirs[:] = sorted(d for d in dirs if d not in SKIP_DIRS)
43
+ for name in sorted(files):
44
+ full = os.path.join(root, name)
45
+ if os.path.splitext(name)[1] in SOURCE_EXTS and cfg.selects(full):
46
+ yield full
47
+
48
+
49
+ def _ignored(rule_id: str, ids: list) -> bool:
50
+ return any(rule_id == i or rule_id.startswith(i + ".") for i in ids)
51
+
52
+
53
+ def _resolve(args: argparse.Namespace) -> Config:
54
+ """Merge the settings file with command line options (options win)."""
55
+ cfg = load_config(args.config or config_from_env(), start=(getattr(args, "paths", None) or ["."])[0])
56
+ cfg.profile = args.profile or cfg.profile
57
+ cfg.level = args.level or cfg.level
58
+ cfg.dash_inline = args.dash_inline or cfg.dash_inline
59
+ cfg.disable = tuple(cfg.disable) + tuple(args.disable)
60
+ if getattr(args, "all", False):
61
+ cfg.scope = "all"
62
+ elif getattr(args, "scope", None):
63
+ cfg.scope = args.scope
64
+ return cfg
65
+
66
+
67
+ def _sanitizer(cfg: Config) -> Sanitizer:
68
+ return Sanitizer(cfg.profile, cfg.level, disable=cfg.disable, dash_inline=cfg.dash_inline)
69
+
70
+
71
+ def _read(path: str):
72
+ try:
73
+ with open(path, encoding="utf-8") as f:
74
+ return f.read()
75
+ except (UnicodeDecodeError, OSError) as e:
76
+ print(f"{path}: skipped ({e})", file=sys.stderr)
77
+ return None
78
+
79
+
80
+ def _line_hits(text: str, sanitizer: Sanitizer, scope: str):
81
+ for lineno, line in enumerate(text.splitlines(), 1):
82
+ if scope != "all" and (line.lstrip().startswith(COMMENT_PREFIXES) or not LOG_CALL.search(line)):
83
+ continue
84
+ ignore = IGNORE.search(line)
85
+ if ignore and not ignore.group(1):
86
+ continue
87
+ ids = [i.strip() for i in ignore.group(1).split(",")] if ignore else []
88
+ for f in sanitizer.findings(line):
89
+ if not _ignored(f.rule_id, ids):
90
+ yield lineno, f.start + 1, f.rule_id, f.text, f.fix
91
+
92
+
93
+ def check(args: argparse.Namespace) -> int:
94
+ cfg = _resolve(args)
95
+ sanitizer = _sanitizer(cfg)
96
+ results = []
97
+ for path in iter_files(args.paths, cfg):
98
+ text = _read(path)
99
+ if text is None:
100
+ continue
101
+ if cfg.scope in ("log-calls", "strings") and path.endswith(".py"):
102
+ # Python: exact string literals from the syntax tree, the same targets fix rewrites.
103
+ try:
104
+ _, edits = fix_source(text, sanitizer, scope=cfg.scope)
105
+ hits = sorted((e.line, e.col, e.rule_id, e.before, e.after) for e in edits)
106
+ except SyntaxError:
107
+ hits = list(_line_hits(text, sanitizer, "log-calls"))
108
+ else:
109
+ hits = list(_line_hits(text, sanitizer, "all" if cfg.scope == "all" else "log-calls"))
110
+ results += [(path, *h) for h in hits]
111
+
112
+ if args.format == "json":
113
+ json.dump([{"path": p, "line": n, "col": c, "rule": r, "text": t, "fix": x} for p, n, c, r, t, x in results],
114
+ sys.stdout, ensure_ascii=False, indent=2)
115
+ print()
116
+ else:
117
+ for path, lineno, col, rule, text, fix in results:
118
+ shown = "report only" if fix is None else f"-> {fix!r}"
119
+ print(f"{path}:{lineno}:{col}: {rule} {text!r} {shown}")
120
+ if results:
121
+ print(f"\n{len(results)} finding(s), profile={cfg.profile}", file=sys.stderr)
122
+ return 1 if results else 0
123
+
124
+
125
+ def fix(args: argparse.Namespace) -> int:
126
+ cfg = _resolve(args)
127
+ sanitizer = _sanitizer(cfg)
128
+ scope = "strings" if cfg.scope == "all" else cfg.scope
129
+ changed_files, changed, skipped = 0, 0, 0
130
+ for path in iter_files(args.paths, cfg):
131
+ if not path.endswith(".py"):
132
+ skipped += 1
133
+ continue
134
+ text = _read(path)
135
+ if text is None:
136
+ continue
137
+ try:
138
+ new, edits = fix_source(text, sanitizer, scope=scope)
139
+ except SyntaxError as e:
140
+ print(f"{path}: skipped (does not parse: {e.msg} at line {e.lineno})", file=sys.stderr)
141
+ continue
142
+ applied = [e for e in edits if e.after is not None]
143
+ if new == text:
144
+ continue
145
+ changed_files += 1
146
+ changed += len(applied)
147
+ if args.diff:
148
+ sys.stdout.writelines(difflib.unified_diff(
149
+ text.splitlines(keepends=True), new.splitlines(keepends=True), f"a/{path}", f"b/{path}"))
150
+ else:
151
+ with open(path, "w", encoding="utf-8") as f:
152
+ f.write(new)
153
+ for e in applied:
154
+ print(f"{path}:{e.line}:{e.col}: {e.rule_id} {e.before!r} -> {e.after!r}")
155
+ verb = "would fix" if args.diff else "fixed"
156
+ note = f", {skipped} non-Python file(s) skipped" if skipped else ""
157
+ print(f"{verb} {changed} issue(s) in {changed_files} file(s){note}", file=sys.stderr)
158
+ return 1 if args.diff and changed_files else 0
159
+
160
+
161
+ def clean(args: argparse.Namespace) -> int:
162
+ sanitizer = _sanitizer(_resolve(args))
163
+ for line in sys.stdin:
164
+ cleaned = sanitizer.clean(line)
165
+ if cleaned:
166
+ sys.stdout.write(cleaned + "\n")
167
+ return 0
168
+
169
+
170
+ def main(argv: list = None) -> int:
171
+ parser = argparse.ArgumentParser(prog="slopdam", description="Strip AI-generated patterns from log output.")
172
+ parser.add_argument("--config", metavar="FILE", help="settings file (default: nearest slopdam.toml or pyproject.toml)")
173
+ parser.add_argument("--profile", choices=PROFILES, help="reader: for end users (default), dev: for developers")
174
+ parser.add_argument("--level", choices=LEVELS)
175
+ parser.add_argument("--dash-inline", choices=tuple(DASH_INLINE), help="reader: what an inline dash becomes")
176
+ parser.add_argument("--disable", action="append", default=[], metavar="RULE_ID", help="turn off a rule or rule group (repeatable)")
177
+ sub = parser.add_subparsers(dest="command", required=True)
178
+
179
+ p_check = sub.add_parser("check", help="lint source files")
180
+ p_check.add_argument("paths", nargs="*", default=["."])
181
+ p_check.add_argument("--scope", choices=SCOPES, help="log-calls (default), strings (Python literals), all (every line)")
182
+ p_check.add_argument("--all", action="store_true", help="same as --scope all")
183
+ p_check.add_argument("--format", choices=("text", "json"), default="text")
184
+ p_check.set_defaults(func=check)
185
+
186
+ p_fix = sub.add_parser("fix", help="rewrite string literals in Python files")
187
+ p_fix.add_argument("paths", nargs="*", default=["."])
188
+ p_fix.add_argument("--scope", choices=("log-calls", "strings"), help="log-calls (default) or every non-docstring string")
189
+ p_fix.add_argument("--diff", action="store_true", help="print a diff instead of writing; exit 1 if anything would change")
190
+ p_fix.set_defaults(func=fix)
191
+
192
+ p_clean = sub.add_parser("clean", help="clean stdin to stdout")
193
+ p_clean.set_defaults(func=clean)
194
+
195
+ args = parser.parse_args(argv)
196
+ try:
197
+ return args.func(args)
198
+ except ConfigError as e:
199
+ print(f"slopdam: {e}", file=sys.stderr)
200
+ return 2
201
+
202
+
203
+ if __name__ == "__main__":
204
+ sys.exit(main())
slopdam/config.py ADDED
@@ -0,0 +1,242 @@
1
+ # SPDX-License-Identifier: MIT
2
+ """Project configuration: ``slopdam.toml`` or ``[tool.slopdam]`` in ``pyproject.toml``.
3
+
4
+ profile = "dev" # reader | dev
5
+ level = "basic" # basic | strong | strict
6
+ disable = ["emoji.status"] # rule ids or groups
7
+
8
+ [dash]
9
+ inline = "colon" # colon | comma | hyphen | space (reader only)
10
+
11
+ [lint]
12
+ scope = "log-calls" # log-calls | strings | all
13
+ include = ["src/**"]
14
+ exclude = ["tests/**"]
15
+
16
+ Other tables (for example ``[github]`` written by a hosted service) are ignored.
17
+ Command line options always win over the file.
18
+ """
19
+ from __future__ import annotations
20
+
21
+ import fnmatch
22
+ import os
23
+ import re
24
+ from dataclasses import dataclass, field
25
+ from pathlib import Path
26
+ from typing import Optional
27
+
28
+ from .rules import DASH_INLINE, LEVELS, PROFILES
29
+
30
+ SCOPES = ("log-calls", "strings", "all")
31
+
32
+
33
+ class ConfigError(ValueError):
34
+ pass
35
+
36
+
37
+ @dataclass
38
+ class Config:
39
+ profile: str = "reader"
40
+ level: str = "basic"
41
+ disable: tuple = ()
42
+ dash_inline: str = "colon"
43
+ scope: str = "log-calls"
44
+ include: tuple = ()
45
+ exclude: tuple = ()
46
+ root: Optional[Path] = None # directory paths in include/exclude are relative to
47
+ source: Optional[Path] = field(default=None) # file the settings came from
48
+
49
+ def selects(self, path: str) -> bool:
50
+ """Whether a file path passes include/exclude (globs relative to the config directory)."""
51
+ try:
52
+ rel = Path(path).resolve().relative_to((self.root or Path.cwd()).resolve()).as_posix()
53
+ except ValueError:
54
+ rel = Path(path).as_posix()
55
+ if self.include and not any(fnmatch.fnmatch(rel, g) for g in self.include):
56
+ return False
57
+ return not any(fnmatch.fnmatch(rel, g) for g in self.exclude)
58
+
59
+
60
+ def find_config(start: str = ".") -> Optional[Path]:
61
+ """Nearest slopdam.toml, or pyproject.toml with [tool.slopdam], from start up to the repo root."""
62
+ here = Path(start).resolve()
63
+ if here.is_file():
64
+ here = here.parent
65
+ for d in (here, *here.parents):
66
+ if (d / "slopdam.toml").is_file():
67
+ return d / "slopdam.toml"
68
+ pyproject = d / "pyproject.toml"
69
+ if pyproject.is_file() and re.search(r"^\[tool\.slopdam(\.|\])", pyproject.read_text(encoding="utf-8"), re.M):
70
+ return pyproject
71
+ if (d / ".git").exists():
72
+ break
73
+ return None
74
+
75
+
76
+ def load_config(path: Optional[str] = None, start: str = ".") -> Config:
77
+ """Load settings from an explicit file or the nearest one; defaults when there is none."""
78
+ found = Path(path) if path else find_config(start)
79
+ if found is None:
80
+ return Config()
81
+ try:
82
+ data = _parse_toml(found.read_text(encoding="utf-8"))
83
+ except OSError as e:
84
+ raise ConfigError(f"{found}: {e}") from e
85
+ if found.name == "pyproject.toml":
86
+ data = data.get("tool", {}).get("slopdam", {})
87
+ return _from_dict(data, found)
88
+
89
+
90
+ def _from_dict(data: dict, source: Path) -> Config:
91
+ def pick(value, allowed, name):
92
+ if value not in allowed:
93
+ raise ConfigError(f"{source}: {name} must be one of {', '.join(allowed)} (got {value!r})")
94
+ return value
95
+
96
+ def strings(value, name):
97
+ if not isinstance(value, list) or not all(isinstance(v, str) for v in value):
98
+ raise ConfigError(f"{source}: {name} must be a list of strings")
99
+ return tuple(value)
100
+
101
+ lint = data.get("lint", {})
102
+ dash = data.get("dash", {})
103
+ return Config(
104
+ profile=pick(data.get("profile", "reader"), PROFILES, "profile"),
105
+ level=pick(data.get("level", "basic"), LEVELS, "level"),
106
+ disable=strings(data.get("disable", []), "disable"),
107
+ dash_inline=pick(dash.get("inline", "colon"), tuple(DASH_INLINE), "dash.inline"),
108
+ scope=pick(lint.get("scope", "log-calls"), SCOPES, "lint.scope"),
109
+ include=strings(lint.get("include", []), "lint.include"),
110
+ exclude=strings(lint.get("exclude", []), "lint.exclude"),
111
+ root=source.parent,
112
+ source=source,
113
+ )
114
+
115
+
116
+ def _parse_toml(text: str) -> dict:
117
+ try:
118
+ import tomllib # Python 3.11+
119
+ except ImportError:
120
+ try:
121
+ import tomli as tomllib # optional backport
122
+ except ImportError:
123
+ return _parse_toml_subset(text)
124
+ try:
125
+ return tomllib.loads(text)
126
+ except tomllib.TOMLDecodeError as e:
127
+ raise ConfigError(f"invalid TOML: {e}") from e
128
+
129
+
130
+ _KEY = re.compile(r'^([A-Za-z0-9_.-]+|"[^"]*")\s*=\s*(.*)$')
131
+
132
+
133
+ def _parse_toml_subset(text: str) -> dict:
134
+ """Small TOML reader for Python < 3.11 without tomli.
135
+
136
+ Enough for slopdam settings and the surrounding pyproject.toml: tables, strings,
137
+ booleans, integers and (multi-line) arrays of those. Anything else is skipped.
138
+ """
139
+ root: dict = {}
140
+ table = root
141
+ pending = ""
142
+ for raw in text.splitlines():
143
+ line = _strip_comment(raw).strip()
144
+ if pending:
145
+ pending += " " + line
146
+ if _balanced(pending):
147
+ _assign(table, pending)
148
+ pending = ""
149
+ continue
150
+ if not line:
151
+ continue
152
+ if line.startswith("[["): # array of tables: not used by slopdam, skip its keys
153
+ table = {}
154
+ continue
155
+ if line.startswith("["):
156
+ table = root
157
+ for part in line.strip("[] ").split("."):
158
+ table = table.setdefault(part.strip().strip('"'), {})
159
+ continue
160
+ if _KEY.match(line):
161
+ if _balanced(line):
162
+ _assign(table, line)
163
+ else:
164
+ pending = line
165
+ return root
166
+
167
+
168
+ def _strip_comment(line: str) -> str:
169
+ out, quote = [], None
170
+ for ch in line:
171
+ if quote:
172
+ if ch == quote:
173
+ quote = None
174
+ elif ch in "\"'":
175
+ quote = ch
176
+ elif ch == "#":
177
+ break
178
+ out.append(ch)
179
+ return "".join(out)
180
+
181
+
182
+ def _balanced(s: str) -> bool:
183
+ depth, quote = 0, None
184
+ for ch in s:
185
+ if quote:
186
+ if ch == quote:
187
+ quote = None
188
+ elif ch in "\"'":
189
+ quote = ch
190
+ elif ch in "[{":
191
+ depth += 1
192
+ elif ch in "]}":
193
+ depth -= 1
194
+ return depth <= 0
195
+
196
+
197
+ def _assign(table: dict, line: str) -> None:
198
+ m = _KEY.match(line)
199
+ if not m:
200
+ return
201
+ key, value = m.group(1).strip('"'), _value(m.group(2).strip())
202
+ if value is not _SKIP:
203
+ table[key] = value
204
+
205
+
206
+ _SKIP = object()
207
+
208
+
209
+ def _value(s: str):
210
+ if s.startswith('"""') or s.startswith("'''"):
211
+ return _SKIP
212
+ if len(s) >= 2 and s[0] == s[-1] and s[0] in "\"'":
213
+ return s[1:-1].replace('\\"', '"').replace("\\\\", "\\") if s[0] == '"' else s[1:-1]
214
+ if s in ("true", "false"):
215
+ return s == "true"
216
+ if re.fullmatch(r"[+-]?\d+", s):
217
+ return int(s)
218
+ if s.startswith("[") and s.endswith("]"):
219
+ items, buf, quote = [], "", None
220
+ for ch in s[1:-1]:
221
+ if quote:
222
+ buf += ch
223
+ if ch == quote:
224
+ quote = None
225
+ elif ch in "\"'":
226
+ quote = ch
227
+ buf += ch
228
+ elif ch == ",":
229
+ items.append(buf.strip())
230
+ buf = ""
231
+ else:
232
+ buf += ch
233
+ if buf.strip():
234
+ items.append(buf.strip())
235
+ values = [_value(i) for i in items if i]
236
+ return _SKIP if any(v is _SKIP for v in values) else values
237
+ return _SKIP
238
+
239
+
240
+ def config_from_env() -> Optional[str]:
241
+ """SLOPDAM_CONFIG points at an explicit settings file."""
242
+ return os.environ.get("SLOPDAM_CONFIG") or None
slopdam/fixer.py ADDED
@@ -0,0 +1,266 @@
1
+ # SPDX-License-Identifier: MIT
2
+ """Rewrite string literals in Python source, keeping everything else byte for byte.
3
+
4
+ The AST picks which literals to touch (log/print call arguments, or every non-docstring
5
+ string); the tokenizer gives their exact text. Only literal text changes: quotes, prefixes,
6
+ escape sequences and f-string ``{...}`` parts are kept, and the result must compile.
7
+ """
8
+ from __future__ import annotations
9
+
10
+ import ast
11
+ import io
12
+ import re
13
+ import tokenize
14
+ from dataclasses import dataclass
15
+ from typing import Iterable, List, Optional, Tuple
16
+
17
+ from .rules import Finding, Sanitizer
18
+
19
+ LOG_METHODS = {"debug", "info", "warning", "warn", "error", "exception", "critical", "fatal", "log"}
20
+ IGNORE = re.compile(r"slopdam:\s*ignore(?:\[([\w.,\s-]+)\])?")
21
+ FSTRING_START = getattr(tokenize, "FSTRING_START", None) # Python 3.12+ splits f-strings into tokens
22
+ FSTRING_END = getattr(tokenize, "FSTRING_END", None)
23
+ _PREFIX = re.compile(r"^([A-Za-z]*)('''|\"\"\"|'|\")", re.S)
24
+
25
+ Pos = Tuple[int, int] # (line, column in characters), as tokenize reports it
26
+
27
+
28
+ @dataclass(frozen=True)
29
+ class Edit:
30
+ line: int
31
+ col: int
32
+ rule_id: str
33
+ before: str
34
+ after: Optional[str] # None: report-only rule, nothing was changed
35
+
36
+
37
+ def _is_log_call(node: ast.Call) -> bool:
38
+ func = node.func
39
+ if isinstance(func, ast.Name):
40
+ return func.id == "print"
41
+ if isinstance(func, ast.Attribute) and func.attr in LOG_METHODS:
42
+ receiver = ast.unparse(func.value) if hasattr(ast, "unparse") else _dotted(func.value)
43
+ return "log" in receiver.lower()
44
+ return False
45
+
46
+
47
+ def _dotted(node: ast.AST) -> str:
48
+ if isinstance(node, ast.Name):
49
+ return node.id
50
+ if isinstance(node, ast.Attribute):
51
+ return _dotted(node.value) + "." + node.attr
52
+ return ""
53
+
54
+
55
+ def _docstring_nodes(tree: ast.AST) -> set:
56
+ out = set()
57
+ for node in ast.walk(tree):
58
+ if isinstance(node, (ast.Module, ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)) and node.body:
59
+ first = node.body[0]
60
+ if isinstance(first, ast.Expr) and isinstance(first.value, ast.Constant) and isinstance(first.value.value, str):
61
+ out.add(id(first.value))
62
+ return out
63
+
64
+
65
+ def _string_nodes(tree: ast.AST, scope: str) -> Iterable[ast.AST]:
66
+ def is_str(n):
67
+ return isinstance(n, ast.JoinedStr) or (isinstance(n, ast.Constant) and isinstance(n.value, str))
68
+
69
+ if scope == "log-calls":
70
+ for node in ast.walk(tree):
71
+ if isinstance(node, ast.Call) and _is_log_call(node):
72
+ for arg in [*node.args, *(k.value for k in node.keywords if k.arg is None)]:
73
+ if is_str(arg):
74
+ yield arg
75
+ return
76
+ docs = _docstring_nodes(tree)
77
+ for node in ast.walk(tree):
78
+ if is_str(node) and id(node) not in docs:
79
+ yield node
80
+
81
+
82
+ def _char_col(lines: List[str], line: int, byte_col: int) -> int:
83
+ """AST columns are UTF-8 byte offsets; tokenize columns are characters."""
84
+ return len(lines[line - 1].encode("utf-8")[:byte_col].decode("utf-8", errors="ignore"))
85
+
86
+
87
+ def _spans(source: str, scope: str) -> List[Tuple[Pos, Pos]]:
88
+ tree = ast.parse(source)
89
+ lines = source.splitlines(keepends=True) or [""]
90
+ out = []
91
+ for node in _string_nodes(tree, scope):
92
+ if getattr(node, "end_lineno", None) is None:
93
+ continue
94
+ start = (node.lineno, _char_col(lines, node.lineno, node.col_offset))
95
+ end = (node.end_lineno, _char_col(lines, node.end_lineno, node.end_col_offset))
96
+ out.append((start, end))
97
+ return out
98
+
99
+
100
+ def _string_tokens(source: str) -> Iterable[Tuple[Pos, Pos]]:
101
+ """(start, end) of every string literal token; a 3.12+ f-string counts as one."""
102
+ depth, fstart = 0, None
103
+ for tok in tokenize.generate_tokens(io.StringIO(source).readline):
104
+ if FSTRING_START is not None and tok.type == FSTRING_START:
105
+ if depth == 0:
106
+ fstart = tok.start
107
+ depth += 1
108
+ elif FSTRING_END is not None and tok.type == FSTRING_END:
109
+ depth -= 1
110
+ if depth == 0:
111
+ yield fstart, tok.end
112
+ elif tok.type == tokenize.STRING and depth == 0:
113
+ yield tok.start, tok.end
114
+
115
+
116
+ def _literal_parts(body: str, is_f: bool) -> List[Tuple[int, int]]:
117
+ """Index ranges of `body` that are literal text (f-string expressions excluded)."""
118
+ if not is_f:
119
+ return [(0, len(body))]
120
+ parts, i, start, n = [], 0, 0, len(body)
121
+ while i < n:
122
+ if body.startswith("{{", i) or body.startswith("}}", i):
123
+ i += 2
124
+ continue
125
+ if body[i] == "{":
126
+ parts.append((start, i))
127
+ depth, i = 1, i + 1
128
+ while i < n and depth:
129
+ depth += {"{": 1, "}": -1}.get(body[i], 0)
130
+ i += 1
131
+ start = i
132
+ continue
133
+ i += 1
134
+ parts.append((start, n))
135
+ return [(a, b) for a, b in parts if b > a]
136
+
137
+
138
+ # Placeholders that stand for a value at run time: "%s", "%(name)d", "{}", "{0:>4}".
139
+ _PLACEHOLDER = re.compile(r"%(?:\([^)]*\))?[-#0 +]*\d*(?:\.\d+)?[sdifrxXeEgGcoaub%]|\{[^{}]*\}")
140
+ _MASK_BASE = 0xF8000 # private use plane, treated as a word character by the dash rule
141
+
142
+
143
+ def _mask(body: str, is_f: bool):
144
+ """Swap f-string expressions and format placeholders for single word-like characters.
145
+
146
+ The rules then judge a dash next to ``{n}`` or ``%s`` as sitting between words, and can
147
+ never edit inside an expression. Returns (masked text, original pieces, index mapper).
148
+ """
149
+ spans = []
150
+ if is_f:
151
+ cursor = 0
152
+ for a, b in _literal_parts(body, True):
153
+ if a > cursor:
154
+ spans.append((cursor, a))
155
+ cursor = b
156
+ if cursor < len(body):
157
+ spans.append((cursor, len(body)))
158
+ else:
159
+ spans = [m.span() for m in _PLACEHOLDER.finditer(body) if m.group(0) != "%%"]
160
+ out, pieces, index, last = [], [], [], 0
161
+ for a, b in spans:
162
+ out.append(body[last:a])
163
+ index.extend(range(last, a))
164
+ out.append(chr(_MASK_BASE + len(pieces)))
165
+ index.append(a)
166
+ pieces.append(body[a:b])
167
+ last = b
168
+ out.append(body[last:])
169
+ index.extend(range(last, len(body) + 1))
170
+ return "".join(out), pieces, lambda i: index[min(i, len(index) - 1)]
171
+
172
+
173
+ def _unmask(text: str, pieces: List[str]) -> Optional[str]:
174
+ seen = [ord(c) - _MASK_BASE for c in text if _MASK_BASE <= ord(c) < _MASK_BASE + len(pieces)]
175
+ if seen != list(range(len(pieces))):
176
+ return None
177
+ return re.sub("[%s-%s]" % (chr(_MASK_BASE), chr(_MASK_BASE + max(len(pieces), 1))),
178
+ lambda m: pieces[ord(m.group(0)) - _MASK_BASE], text)
179
+
180
+
181
+ def _rewrite(text: str, sanitizer: Sanitizer, ignored: List[str], quote: str, raw: bool) -> Tuple[str, List[Finding]]:
182
+ """Apply each finding's fix at its span. Returns new text and the findings (report-only ones too)."""
183
+ out, last, applied = [], 0, []
184
+ for f in sanitizer.findings(text):
185
+ if f.start < last or any(f.rule_id == i or f.rule_id.startswith(i + ".") for i in ignored):
186
+ continue
187
+ if f.fix is None:
188
+ applied.append(f)
189
+ continue
190
+ fix = f.fix
191
+ if quote[0] in fix or "\\" in fix:
192
+ if raw:
193
+ continue # cannot escape inside a raw string
194
+ fix = fix.replace("\\", "\\\\").replace(quote[0], "\\" + quote[0])
195
+ out.append(text[last:f.start])
196
+ out.append(fix)
197
+ last = f.end
198
+ applied.append(f)
199
+ out.append(text[last:])
200
+ return "".join(out), applied
201
+
202
+
203
+ def fix_source(source: str, sanitizer: Sanitizer, scope: str = "log-calls") -> Tuple[str, List[Edit]]:
204
+ """Return the fixed source and the edits made. Raises SyntaxError if the input does not parse."""
205
+ targets = _spans(source, scope)
206
+ if not targets:
207
+ return source, []
208
+ lines = source.splitlines(keepends=True)
209
+ offsets = [0]
210
+ for line in lines:
211
+ offsets.append(offsets[-1] + len(line))
212
+
213
+ def absolute(pos: Pos) -> int:
214
+ return offsets[pos[0] - 1] + pos[1]
215
+
216
+ replacements, edits = [], []
217
+ for start, end in _string_tokens(source):
218
+ if not any(s <= start and end <= e for s, e in targets):
219
+ continue
220
+ line_text = lines[start[0] - 1]
221
+ ignore = IGNORE.search(line_text)
222
+ if ignore and not ignore.group(1):
223
+ continue
224
+ ignored = [i.strip() for i in ignore.group(1).split(",")] if ignore else []
225
+ a, b = absolute(start), absolute(end)
226
+ token = source[a:b]
227
+ m = _PREFIX.match(token)
228
+ if not m or "b" in m.group(1).lower():
229
+ continue
230
+ prefix, quote = m.group(1), m.group(2)
231
+ body_start = len(prefix) + len(quote)
232
+ body = token[body_start:len(token) - len(quote)]
233
+ masked, pieces, where = _mask(body, "f" in prefix.lower())
234
+ text, applied = _rewrite(masked, sanitizer, ignored, quote, "r" in prefix.lower())
235
+ new_body = _unmask(text, pieces)
236
+ if new_body is None:
237
+ continue # a rule swallowed a placeholder: leave this literal alone
238
+ # Do not leave a space at either end that the original literal did not have.
239
+ if body == body.lstrip():
240
+ new_body = new_body.lstrip(" \t")
241
+ if body == body.rstrip():
242
+ new_body = new_body.rstrip(" \t")
243
+ for f in applied:
244
+ edits.append(Edit(start[0], start[1] + body_start + where(f.start) + 1, f.rule_id, f.text,
245
+ None if f.fix is None else f.fix))
246
+ new_token = token[:body_start] + new_body + quote
247
+ if new_token != token:
248
+ replacements.append((a, b, new_token))
249
+
250
+ for a, b, new in sorted(replacements, reverse=True):
251
+ source = source[:a] + new + source[b:]
252
+ compile(source, "<slopdam fix>", "exec") # never hand back code that does not compile
253
+ return source, edits
254
+
255
+
256
+ def fix_file(path: str, sanitizer: Sanitizer, scope: str = "log-calls", write: bool = True) -> Tuple[Optional[str], List[Edit]]:
257
+ """Fix one Python file. Returns (new source or None when unchanged, edits)."""
258
+ with open(path, encoding="utf-8") as f:
259
+ source = f.read()
260
+ new, edits = fix_source(source, sanitizer, scope)
261
+ if new == source:
262
+ return None, []
263
+ if write:
264
+ with open(path, "w", encoding="utf-8") as f:
265
+ f.write(new)
266
+ return new, edits
slopdam/rules.py ADDED
@@ -0,0 +1,281 @@
1
+ # SPDX-License-Identifier: MIT
2
+ """Rule definitions and the sanitizer engine.
3
+
4
+ Detection is shared by every profile; only the action differs:
5
+
6
+ reader (default) plain Korean for end users
7
+ dev grep/parse friendly output for developers and log pipelines
8
+
9
+ An action is a replacement string, a callable ``(match, ctx) -> str``, or ``None``
10
+ (report only, used by the linter). A rule whose ``actions`` has no entry for a
11
+ profile is inactive in that profile. See docs/RULES.md for the rule list.
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import logging
16
+ import re
17
+ from dataclasses import dataclass, field
18
+ from typing import Callable, Dict, Iterable, Optional, Union
19
+
20
+ PROFILES = ("reader", "dev")
21
+ LEVELS = ("basic", "strong", "strict")
22
+ DASH_INLINE = {"comma": ", ", "colon": ": ", "hyphen": " - ", "space": " "}
23
+
24
+ Action = Union[str, Callable[["re.Match[str]", "Context"], str], None]
25
+
26
+
27
+ @dataclass(frozen=True)
28
+ class Context:
29
+ profile: str
30
+ dash_inline: str
31
+ levelno: Optional[int] = None
32
+
33
+
34
+ @dataclass(frozen=True)
35
+ class Rule:
36
+ id: str
37
+ level: str
38
+ pattern: "re.Pattern[str]"
39
+ actions: Dict[str, Action]
40
+ description: str
41
+ ascii: bool = False # can match pure-ASCII text (kept on the fast path)
42
+
43
+ def active(self, profile: str, level: str) -> bool:
44
+ return profile in self.actions and LEVELS.index(self.level) <= LEVELS.index(level)
45
+
46
+
47
+ @dataclass(frozen=True)
48
+ class Finding:
49
+ rule_id: str
50
+ description: str
51
+ start: int
52
+ end: int
53
+ text: str
54
+ fix: Optional[str] = field(default=None)
55
+
56
+
57
+ def _both(action: Action) -> Dict[str, Action]:
58
+ return {"reader": action, "dev": action}
59
+
60
+
61
+ # Protected spans are swapped for one private-use character each and restored at the end.
62
+ _PUA = 0xF0000
63
+ _PUA_RE = re.compile("[\U000F0000-\U000FFFFD]")
64
+ _PROTECT = re.compile(
65
+ r"(?P<url>https?://[^\s<>\"'`]+)"
66
+ r"|`(?P<code>[^`\n]+)`"
67
+ r"|(?<=\w=)(?P<kv>\"[^\"\n]*\"|'[^'\n]*'|[^\s,;)]+)"
68
+ r"|(?<![\w/])(?P<path>(?:~|\.{1,2})?/[^\s\"'`,;)]+)"
69
+ )
70
+
71
+ # --- dash family -------------------------------------------------------------
72
+ HYPHENS = "‐‑−﹣-"
73
+ DASHES = "‒–—―⸺⸻﹘︱︲⹀"
74
+ EU = "ㅡᅳᅳ" # ㅡ, halfwidth ㅡ, conjoining jungseong ㅡ (NFKC target of ㅡ)
75
+ LINES = "─━┄-┋╌-╏═⎯⏤➖"
76
+ _CHOSEONG = re.compile("[ᄀ-ᅟꥠ-ꥼ]")
77
+ _OPENERS, _CLOSERS = "([{", ")]}%"
78
+
79
+
80
+ def _wordish(c: str, extra: str) -> bool:
81
+ return c.isalnum() or c in extra or bool(_PUA_RE.match(c))
82
+
83
+
84
+ def _dash_run(m: "re.Match[str]", ctx: Context) -> str:
85
+ s, (start, end) = m.string, m.span()
86
+ lead, run = m.group(1), m.group(2)
87
+ if run[0] == "ᅳ" and start + len(lead) > 0 and _CHOSEONG.match(s[start + len(lead) - 1]):
88
+ return m.group(0) # part of a composed old-style syllable
89
+ prev = s[start - 1] if start else "\n"
90
+ nxt = s[end] if end < len(s) else "\n"
91
+ inner = _wordish(prev, _CLOSERS) and _wordish(nxt, _OPENERS)
92
+ if len(run) >= 2: # separator or emoticon: ㅡㅡ, ──, ——
93
+ return " " if inner else ""
94
+ if prev.isdigit() and nxt.isdigit():
95
+ return "~" if ctx.profile == "reader" else "-"
96
+ head = s[s.rfind("\n", 0, start) + 1 : start]
97
+ if not head.strip() and m.group(3): # list marker at line start
98
+ return lead + "- "
99
+ if inner:
100
+ return ctx.dash_inline if ctx.profile == "reader" else " - "
101
+ return ""
102
+
103
+
104
+ # --- emoji (section 5) ----------------------------------------------------------
105
+ _EMOJI_CLASS = (
106
+ "\U0001F000-\U0001FAFF☀-➿⬀-⯿⌀-⏿■-◿"
107
+ "‼⁉ℹ〰〽㊗㊙Ⓜ"
108
+ "\U0001F1E6-\U0001F1FF\U000E0020-\U000E007F️⃣‍"
109
+ )
110
+ _STATUS = {
111
+ "[OK]": "✅✔☑\U0001F7E2\U0001F197",
112
+ "[FAIL]": "❌✖\U0001F534⛔\U0001F6AB❎",
113
+ "[WARN]": "⚠\U0001F7E1\U0001F6A8",
114
+ }
115
+ _STATUS_OF = {c: tag for tag, chars in _STATUS.items() for c in chars}
116
+ _REDUNDANT = {"[FAIL]": logging.ERROR, "[WARN]": logging.WARNING}
117
+
118
+
119
+ def _status_tag(m: "re.Match[str]", ctx: Context) -> str:
120
+ tag = _STATUS_OF[m.group(1)]
121
+ if ctx.levelno is not None and ctx.levelno >= _REDUNDANT.get(tag, logging.CRITICAL + 1):
122
+ return "" # the log level already says it
123
+ return tag + " "
124
+
125
+
126
+ _FULLWIDTH = str.maketrans(",:;()!?[]{}", ",:;()!?[]{}")
127
+
128
+
129
+ def _r(id: str, level: str, pattern: str, actions: Dict[str, Action], description: str,
130
+ flags: int = 0, ascii: bool = False) -> Rule:
131
+ return Rule(id, level, re.compile(pattern, flags), actions, description, ascii)
132
+
133
+
134
+ # Order matters: later rules assume earlier ones already ran.
135
+ RULES: tuple = (
136
+ _r("space.invisible", "basic", "[​‌⁠­]", _both(""), "zero-width or invisible character"),
137
+ _r("space.nbsp", "basic", "[   ]", _both(" "), "non-breaking or ideographic space"),
138
+ _r("width.fullwidth", "basic", "[,:;()!?[]{}]", _both(lambda m, c: m.group(0).translate(_FULLWIDTH)), "full-width punctuation"),
139
+ _r("quote.double", "basic", "[“”„]", _both('"'), "curly double quote"),
140
+ _r("quote.single", "basic", "[‘’‚]", _both("'"), "curly single quote"),
141
+ _r("punct.ellipsis", "basic", "[…⋯]", _both("..."), "ellipsis character"),
142
+ _r("punct.arrow", "basic", "[↔⇔]️?", _both("<->"), "arrow symbol"),
143
+ _r("punct.arrow", "basic", "[→⇒➜➡➔⟶]️?", _both("->"), "arrow symbol"),
144
+ _r("punct.arrow", "basic", "[←⇐⬅⟵]️?", _both("<-"), "arrow symbol"),
145
+ _r("punct.bullet", "basic", "^([ \t]*)[•◦▪●▶►][ \t]*", _both(r"\1- "), "bullet symbol", re.M),
146
+ _r("emoji.status", "basic", "\\A[ \t]*([" + "".join(_STATUS_OF) + "])️?[ \t]*", {"dev": _status_tag}, "status emoji"),
147
+ # An emoji takes its trailing spaces with it, so "⚠️ text" on any line leaves no stray indent.
148
+ # Bare combining marks (keycap "1️⃣ ") keep the space, or "1 단계" would become "1단계".
149
+ _r("emoji.pictograph", "basic",
150
+ "[" + _EMOJI_CLASS.replace("️⃣‍", "") + "][" + _EMOJI_CLASS + "]*[ \t]*|[️⃣‍]+",
151
+ _both(""), "emoji or pictograph"),
152
+ _r("md.heading", "basic", r"^[ \t]*#{1,6}[ \t]+", _both(""), "markdown heading marker", re.M, ascii=True),
153
+ _r("md.fence", "basic", r"^[ \t]*```[^\n]*\n?", _both(""), "markdown code fence", re.M, ascii=True),
154
+ _r("md.hr", "basic", r"^[ \t]*([-*_])[ \t]*(?:\1[ \t]*){2,}$", _both(""), "markdown horizontal rule", re.M, ascii=True),
155
+ # Only **bold**: __x__ is far more often a Python name (__name__, __init__) than emphasis.
156
+ _r("md.emphasis", "basic", r"\*\*(.+?)\*\*", _both(lambda m, c: m.group(1)), "markdown bold", ascii=True),
157
+ _r("dash.hyphen", "basic", f"[{HYPHENS}]", _both("-"), "Unicode hyphen or minus"),
158
+ _r("dash.wave", "basic", "[〜~]", _both("~"), "wave dash or full-width tilde"),
159
+ _r("dash.run", "basic", f"([ \t]*)([{DASHES}{EU}{LINES}]+)([ \t]*)", _both(_dash_run), "dash, 'ㅡ' or line character"),
160
+ _r("dash.cjk-run", "basic", "[一ーー]{3,}", _both(""), "CJK look-alike separator"),
161
+ _r("dash.ascii-run", "basic", r"([=\-*#~_+])\1{4,}", _both(""), "ASCII separator", ascii=True),
162
+ _r("tone.exclaim", "basic", "(?<=[가-힣])!+", {"reader": "."}, "exclamation after Korean text"),
163
+ _r("tone.filler", "strong", r"(?:성공적으로|완벽하게|무사히|드디어)[ \t]*", {"reader": "", "dev": None}, "filler phrase"),
164
+ _r("tone.chatty", "basic", r"\b(?:Let's|Here's|Voila|Awesome)\b|해볼게요|해보겠습니다|살펴보겠습니다", _both(None), "chatty phrase", ascii=True),
165
+ _r("symbol.non-ascii", "strict", "[^\\w\\s\x00-\x7f\U000F0000-\U000FFFFD]", _both(None), "non-ASCII symbol"),
166
+ )
167
+
168
+ _MULTI_SPACE = re.compile(r"(?<=\S)[ \t]{2,}")
169
+
170
+
171
+ class Sanitizer:
172
+ def __init__(self, profile: str = "reader", level: str = "basic", disable: Iterable[str] = (),
173
+ dash_inline: str = "colon", extra: Iterable[Rule] = ()):
174
+ if profile not in PROFILES:
175
+ raise ValueError(f"unknown profile {profile!r}, expected one of {PROFILES}")
176
+ if level not in LEVELS:
177
+ raise ValueError(f"unknown level {level!r}, expected one of {LEVELS}")
178
+ if dash_inline not in DASH_INLINE:
179
+ raise ValueError(f"unknown dash_inline {dash_inline!r}, expected one of {tuple(DASH_INLINE)}")
180
+ off = tuple(disable)
181
+ self.profile = profile
182
+ self.dash_inline = DASH_INLINE[dash_inline]
183
+ self.rules = tuple(
184
+ r for r in (*RULES, *extra)
185
+ if r.active(profile, level) and not any(r.id == d or r.id.startswith(d + ".") for d in off)
186
+ )
187
+ self._ascii_rules = tuple(r for r in self.rules if r.ascii)
188
+
189
+ def _context(self, levelno: Optional[int]) -> Context:
190
+ return Context(self.profile, self.dash_inline, levelno)
191
+
192
+ def _apply(self, text: str, ctx: Context) -> str:
193
+ for rule in self._ascii_rules if text.isascii() else self.rules:
194
+ action = rule.actions[self.profile]
195
+ if action is None:
196
+ continue
197
+ if callable(action):
198
+ text = rule.pattern.sub(lambda m, a=action: a(m, ctx), text)
199
+ else:
200
+ text = rule.pattern.sub(action, text)
201
+ return text
202
+
203
+ def clean(self, text: str, levelno: Optional[int] = None) -> str:
204
+ ctx = self._context(levelno)
205
+ saved: list = []
206
+
207
+ def protect(m: "re.Match[str]") -> str:
208
+ kept = m.group(0)
209
+ # reader drops the backticks, but only when the code itself is clean;
210
+ # otherwise its content would be exposed to the rules on the next pass.
211
+ if m.lastgroup == "code" and self.profile == "reader" and self._apply(m.group("code"), ctx) == m.group("code"):
212
+ kept = m.group("code")
213
+ saved.append(kept)
214
+ return chr(_PUA + len(saved) - 1)
215
+
216
+ text = self._apply(_PROTECT.sub(protect, text), ctx)
217
+ text = _PUA_RE.sub(lambda m: saved[ord(m.group(0)) - _PUA], text)
218
+
219
+ lines = []
220
+ for i, line in enumerate(text.splitlines()):
221
+ line = _MULTI_SPACE.sub(" ", line)
222
+ line = line.strip() if i == 0 else line.rstrip()
223
+ if line:
224
+ lines.append(line)
225
+ return "\n".join(lines)
226
+
227
+ def findings(self, text: str) -> list:
228
+ """What clean() would change, with spans in the original text.
229
+
230
+ Rules run in the same order and on the same intermediate text as clean(), so a
231
+ fix reflects earlier rules (e.g. a dash next to ``**bold**`` is judged after the
232
+ markdown is gone). Each character of the working text remembers the span of the
233
+ original text it came from.
234
+ """
235
+ ctx = self._context(None)
236
+ cur, spans = [], [] # working text and, per character, its (start, end) in `text`
237
+ last = 0
238
+ for m in _PROTECT.finditer(text):
239
+ for i in range(last, m.start()):
240
+ cur.append(text[i])
241
+ spans.append((i, i + 1))
242
+ cur.append(chr(_PUA)) # protected spans are opaque to every rule
243
+ spans.append(m.span())
244
+ last = m.end()
245
+ for i in range(last, len(text)):
246
+ cur.append(text[i])
247
+ spans.append((i, i + 1))
248
+ work = "".join(cur)
249
+
250
+ found = []
251
+ for rule in self._ascii_rules if work.isascii() else self.rules:
252
+ action = rule.actions[self.profile]
253
+ pieces, new_spans, pos = [], [], 0
254
+ for m in rule.pattern.finditer(work):
255
+ if action is None:
256
+ fix = None
257
+ elif callable(action):
258
+ fix = action(m, ctx)
259
+ else:
260
+ fix = m.expand(action)
261
+ if m.end() > m.start():
262
+ start, end = spans[m.start()][0], spans[m.end() - 1][1]
263
+ if m.group(0).strip() and fix != m.group(0):
264
+ found.append(Finding(rule.id, rule.description, start, end, text[start:end].strip(), fix))
265
+ if fix is None or fix == m.group(0):
266
+ continue
267
+ pieces.append(work[pos:m.start()])
268
+ new_spans.extend(spans[pos:m.start()])
269
+ pieces.append(fix)
270
+ new_spans.extend([(start, end)] * len(fix))
271
+ pos = m.end()
272
+ if pieces:
273
+ pieces.append(work[pos:])
274
+ new_spans.extend(spans[pos:])
275
+ work, spans = "".join(pieces), new_spans
276
+ found.sort(key=lambda f: (f.start, -f.end))
277
+ kept: list = []
278
+ for f in found: # drop hits nested inside an earlier one (e.g. emoji.status + emoji.pictograph)
279
+ if not any(k.start <= f.start and f.end <= k.end for k in kept):
280
+ kept.append(f)
281
+ return kept
slopdam/stdlib.py ADDED
@@ -0,0 +1,51 @@
1
+ # SPDX-License-Identifier: MIT
2
+ """Adapters for the standard library ``logging`` module.
3
+
4
+ Typical routing: one logger, a ``reader`` handler for people and a ``dev`` handler for files.
5
+
6
+ console.setFormatter(PlainFormatter("%(message)s")) # reader (default)
7
+ file.setFormatter(PlainFormatter("%(asctime)s %(message)s", profile="dev"))
8
+ """
9
+ from __future__ import annotations
10
+
11
+ import logging as _logging
12
+ from typing import Optional
13
+
14
+ from .rules import Sanitizer
15
+
16
+
17
+ class PlainFormatter(_logging.Formatter):
18
+ """Formatter that cleans the message before formatting.
19
+
20
+ The original record is left untouched so other handlers still see it.
21
+ Exception tracebacks are never modified.
22
+ """
23
+
24
+ def __init__(self, *args, profile: str = "reader", level: str = "basic",
25
+ sanitizer: Optional[Sanitizer] = None, **kwargs):
26
+ super().__init__(*args, **kwargs)
27
+ self.sanitizer = sanitizer or Sanitizer(profile, level)
28
+
29
+ def format(self, record: _logging.LogRecord) -> str:
30
+ copy = _logging.makeLogRecord(record.__dict__)
31
+ copy.msg, copy.args = self.sanitizer.clean(record.getMessage(), record.levelno), None
32
+ return super().format(copy)
33
+
34
+
35
+ class PlainFilter(_logging.Filter):
36
+ """Filter that cleans the message in place and drops records that end up empty
37
+ (for example a log call that only prints a separator line)."""
38
+
39
+ def __init__(self, name: str = "", *, profile: str = "reader", level: str = "basic",
40
+ sanitizer: Optional[Sanitizer] = None):
41
+ super().__init__(name)
42
+ self.sanitizer = sanitizer or Sanitizer(profile, level)
43
+
44
+ def filter(self, record: _logging.LogRecord) -> bool:
45
+ if not super().filter(record):
46
+ return False
47
+ cleaned = self.sanitizer.clean(record.getMessage(), record.levelno)
48
+ if not cleaned and not record.exc_info:
49
+ return False
50
+ record.msg, record.args = cleaned, None
51
+ return True
@@ -0,0 +1,140 @@
1
+ Metadata-Version: 2.5
2
+ Name: slopdam
3
+ Version: 0.2.0
4
+ Summary: Logger formatter and linter that strips AI-generated patterns (em dashes, emoji, separators, markdown) from log output
5
+ Project-URL: Homepage, https://github.com/slopdam/slopdam
6
+ Project-URL: Repository, https://github.com/slopdam/slopdam
7
+ Project-URL: Issues, https://github.com/slopdam/slopdam/issues
8
+ Project-URL: Changelog, https://github.com/slopdam/slopdam/blob/main/CHANGELOG.md
9
+ Author: The slopdam Authors
10
+ License-Expression: MIT
11
+ License-File: LICENSE
12
+ License-File: THIRD_PARTY_NOTICES.md
13
+ Keywords: ai,formatter,korean,linter,logging,slop
14
+ Classifier: Development Status :: 3 - Alpha
15
+ Classifier: Intended Audience :: Developers
16
+ Classifier: Natural Language :: English
17
+ Classifier: Natural Language :: Korean
18
+ Classifier: Programming Language :: Python :: 3
19
+ Classifier: Topic :: System :: Logging
20
+ Requires-Python: >=3.9
21
+ Description-Content-Type: text/markdown
22
+
23
+ # slopdam
24
+
25
+ [English](https://github.com/slopdam/slopdam/blob/main/README.md) | [한국어](https://github.com/slopdam/slopdam/blob/main/README.ko.md)
26
+
27
+ [![ci](https://github.com/slopdam/slopdam/actions/workflows/ci.yml/badge.svg)](https://github.com/slopdam/slopdam/actions/workflows/ci.yml)
28
+ [![license: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/slopdam/slopdam/blob/main/LICENSE)
29
+
30
+ **A dam for AI slop in your logs.** slopdam removes the traces AI coding tools leave in log messages and console output: em dashes and look-alike lines (including the Korean `ㅡ`), decorative emoji, separator lines, markdown bold, curly quotes and invisible characters.
31
+
32
+ It works in two places:
33
+
34
+ - **When logs are written**: a `logging` formatter and filter clean every message on its way out, with no change to your code.
35
+ - **Before code is committed**: a linter reports traces in log calls, and a fixer rewrites the string literals in Python files.
36
+
37
+ The same trace becomes different text depending on who reads it:
38
+
39
+ ```
40
+ input : ✅ **Saved** — 3 rows processed 🎉
41
+ reader: Saved: 3 rows processed
42
+ dev : [OK] Saved - 3 rows processed
43
+ ```
44
+
45
+ - `reader` (default): plain text for people, such as CLI messages, alerts and reports.
46
+ - `dev`: grep-friendly text for log files and collectors. Status emoji become `[OK]`, `[FAIL]` and `[WARN]` instead of disappearing.
47
+
48
+ slopdam uses deterministic rules (regular expressions and character tables). It does not try to guess whether a model wrote the text: the same input always gives the same output, and every change can be traced to a rule ID.
49
+
50
+ ## Install
51
+
52
+ slopdam needs Python 3.9 or later and has no runtime dependencies.
53
+
54
+ ```bash
55
+ pip install slopdam
56
+ ```
57
+
58
+ ## Clean log output
59
+
60
+ ```python
61
+ import logging
62
+ from slopdam import PlainFilter, PlainFormatter
63
+
64
+ console = logging.StreamHandler() # read by people
65
+ console.setFormatter(PlainFormatter("%(message)s")) # reader profile
66
+ console.addFilter(PlainFilter()) # drop lines that were only separators
67
+
68
+ file = logging.FileHandler("app.log") # read by tools
69
+ file.setFormatter(PlainFormatter("%(asctime)s %(levelname)s %(message)s", profile="dev"))
70
+
71
+ logging.basicConfig(level=logging.INFO, handlers=[console, file])
72
+ logging.info("✅ Synced %s rows — done", 3)
73
+ # console: Synced 3 rows: done
74
+ # app.log: 2026-11-23 ... INFO [OK] Synced 3 rows - done
75
+ ```
76
+
77
+ - The original log record is never modified, so each handler can use its own profile.
78
+ - Tracebacks, URLs, inline code, `key=value` values and file paths are left untouched.
79
+
80
+ ## Lint and fix source code
81
+
82
+ ```bash
83
+ slopdam check src/ # report traces in log and print calls; exit 1 if any
84
+ slopdam --profile dev check src/ # suggestions in the dev style
85
+ slopdam fix --diff src/ # preview fixes to Python string literals
86
+ slopdam fix src/ # apply them
87
+ ```
88
+
89
+ ```
90
+ src/sync.py:8:18: emoji.pictograph '✅' -> ''
91
+ src/sync.py:8:25: dash.run 'ㅡ' -> ': '
92
+ ```
93
+
94
+ Python files are read through the syntax tree, so slopdam knows exactly where each string starts and ends. `slopdam fix` changes only the literal text: quotes, prefixes, escape sequences, f-string expressions and placeholders such as `%s` stay as they are, and nothing is written unless the result compiles. Other languages are checked line by line.
95
+
96
+ To skip a line, add `# slopdam: ignore`, or `# slopdam: ignore[emoji]` to skip one rule group.
97
+
98
+ ## Configuration
99
+
100
+ slopdam reads `slopdam.toml`, or the `[tool.slopdam]` table in `pyproject.toml`. Command line options take precedence.
101
+
102
+ ```toml
103
+ profile = "dev"
104
+ level = "basic"
105
+ disable = ["emoji.status"]
106
+
107
+ [lint]
108
+ scope = "log-calls" # log-calls | strings | all
109
+ exclude = ["tests/**"]
110
+ ```
111
+
112
+ ## pre-commit and CI
113
+
114
+ ```yaml
115
+ repos:
116
+ - repo: https://github.com/slopdam/slopdam
117
+ rev: v0.2.0
118
+ hooks:
119
+ - id: slopdam
120
+ ```
121
+
122
+ Exit codes: `0` clean, `1` traces found (or `fix --diff` has changes), `2` usage or configuration error.
123
+
124
+ ## Pipe
125
+
126
+ ```bash
127
+ tail -f app.log | slopdam clean
128
+ ```
129
+
130
+ ## Rules
131
+
132
+ Every rule has an ID such as `dash.run` or `emoji.pictograph`, and can be turned off by ID or by group (`--disable emoji`). Profiles decide what a trace becomes, and levels decide how far slopdam goes: `basic` (default), `strong` (also removes filler words) and `strict` (also reports any non-ASCII symbol). See [docs/RULES.md](https://github.com/slopdam/slopdam/blob/main/docs/RULES.md) for the full catalog.
133
+
134
+ ## Contributing
135
+
136
+ Contributions are welcome. Please read [CONTRIBUTING.md](https://github.com/slopdam/slopdam/blob/main/CONTRIBUTING.md); every commit needs a DCO sign-off (`git commit -s`). Report security issues privately as described in [SECURITY.md](https://github.com/slopdam/slopdam/blob/main/SECURITY.md). Project decisions follow [GOVERNANCE.md](https://github.com/slopdam/slopdam/blob/main/GOVERNANCE.md), and the community follows the [Code of Conduct](https://github.com/slopdam/slopdam/blob/main/CODE_OF_CONDUCT.md).
137
+
138
+ ## License
139
+
140
+ [MIT](https://github.com/slopdam/slopdam/blob/main/LICENSE). Third-party components are listed in [THIRD_PARTY_NOTICES.md](https://github.com/slopdam/slopdam/blob/main/THIRD_PARTY_NOTICES.md).
@@ -0,0 +1,12 @@
1
+ slopdam/__init__.py,sha256=k7PCGlPqcV0ajAK5BIl5L_iVMgTZhLx0P440ophpWcQ,414
2
+ slopdam/cli.py,sha256=lSwItrlaPmlNRYEgbaGgx9T0bmxwKeMcdMNBLGXLUZo,8744
3
+ slopdam/config.py,sha256=PmivGvLMF6B7Jh2kCkHnJMnQFcL86cuffCSifl5c8F0,7681
4
+ slopdam/fixer.py,sha256=-S6yDAR1zUkHrKL0FxtjpFZze5JYBrxqlYKxP9z5tVU,10407
5
+ slopdam/rules.py,sha256=ub5MMF1X5IKFnmjOqg191gBsQT9JMz6qxIng6aiXOGs,12700
6
+ slopdam/stdlib.py,sha256=GM5nP7BT7lOinPGKt0oOqupcj4Z__bZzFV_zLYdZJD8,1985
7
+ slopdam-0.2.0.dist-info/METADATA,sha256=oZ-ZelubJomXTOMCR4BVvKQnXNYgURvnzV_i8tYpgWI,6111
8
+ slopdam-0.2.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
9
+ slopdam-0.2.0.dist-info/entry_points.txt,sha256=1Un1jFidZNIjjzBsRhFJbOYidK_LROUqYZd2H37BCts,45
10
+ slopdam-0.2.0.dist-info/licenses/LICENSE,sha256=53DCEXK8pz_GOvyJbiiZo9XU1v4tlWNATBAPVEOzgn8,1076
11
+ slopdam-0.2.0.dist-info/licenses/THIRD_PARTY_NOTICES.md,sha256=C3V7zGJTh8XbIbLtc_O_CFezPHJhxP7AWAJb5I9aFWk,1132
12
+ slopdam-0.2.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.33.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ slopdam = slopdam.cli:main
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 The slopdam Authors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,25 @@
1
+ # Third-Party Notices
2
+
3
+ External components that slopdam includes or uses, and their licenses. The policy for adding new ones is in [CONTRIBUTING.md](CONTRIBUTING.md#dependencies-and-data).
4
+
5
+ ## Current state (0.2.0)
6
+
7
+ - **Runtime dependencies: none.** slopdam uses only the Python standard library.
8
+ - **Bundled third-party code or data: none.**
9
+
10
+ ## Development dependencies (not shipped)
11
+
12
+ | Component | Used for | License (SPDX) |
13
+ |---|---|---|
14
+ | [pytest](https://pypi.org/project/pytest/) | Tests | `MIT` |
15
+ | [hatchling](https://pypi.org/project/hatchling/) | Build backend | `MIT` |
16
+
17
+ ## Planned data
18
+
19
+ When any of the following is bundled, its full license text will be added here and the data file will record its source and version.
20
+
21
+ | Data | Purpose | License (SPDX) | Source |
22
+ |---|---|---|---|
23
+ | Unicode `emoji-data.txt` | Generate the emoji ranges | `Unicode-3.0` | https://www.unicode.org/Public/UCD/latest/ucd/emoji/ |
24
+ | Unicode `confusables.txt` (UTS #39) | Reference for dash look-alikes | `Unicode-3.0` | https://www.unicode.org/Public/security/latest/ |
25
+ | Unicode license text | | | https://www.unicode.org/license.txt |