slopcount 0.1.0__py3-none-any.whl

slopcount/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

slopcount/app.py

@@ -0,0 +1,206 @@
+from __future__ import annotations
+
+import sys
+from collections import Counter
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from slopcount import scc
+from slopcount.detectors.code_style import CodeStyleDetector
+from slopcount.detectors.docs_bloat import DocsBloatDetector, repo_bloat_evidence
+from slopcount.detectors.env_markers import EnvMarkerDetector
+from slopcount.detectors.phrase import PhraseDetector
+from slopcount.evidence import (
+    Evidence,
+    LanguageRow,
+    Report,
+    ScannedFile,
+    VolumeStats,
+    aggregate_slop,
+    read_text,
+    safe_ratio,
+)
+from slopcount.i18n import _
+from slopcount.metrics.costs import attribute_locomo, split_cocomo
+from slopcount.metrics.slocomo import compute as slocomo_compute
+from slopcount.rules import load_languages, load_rules
+
+
+@dataclass
+class Options:
+    paths: list[str] = field(default_factory=lambda: ["."])
+    evidence: bool = False
+    json_out: bool = False
+    csv_out: bool = False
+    history: int | None = None
+    perplexity: bool = False
+    rules: list[Path] = field(default_factory=list)
+    lang: str | None = None
+    personcost: float = 4690.50
+    overhead: float = 2.4
+    coffee_price: float = 4.0
+    no_therapy: bool = False
+    wide: bool = False
+    scc_path: str | None = None
+
+
+def run(opts: Options) -> Report:
+    root = Path(opts.paths[0])
+    if len(opts.paths) > 1:
+        print(
+            _("slopcount: multiple paths given, scanning only the first: %s") % opts.paths[0],
+            file=sys.stderr,
+        )
+    langmap = load_languages(opts.rules)
+    manifest = scc.collect(
+        root, personcost=opts.personcost, overhead=opts.overhead, scc_path=opts.scc_path
+    )
+    # kind/language из каталога языков; неизвестный scc-язык → ("data", None)
+    files: list[ScannedFile] = []
+    for f in manifest.files:
+        kind_lang = langmap.get(f.language_name, ("data", None))
+        files.append(ScannedFile(f.path, kind_lang[1], kind_lang[0], f.size))
+
+    # ── Фаза volume: объём проекта из манифеста (спека §4) ──────────────
+    # SLOC/комментарии/когнитива — по манифесту целиком: нечитаемые (skip)
+    # файлы входят в объём, как и в COCOMO/LOCOMO scc (спека §4);
+    # детекторы такие файлы по-прежнему не видят.
+    volume = VolumeStats(files_total=len(manifest.files))
+    lang_files: Counter[str] = Counter()
+    lang_sloc: Counter[str] = Counter()
+    bucket_lines: Counter[str] = Counter()  # kind → Σ Code (корзины стоимостей)
+    for f, sf in zip(manifest.files, files, strict=True):
+        bucket_lines[sf.kind] += f.code
+        if sf.kind != "code":
+            continue
+        volume.sloc += f.code
+        volume.comment_lines += f.comment
+        volume.complexity += f.complexity
+        volume.cognitive_total += f.cognitive
+        lang_files[f.language_name] += 1
+        lang_sloc[f.language_name] += f.code
+    volume.languages = [
+        LanguageRow(lang, lang_files[lang], lang_sloc[lang])
+        for lang in sorted(lang_sloc, key=lambda lg: (-lang_sloc[lg], lg))
+    ]
+    docs_bucket_lines = bucket_lines["markdown"] + bucket_lines["prose"]
+
+    # ── Фаза detect: один проход по читаемым файлам ─────────────────────
+    phrase = PhraseDetector(load_rules(opts.rules))
+    docs_bloat = DocsBloatDetector()
+    style_detector = CodeStyleDetector()
+    pplx = None
+    if opts.perplexity:
+        from slopcount.detectors.perplexity import PerplexityDetector, available
+
+        if not available():
+            raise RuntimeError(
+                _(
+                    "slopcount: --perplexity requires extras; "
+                    "pipx install 'slopcount[perplexity]' and "
+                    "python -m slopcount.download_model"
+                )
+            )
+        pplx = PerplexityDetector()
+    by_path = {f.path: f for f in manifest.files}
+    evidences: list[Evidence] = []
+    infected: list[tuple[str, int]] = []
+    md_files = 0
+    md_lines = 0
+    md_words = 0
+    skip = 0
+    n = 0
+    progressed = False
+    is_stderr_tty = sys.stderr.isatty()
+    total = sum(1 for f in files if f.kind in ("code", "markdown", "prose"))
+    for sf in files:
+        if sf.kind not in ("code", "markdown", "prose"):
+            continue
+        text = read_text(root / sf.path)
+        if text is None:
+            skip += 1
+            continue
+        n += 1
+        if is_stderr_tty and n % 200 == 0:
+            print(f"\rscanned {n}/{total} files...", end="", file=sys.stderr)
+            progressed = True
+        if sf.kind == "code":
+            evidences.extend(style_detector.detect(sf, text))
+        else:  # markdown | prose — объём доков (спека §5.1)
+            md_words += len(text.split())
+            if sf.kind == "markdown":
+                if sf.path.lower().endswith((".md", ".markdown")):
+                    md_files += 1
+                    md_lines += by_path[sf.path].lines
+                bloat = docs_bloat.detect(sf, text)  # детекция — только markdown (спека §4)
+                evidences.extend(bloat.evidences)
+                infected.extend(bloat.infected)
+        evidences.extend(phrase.detect(sf, text))
+        if pplx is not None and sf.kind in ("markdown", "prose"):
+            evidences.extend(pplx.detect(sf, text))
+    if progressed:
+        print(file=sys.stderr)  # завершаем строку прогресса
+    rb = repo_bloat_evidence(files, volume.sloc)
+    if rb:
+        evidences.append(rb)
+    evidences.extend(EnvMarkerDetector().detect(root, files, read_text))
+    history_commits: int | None = None
+    if opts.history:
+        from slopcount.detectors.git_history import GitUnavailable
+        from slopcount.detectors.git_history import detect as git_detect
+
+        try:
+            hist_evs, commits = git_detect(root, opts.history)
+            evidences.extend(hist_evs)
+            history_commits = commits
+        except GitUnavailable:
+            print(_("slopcount: git history unavailable; skipping archaeology"), file=sys.stderr)
+    volume.md_files = md_files
+    volume.md_lines = md_lines
+    volume.md_words = md_words
+    volume.md_sloc_ratio = safe_ratio(md_lines, volume.sloc)
+    volume.comment_sloc_ratio = safe_ratio(volume.comment_lines, volume.sloc)
+
+    # ── Фаза aggregate: SlopStats ───────────────────────────────────────
+    slop = aggregate_slop(evidences, sloc=volume.sloc, infected=infected)
+
+    # ── Фаза comprehension: SLOCOMO + разбивки стоимостей ───────────────
+    # (slocomo_compute импортирован наверху: цикла больше нет — Options в
+    # slocomo.py живёт только в TYPE_CHECKING)
+    slocomo = slocomo_compute(
+        md_words=md_words,
+        comment_lines=volume.comment_lines,
+        sloc=volume.sloc,
+        cognitive_total=volume.cognitive_total,
+        slop_ratio=slop.ratio,
+        locomo=manifest.locomo,
+        opts=opts,
+    )
+    report = Report(
+        root=str(root),
+        skip_count=skip,
+        history_commits=history_commits,
+        volume=volume,
+        slop=slop,
+        scc_version=manifest.scc_version,
+        cocomo=manifest.cocomo,
+        locomo=manifest.locomo,
+        slocomo=slocomo,
+    )
+    if manifest.cocomo is not None:
+        report.cocomo_breakdown = split_cocomo(
+            manifest.cocomo,
+            docs_lines=docs_bucket_lines,
+            code_lines=bucket_lines["code"],
+            data_lines=bucket_lines["data"],
+            personcost=opts.personcost,
+            overhead=opts.overhead,
+        )
+    if manifest.locomo is not None:
+        report.locomo_breakdown = attribute_locomo(
+            manifest.locomo,
+            docs_lines=docs_bucket_lines,
+            code_lines=bucket_lines["code"],
+            data_lines=bucket_lines["data"],
+        )
+    return report

slopcount/cli.py

@@ -0,0 +1,114 @@
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from typing import Literal
+
+import typer
+
+from slopcount import __version__, i18n
+from slopcount.app import Options, run
+from slopcount.i18n import _
+from slopcount.render.csv_out import render_csv
+from slopcount.render.json_out import render_json
+from slopcount.render.text import (
+    render_comprehension,
+    render_evidence,
+    render_slop,
+    render_volume,
+)
+
+app = typer.Typer()
+
+
+def _version(value: bool) -> None:
+    if value:
+        typer.echo(f"slopcount {__version__}")
+        raise typer.Exit()
+
+
+@app.command(context_settings={"help_option_names": ["-h", "--help"]})
+def scan(
+    paths: list[str] | None = typer.Argument(  # noqa: B008 — идиоматика typer
+        None, help="Directory to scan (default: .)"
+    ),
+    evidence: bool = typer.Option(
+        False, "--evidence", help="List every finding with its source line"
+    ),
+    json_out: bool = typer.Option(False, "--json", help="Print JSON report"),
+    csv_out: bool = typer.Option(False, "--csv", help="Print CSV report"),
+    history: int | None = typer.Option(None, "--history", help="Git history depth to scan"),
+    perplexity: bool = typer.Option(False, "--perplexity", help="Perplexity detector (extras)"),
+    rules: list[Path] | None = typer.Option(  # noqa: B008 — идиоматика typer
+        None, "--rules", help="Extra rules TOML (repeatable)"
+    ),
+    scc_path: str | None = typer.Option(
+        None, "--scc-path", help="Path to the scc binary (else PATH/env/download)"
+    ),
+    lang: Literal["en", "ru"] | None = typer.Option(None, "--lang", help="Interface language"),
+    personcost: float = typer.Option(4690.50, "--personcost", help="Monthly person cost, USD"),
+    overhead: float = typer.Option(
+        2.4, "--overhead", help="Cost overhead multiplier (COCOMO and SLOCOMO)"
+    ),
+    coffee_price: float = typer.Option(4.0, "--coffee-price", help="Coffee cup price, USD"),
+    no_therapy: bool = typer.Option(False, "--no-therapy", help="Skip the therapy estimate"),
+    wide: bool = typer.Option(False, "--wide", help="Wide table layout"),
+    version: bool | None = typer.Option(
+        None, "--version", callback=_version, is_eager=True, help="Show version and exit"
+    ),
+) -> None:
+    """Count the AI slop in a project and the cost of comprehending it."""
+    i18n.setup(lang)
+    opts = Options(
+        paths=paths or ["."],
+        evidence=evidence,
+        json_out=json_out,
+        csv_out=csv_out,
+        history=history,
+        perplexity=perplexity,
+        rules=rules or [],
+        scc_path=scc_path,
+        lang=lang,
+        personcost=personcost,
+        overhead=overhead,
+        coffee_price=coffee_price,
+        no_therapy=no_therapy,
+        wide=wide,
+    )
+    root = Path(opts.paths[0])
+    if not root.exists():
+        print(_("slopcount: path not found: {path}").format(path=root), file=sys.stderr)
+        raise typer.Exit(code=2)
+    if root.is_file():
+        print(
+            _("slopcount: path is a file, directory expected: {path}").format(path=root),
+            file=sys.stderr,
+        )
+        raise typer.Exit(code=2)
+    try:
+        report = run(opts)
+    except RuntimeError as e:  # напр. --perplexity без extras: подсказка, exit 2
+        print(e, file=sys.stderr)
+        raise typer.Exit(code=2) from e
+    if json_out:
+        print(render_json(report))
+    elif csv_out:
+        print(render_csv(report), end="")
+    else:
+        print(render_volume(report))
+        print(render_comprehension(report))
+        print(render_slop(report))
+        if evidence:
+            print(render_evidence(report))
+
+
+def main(argv: list[str] | None = None) -> int:
+    try:
+        app(argv)
+    except SystemExit as e:  # typer/click: --version (0) / usage error (2) / Exit(N)
+        return int(e.code or 0)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

slopcount/detectors/__init__.py

@@ -0,0 +1,4 @@
+import re
+
+# Общий эмодзи-класс для детекторов (docs_bloat: заголовки, code_style: комментарии)
+EMOJI_RE = re.compile(r"[\U0001F300-\U0001FAFF☀-⟿⬀-⯿]")

slopcount/detectors/code_style.py

@@ -0,0 +1,145 @@
+from __future__ import annotations
+
+import re
+from collections.abc import Callable
+
+from slopcount.detectors import EMOJI_RE
+from slopcount.evidence import Category, Evidence, ScannedFile
+from slopcount.extractors import extract_comments
+from slopcount.i18n import _, ngettext
+
+_TRIVIAL_DOC = re.compile(
+    r"^(Adds?|Returns?|Gets?|Sets?|Creates?|Initiali[sz]es?|Updates?|Checks?)\b", re.I
+)
+_CATCH_ALL = re.compile(
+    r"except\s+(Exception|BaseException)|catch\s*\(\s*(e|err|error|Exception)\b"
+)
+
+
+class CodeStyleDetector:
+    category = Category.STYLE
+
+    def detect(self, sf: ScannedFile, text: str) -> list[Evidence]:
+        checks: list[Callable[[ScannedFile, str], list[Evidence]]] = [
+            self._docstrings,
+            self._catch_all,
+            self._emoji_comments,
+            self._docstring_perfection,
+            self._monotone_comments,
+        ]
+        out: list[Evidence] = []
+        for check in checks:
+            out.extend(check(sf, text))
+        return out
+
+    def _docstrings(self, sf, text) -> list[Evidence]:
+        evs = []
+        blocks = [b for b in extract_comments(text, sf.language or "") if b.is_docstring]
+        code_lines = len([line for line in text.split("\n") if line.strip()])
+        for b in blocks:
+            n = len(b.lines)
+            first = b.lines[0] if b.lines else ""
+            if n <= 2 and _TRIVIAL_DOC.match(first):
+                evs.append(
+                    Evidence(
+                        sf.path,
+                        b.start_line,
+                        self.category,
+                        2,
+                        _("trivial docstring on obvious function"),
+                    )
+                )
+            if code_lines and n / code_lines > 0.5 and n >= 5:
+                evs.append(
+                    Evidence(
+                        sf.path,
+                        b.start_line,
+                        self.category,
+                        2,
+                        ngettext(
+                            "docstring longer than body (%d line)",
+                            "docstring longer than body (%d lines)",
+                            n,
+                        )
+                        % n,
+                    )
+                )
+        return evs
+
+    def _catch_all(self, sf, text) -> list[Evidence]:
+        evs = []
+        total = 0
+        for i, line in enumerate(text.split("\n"), 1):
+            if _CATCH_ALL.search(line):
+                total += 1
+                evs.append(
+                    Evidence(sf.path, i, self.category, 1, _("catch-all exception swallowing"))
+                )
+        if total >= 5:
+            evs.append(
+                Evidence(
+                    sf.path, 0, self.category, 2, _("defensive catch-all density (%d)") % total
+                )
+            )
+        return evs
+
+    def _emoji_comments(self, sf, text) -> list[Evidence]:
+        evs = []
+        for b in extract_comments(text, sf.language or ""):
+            for k, line in enumerate(b.lines):
+                if EMOJI_RE.search(line):
+                    evs.append(
+                        Evidence(
+                            sf.path, b.start_line + k, self.category, 2, _("emoji in code comment")
+                        )
+                    )
+        return evs
+
+    _GOOGLE = re.compile(r"\b(Args|Parameters|Returns|Raises)\s*:", re.I)
+    _DEF_LINE = re.compile(r"^\s*(?:async\s+)?def\s+\w+")
+
+    def _docstring_perfection(self, sf, text) -> list[Evidence]:
+        lines = text.split("\n")
+        defs = [i for i, line in enumerate(lines, 1) if self._DEF_LINE.match(line)]
+        if len(defs) < 5:
+            return []
+        doc_starts = {
+            b.start_line for b in extract_comments(text, sf.language or "") if b.is_docstring
+        }
+        perfect = sum(
+            1
+            for d in defs
+            if any(ds == d + 1 for ds in doc_starts)
+            and any(self._GOOGLE.search(line) for line in lines[d : d + 15])
+        )
+        if perfect / len(defs) >= 0.8:
+            return [
+                Evidence(
+                    sf.path,
+                    0,
+                    self.category,
+                    2,
+                    _("textbook-perfect docstrings on %d/%d functions") % (perfect, len(defs)),
+                )
+            ]
+        return []
+
+    def _monotone_comments(self, sf, text) -> list[Evidence]:
+        lens = [
+            len(line) for b in extract_comments(text, sf.language or "") for line in b.lines if line
+        ]
+        if len(lens) < 10:
+            return []
+        mean = sum(lens) / len(lens)
+        var = sum((x - mean) ** 2 for x in lens) / len(lens)
+        if var < 25:
+            return [
+                Evidence(
+                    sf.path,
+                    0,
+                    self.category,
+                    2,
+                    _("monotone comment length (var=%.1f) — machine cadence") % var,
+                )
+            ]
+        return []

slopcount/detectors/docs_bloat.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+from slopcount.detectors import EMOJI_RE
+from slopcount.evidence import Category, Evidence, ScannedFile
+from slopcount.i18n import _, ngettext
+
+_EMOJI_HEADER = re.compile(r"^#{1,6}\s.*" + EMOJI_RE.pattern)
+_GIANT_LINES = 500  # спека §4.2: «спеки-гиганты»
+
+
+@dataclass(frozen=True)
+class DocsBloatResult:
+    evidences: list[Evidence]
+    infected: list[tuple[str, int]]  # (путь, всего строк файла)
+
+
+class DocsBloatDetector:
+    category = Category.DOCS
+
+    def detect(self, sf: ScannedFile, text: str) -> DocsBloatResult:
+        lines = text.split("\n")
+        # физические строки: хвостовой "\n" даёт пустой последний элемент
+        total = len(lines) - (1 if lines and lines[-1] == "" else 0)
+        ev: list[Evidence] = []
+        weight = 0
+        if total > _GIANT_LINES:
+            ev.append(
+                Evidence(
+                    sf.path,
+                    0,
+                    self.category,
+                    5,
+                    ngettext("spec giant: %d line", "spec giant: %d lines", total) % total,
+                )
+            )
+            weight += 5
+        for i, line in enumerate(lines, 1):
+            if _EMOJI_HEADER.match(line):
+                ev.append(
+                    Evidence(sf.path, i, self.category, 2, _("emoji-decorated section header"))
+                )
+                weight += 2
+        infected: list[tuple[str, int]] = []
+        # гигант заражён целиком: «гигантскость» — утверждение обо всём файле,
+        # плотностной порог его никогда не достижим (вес маркера 5 фиксирован);
+        # для остальных файлов заражает плотность улико-баллов (спека §4.2)
+        if total > _GIANT_LINES or (total > 0 and weight / total > 0.1):
+            infected.append((sf.path, total))
+        return DocsBloatResult(ev, infected)
+
+
+def repo_bloat_evidence(files: list[ScannedFile], sloc: int) -> Evidence | None:
+    """Spec-to-Code Ratio из спеки §4.2: >100 КБ markdown на KLOC — тревога."""
+    docs_bytes = sum(f.size for f in files if f.kind == "markdown")
+    if sloc == 0:
+        # docs-only repo: ratio undefined, signal comes from infection instead
+        return None
+    kb_per_kloc = docs_bytes / 1024 / (sloc / 1000)
+    if kb_per_kloc > 100:
+        return Evidence(
+            "<repo>",
+            0,
+            Category.DOCS,
+            4,
+            _("docs bloat: %.0f KB of markdown per KLOC") % kb_per_kloc,
+        )
+    return None

slopcount/detectors/env_markers.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+import fnmatch
+import re
+from collections.abc import Callable
+from dataclasses import dataclass
+from importlib import resources
+from pathlib import Path
+
+from slopcount.evidence import Category, Evidence, ScannedFile
+from slopcount.i18n import _
+
+
+@dataclass(frozen=True)
+class _Header:
+    pattern: re.Pattern
+    weight: int
+    description: str
+
+
+def _load():
+    """Внутренний каталог; битый TOML падает сразу (fail fast, ассет в пакете)."""
+    import tomllib
+
+    base = resources.files("slopcount").joinpath("rules/env_markers.toml")
+    data = tomllib.loads(Path(str(base)).read_text(encoding="utf-8"))
+    paths = [(m["path"], m["weight"], _(m["description"])) for m in data.get("marker", [])]
+    headers = [
+        _Header(re.compile(h["pattern"]), h["weight"], _(h["description"]))
+        for h in data.get("header", [])
+    ]
+    return paths, headers
+
+
+class EnvMarkerDetector:
+    category = Category.AGENCY
+
+    def __init__(self):
+        self.paths, self.headers = _load()
+
+    def detect(
+        self, root: Path, scanned: list[ScannedFile], reader: Callable[[Path], str | None]
+    ) -> list[Evidence]:
+        evs: list[Evidence] = []
+        names = [sf.path for sf in scanned] + _all_entries(root)
+        for pat, weight, desc in self.paths:
+            for name in names:
+                if fnmatch.fnmatch(name, pat) or fnmatch.fnmatch(Path(name).name, pat):
+                    evs.append(Evidence(name, 0, self.category, weight, desc))
+                    break
+        for sf in scanned:
+            # data-файлы не читаем вовсе — ни байта контента до детекторов
+            # (матрица §5.3 спеки scc-delegation)
+            if sf.kind == "data":
+                continue
+            text = reader(root / sf.path)
+            if not text:
+                continue
+            for line in text.split("\n")[:5]:
+                for h in self.headers:
+                    if h.pattern.search(line):
+                        evs.append(Evidence(sf.path, 1, self.category, h.weight, h.description))
+        return evs
+
+
+def _all_entries(root: Path) -> list[str]:
+    out = []
+    for p in root.iterdir():
+        out.append(p.name)
+    return out