codeabc 0.1.0__py3-none-any.whl

backend/services/duplicate_code.py

@@ -0,0 +1,165 @@
+"""Flag duplicated code blocks — the same code copied into more than one place.
+
+Copy-paste is the quietest way a codebase rots. The first copy is fine; the
+problem shows up three months later, when someone fixes a bug in one copy and
+the others keep the old behavior. Nothing warns about that — the copies don't
+know about each other, and they drift apart one silent edit at a time. For a
+beginner this is also the easiest maintenance trap to understand: "the same
+thing written twice means every change has to happen twice."
+
+None of the other readability analyzers catch this: a copied block can be
+short, shallow, low-complexity, and have few parameters. Duplication is its own
+axis.
+
+The detection is deliberately simple and honest: normalize each file's lines
+(drop blanks, comments, and imports, collapse whitespace), hash every window
+of a few consecutive lines, and report windows that show up in more than one
+file, or three or more times in one file. Exact-duplicate only — fuzzy
+"similar" matching belongs to a much bigger tool, and guessing wrong there
+costs more trust than it earns.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import re
+
+# Consecutive lines a window must span to count as a block.
+_WINDOW = 6
+# Same-file copies needed to flag (cross-file only needs two).
+_SAME_FILE_MIN = 3
+
+_BLANK_RE = re.compile(r"^\s*$")
+_COMMENT_RE = re.compile(r"^\s*#")
+_IMPORT_RE = re.compile(r"^\s*(import\s|from\s)")
+_WS_RE = re.compile(r"\s+")
+
+
+def _normalized_lines(content: str) -> list[tuple[int, str]]:
+    """Return (line number, normalized text) pairs, skipping noise lines."""
+    out: list[tuple[int, str]] = []
+    for lineno, raw in enumerate(content.splitlines(), 1):
+        if _BLANK_RE.match(raw) or _COMMENT_RE.match(raw) or _IMPORT_RE.match(raw):
+            continue
+        out.append((lineno, _WS_RE.sub(" ", raw.strip())))
+    return out
+
+
+def _is_test(path: str) -> bool:
+    base = path.rsplit("/", 1)[-1]
+    return base.startswith("test_") or base.endswith("_test.py") or "/tests/" in path
+
+
+def _merge_overlapping(clusters: list[list[dict]], window: int) -> list[list[dict]]:
+    """Union clusters whose occurrences sit in the same file within ``window``
+    lines — those are sliding windows of one physical copy, not two copies."""
+    parent = list(range(len(clusters)))
+
+    def find(x: int) -> int:
+        while parent[x] != x:
+            parent[x] = parent[parent[x]]
+            x = parent[x]
+        return x
+
+    def union(a: int, b: int) -> None:
+        ra, rb = find(a), find(b)
+        if ra != rb:
+            parent[rb] = ra
+
+    for i in range(len(clusters)):
+        for j in range(i + 1, len(clusters)):
+            if any(
+                oa["path"] == ob["path"] and abs(oa["line"] - ob["line"]) < window
+                for oa in clusters[i]
+                for ob in clusters[j]
+            ):
+                union(i, j)
+
+    merged: dict[int, list[dict]] = {}
+    for idx, occurrences in enumerate(clusters):
+        merged.setdefault(find(idx), [])
+        merged[find(idx)].extend(occurrences)
+    return list(merged.values())
+
+
+def _collapse_window_ghosts(occurrences: list[dict], window: int) -> list[dict]:
+    """Keep one occurrence per physical copy: within a file, windows starting
+    less than ``window`` lines apart are the same copy seen twice."""
+    kept: list[dict] = []
+    for o in sorted(occurrences, key=lambda o: (o["path"], o["line"])):
+        if kept and kept[-1]["path"] == o["path"] and o["line"] - kept[-1]["line"] < window:
+            continue
+        kept.append(o)
+    return kept
+
+
+def scan_duplicate_code(
+    file_contents: dict[str, str], *, window: int = _WINDOW, limit: int = 10
+) -> dict:
+    """Find line windows that appear in more than one place.
+
+    Args:
+        file_contents: mapping of path to file text (as CodeABC already read it).
+        window: consecutive normalized lines that make up one block.
+        limit: how many clusters to return.
+
+    Returns ``{"total", "total_occurrences", "window", "clusters"}`` where each
+    cluster is a list of ``{"path", "line"}`` occurrences (one per physical
+    copy, sorted). A block counts when it appears in two different files, or
+    at least three times in one file. Test files and non-Python files are
+    skipped.
+    """
+    blocks: dict[str, list[dict]] = {}
+    for path, content in file_contents.items():
+        if not content or not path.endswith(".py") or _is_test(path):
+            continue
+        lines = _normalized_lines(content)
+        for i in range(len(lines) - window + 1):
+            window_text = "\n".join(text for _, text in lines[i : i + window])
+            digest = hashlib.sha1(window_text.encode()).hexdigest()[:16]
+            occurrence = {"path": path, "line": lines[i][0]}
+            bucket = blocks.setdefault(digest, [])
+            if occurrence not in bucket:
+                bucket.append(occurrence)
+
+    clusters = []
+    # singleton windows can never be duplicates; only multi-occurrence digests
+    # are candidates, and spatial merging happens within that set so unique
+    # neighboring windows don't get absorbed into a real cluster
+    candidates = [occurrences for occurrences in blocks.values() if len(occurrences) >= 2]
+    for occurrences in _merge_overlapping(candidates, window):
+        copies = _collapse_window_ghosts(occurrences, window)
+        paths = {o["path"] for o in copies}
+        if len(paths) >= 2 or len(copies) >= _SAME_FILE_MIN:
+            clusters.append(sorted(copies, key=lambda o: (o["path"], o["line"])))
+    clusters.sort(key=lambda c: (-len(c), c[0]["path"], c[0]["line"]))
+
+    return {
+        "total": len(clusters),
+        "total_occurrences": sum(len(c) for c in clusters),
+        "window": window,
+        "clusters": clusters[:limit],
+    }
+
+
+def render_duplicate_code_markdown(project_name: str, data: dict | None) -> str:
+    """Render the duplicated-code section, or an empty string when none is found."""
+    if not data or not data.get("clusters"):
+        return ""
+
+    window = data.get("window", _WINDOW)
+    lines = [
+        f"## 重复的代码（{project_name}）",
+        "",
+        "下面这些代码块在不止一个地方出现，长得几乎一模一样（连续 "
+        f"{window} 行以上）。复制的时候省事，麻烦在后面：哪天改其中一处，"
+        "其余几处不会跟着变，bug 就这样一个版本一个版本地留下来。建议把它们"
+        "抽成一个公共函数，以后只改一处：",
+        "",
+        "| 出现位置 |",
+        "| --- |",
+    ]
+    for cluster in data["clusters"]:
+        spots = "<br>".join(f"`{o['path']}` 第 {o['line']} 行" for o in cluster)
+        lines.append(f"| {spots} |")
+    return "\n".join(lines)

backend/services/entrypoints.py

@@ -0,0 +1,152 @@
+"""Entry points: where a project actually starts running.
+
+The reading map tells a newcomer what to read first to *understand* the code;
+this answers the other half of the question — "how do I run it, and where does
+execution begin?". It collects the runnable "front doors" from the source
+CodeABC already loaded: files with a ``__main__`` guard, the console commands
+declared in ``pyproject.toml`` / ``setup.cfg``, the ``bin`` of a ``package.json``,
+and a few conventional entry filenames — so a reader doesn't have to guess where
+to start the program.
+
+:func:`find_entry_points` is pure over the file contents, so it is unit-testable
+with plain strings and needs no repository.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+
+# `if __name__ == "__main__":` — running the file executes this block.
+_MAIN_GUARD_RE = re.compile(r"""^\s*if\s+__name__\s*==\s*['"]__main__['"]\s*:""", re.M)
+# A console-script entry inside a [project.scripts]-style table: name = "target".
+_SCRIPT_ENTRY_RE = re.compile(r"""^\s*([\w][\w\-.]*)\s*=\s*['"]([^'"]+)['"]""", re.M)
+
+# Conventional entry filenames and what running them usually means. Only used as
+# a fallback hint when nothing more explicit points at the file.
+_CONVENTIONAL = {
+    "__main__.py": "包入口，可用 `python -m <package>` 运行",
+    "main.py": "常见的程序主入口",
+    "manage.py": "Django 管理命令入口（`python manage.py ...`）",
+    "wsgi.py": "WSGI 服务器入口（gunicorn/uwsgi 从这里加载应用）",
+    "asgi.py": "ASGI 服务器入口（uvicorn 从这里加载应用）",
+    "cli.py": "命令行入口",
+    "server.js": "Node 服务器入口",
+    "index.js": "Node 程序入口",
+}
+
+# Priority for ranking: an explicitly declared command beats a runnable script,
+# which beats a by-convention guess.
+_KIND_RANK = {"command": 0, "script": 1, "convention": 2}
+
+_SCRIPT_REASON = '直接运行会执行文件里的 `if __name__ == "__main__"` 代码'
+
+
+def _section(content: str, header: str) -> str:
+    """Return the text of a TOML/INI ``[header]`` section (until the next header)."""
+    start = content.find(f"[{header}]")
+    if start == -1:
+        return ""
+    rest = content[start + len(header) + 2 :]
+    end = re.search(r"^\s*\[", rest, re.M)
+    return rest[: end.start()] if end else rest
+
+
+def _declared_commands(path: str, content: str) -> list[tuple[str, str]]:
+    """Console commands declared in pyproject.toml / setup.cfg / package.json."""
+    name = path.replace("\\", "/").rsplit("/", 1)[-1]
+    commands: list[tuple[str, str]] = []
+
+    if name == "pyproject.toml":
+        for header in ("project.scripts", "project.gui-scripts", "tool.poetry.scripts"):
+            for cmd, target in _SCRIPT_ENTRY_RE.findall(_section(content, header)):
+                commands.append((cmd, target))
+
+    elif name == "setup.cfg":
+        # console_scripts live under [options.entry_points] as "cmd = module:fn".
+        block = _section(content, "options.entry_points")
+        if "console_scripts" in block:
+            for cmd, target in _SCRIPT_ENTRY_RE.findall(block):
+                if cmd != "console_scripts":
+                    commands.append((cmd, target))
+
+    elif name == "package.json":
+        try:
+            data = json.loads(content)
+        except (json.JSONDecodeError, ValueError):
+            return commands
+        bins = data.get("bin")
+        if isinstance(bins, str):
+            commands.append((str(data.get("name") or "bin"), bins))
+        elif isinstance(bins, dict):
+            commands.extend((str(k), str(v)) for k, v in bins.items())
+
+    return commands
+
+
+def find_entry_points(file_contents: dict[str, str], *, limit: int = 40) -> dict:
+    """Collect the project's runnable entry points from its source.
+
+    Args:
+        file_contents: mapping of path to file text (as CodeABC already read it).
+        limit: how many entry points to return in the ranked list.
+
+    Returns ``{"total", "entry_points"}`` where each entry point is
+    ``{"path", "kind", "command", "reason"}`` and ``kind`` is one of
+    ``command`` (a declared console command), ``script`` (a file with a
+    ``__main__`` guard) or ``convention`` (a conventional entry filename).
+    """
+    # path -> best entry (keep the highest-priority kind seen for a path/command).
+    found: dict[tuple[str, str], dict] = {}
+
+    def _add(path: str, kind: str, command: str, reason: str) -> None:
+        key = (path, command)
+        existing = found.get(key)
+        if existing is None or _KIND_RANK[kind] < _KIND_RANK[existing["kind"]]:
+            found[key] = {"path": path, "kind": kind, "command": command, "reason": reason}
+
+    for path, content in file_contents.items():
+        if not content:
+            continue
+        name = path.replace("\\", "/").rsplit("/", 1)[-1]
+
+        for cmd, target in _declared_commands(path, content):
+            _add(path, "command", cmd, f"命令行命令 `{cmd}`（运行 {target}）")
+
+        if name.endswith(".py") and _MAIN_GUARD_RE.search(content):
+            _add(path, "script", f"python {path}", _SCRIPT_REASON)
+        elif name in _CONVENTIONAL:
+            _add(path, "convention", path, _CONVENTIONAL[name])
+
+    entries = sorted(found.values(), key=lambda e: (_KIND_RANK[e["kind"]], e["path"]))
+    return {"total": len(entries), "entry_points": entries[:limit]}
+
+
+def render_entrypoints_markdown(project_name: str, data: dict | None) -> str:
+    """Render the entry points as Markdown, or ``""`` if none were found."""
+    entries = (data or {}).get("entry_points") or []
+    if not entries:
+        return ""
+
+    labels = {
+        "command": "命令行命令",
+        "script": "可直接运行的脚本",
+        "convention": "按惯例的入口文件",
+    }
+    lines = [
+        f"# {project_name} — 怎么跑起来（入口点）",
+        "",
+        "> 程序从这些“门”开始运行。命令行命令是作者明确声明的，脚本是能直接跑的文件，"
+        "其余是按文件名惯例推断的入口。",
+        "",
+    ]
+    for kind in ("command", "script", "convention"):
+        group = [e for e in entries if e["kind"] == kind]
+        if not group:
+            continue
+        lines.append(f"## {labels[kind]}")
+        lines.append("")
+        lines.extend(f"- `{e['command']}` — {e['reason']}" for e in group)
+        lines.append("")
+
+    return "\n".join(lines).rstrip() + "\n"

backend/services/envscan.py

@@ -0,0 +1,167 @@
+"""Environment-variable surface: what a project needs configured to run.
+
+One of the first questions a newcomer has is "what do I have to set up before
+this will run?". Most of that answer lives in the environment variables the code
+reads. This collects them from the source CodeABC already loaded — which names
+are read, whether they are required (the code crashes if they are missing) or
+optional (a default is supplied) — so a reader gets a setup checklist without
+hunting through the codebase.
+
+:func:`scan_env_vars` is pure: it matches the common read patterns over the file
+contents, so it is unit-testable with plain strings and needs no repository:
+
+- stdlib ``os.environ["X"]`` (required — raises when unset), ``os.getenv("X")`` /
+  ``os.environ.get("X")`` (optional — a default may follow),
+- JS ``process.env.X`` (optional),
+- Go ``os.Getenv("X")`` / ``os.LookupEnv("X")`` (optional — return "" / a found
+  flag rather than panicking) and Rust ``env::var("X")`` (required when
+  ``.unwrap()`` / ``.expect(...)`` panics on an unset value, optional otherwise),
+- the env-loader libraries many Django/Flask projects use instead of the
+  stdlib: ``python-decouple`` (``config("X")``) and ``environs`` /
+  ``django-environ`` (``env("X")`` / ``env.str("X")`` / ``env.int(...)`` …).
+
+A name is *required* if it is ever read without a default (``os.environ["X"]``, or
+a bare ``config("X")`` / ``env("X")`` — both raise when the variable is unset) and
+*optional* if every read supplies a fallback.
+"""
+
+from __future__ import annotations
+
+import re
+
+# Required reads: os.environ["X"] / os.environ['X'] — a missing key raises.
+_REQUIRED_RE = re.compile(r"""os\.environ\[\s*['"](\w+)['"]\s*\]""")
+# Optional reads: os.getenv("X"...) / os.environ.get("X"...) — a default may follow.
+# os.environ.setdefault("X", ...) always supplies a fallback, so it is optional too.
+_OPTIONAL_RE = re.compile(
+    r"""os\.(?:getenv|environ\.get|environ\.setdefault)\(\s*['"](\w+)['"]\s*(,)?"""
+)
+# JS: process.env.X / process.env["X"] — always optional (undefined if unset).
+_JS_RE = re.compile(r"""process\.env(?:\.(\w+)|\[\s*['"](\w+)['"]\s*\])""")
+# python-decouple: config("X") / config("X", default=...). A bare read raises
+# UndefinedValueError when the var is unset, so config("X") with no default is
+# required; a default arg (trailing comma) makes it optional. The lookbehind
+# avoids matching attribute calls like ``app.config(...)``.
+_DECOUPLE_RE = re.compile(r"""(?<![\w.])config\(\s*['"](\w+)['"]\s*(,)?""")
+# environs / django-environ: env("X"), env.str("X"), env.int(...), env.bool(...),
+# etc. Same rule as decouple — a bare read raises when the var is unset, a
+# default arg makes it optional. ``os.getenv`` / ``os.environ`` never match here
+# because the char before this ``env`` is a word char / dot.
+_ENVIRONS_RE = re.compile(
+    r"""(?<![\w.])env(?:\.(?:str|int|bool|float|list|json|url|path|dict|log_level))?"""
+    r"""\(\s*['"](\w+)['"]\s*(,)?"""
+)
+# Go: os.Getenv("X") returns "" when the variable is unset and os.LookupEnv("X")
+# returns (value, ok); both are graceful, so the variable is optional.
+_GO_RE = re.compile(r"""os\.(?:Getenv|LookupEnv)\(\s*['"](\w+)['"]""")
+# Rust: env::var("X") / std::env::var("X") (and the *_os variants) return a
+# ``Result``. A trailing ``.unwrap()`` / ``.expect(...)`` panics when the variable
+# is unset (required); ``.unwrap_or(...)`` or any other handling is graceful
+# (optional).
+_RUST_RE = re.compile(
+    r"""(?:std::)?env::var(?:_os)?\(\s*['"](\w+)['"]\s*\)\s*(\.(?:unwrap|expect)\s*\()?"""
+)
+
+
+def scan_env_vars(file_contents: dict[str, str], *, limit: int = 40) -> dict:
+    """Collect environment variables read across the project's source.
+
+    Args:
+        file_contents: mapping of path to file text (as CodeABC already read it).
+        limit: how many variables to return in the ranked list.
+
+    Returns ``{"total", "required", "vars"}`` where ``vars`` is every variable
+    name with its read count and whether it is required (read with no default
+    anywhere), and ``required`` is just the names of the required ones.
+    """
+    # name -> {"count": int, "required": bool, "first": (path, line)}
+    found: dict[str, dict] = {}
+
+    def _record(name: str, required: bool, path: str, line_no: int) -> None:
+        entry = found.get(name)
+        if entry is None:
+            found[name] = {"count": 1, "required": required, "first": (path, line_no)}
+            return
+        entry["count"] += 1
+        # Required wins: if any read is ``os.environ["X"]`` (raises when unset),
+        # the variable must be set regardless of graceful reads elsewhere.
+        entry["required"] = entry["required"] or required
+
+    for path, content in file_contents.items():
+        if not content:
+            continue
+        for line_no, line in enumerate(content.splitlines(), start=1):
+            for m in _REQUIRED_RE.finditer(line):
+                _record(m.group(1), True, path, line_no)
+            for m in _OPTIONAL_RE.finditer(line):
+                _record(m.group(1), False, path, line_no)
+            for m in _JS_RE.finditer(line):
+                _record(m.group(1) or m.group(2), False, path, line_no)
+            # decouple/environs: required unless a default arg (trailing comma) follows.
+            for m in _DECOUPLE_RE.finditer(line):
+                _record(m.group(1), m.group(2) is None, path, line_no)
+            for m in _ENVIRONS_RE.finditer(line):
+                _record(m.group(1), m.group(2) is None, path, line_no)
+            for m in _GO_RE.finditer(line):
+                _record(m.group(1), False, path, line_no)
+            # Rust: required when the read is unwrap()/expect()-ed (panics if unset).
+            for m in _RUST_RE.finditer(line):
+                _record(m.group(1), m.group(2) is not None, path, line_no)
+
+    vars_list = [
+        {
+            "name": name,
+            "count": info["count"],
+            "required": info["required"],
+            "path": info["first"][0],
+            "line": info["first"][1],
+        }
+        for name, info in found.items()
+    ]
+    # Required first, then by how widely the name is read, then alphabetical.
+    vars_list.sort(key=lambda v: (not v["required"], -v["count"], v["name"]))
+
+    required = [v["name"] for v in vars_list if v["required"]]
+
+    return {
+        "total": len(vars_list),
+        "required": required,
+        "vars": vars_list[:limit],
+    }
+
+
+def render_env_markdown(project_name: str, env_data: dict | None) -> str:
+    """Render the environment-variable surface as Markdown, or ``""`` if none."""
+    data = env_data or {}
+    vars_list = data.get("vars") or []
+    if not vars_list:
+        return ""
+
+    total = data.get("total", 0)
+    required = data.get("required") or []
+    lines = [
+        f"# {project_name} — 环境变量",
+        "",
+        f"> 代码里读取的环境变量共 {total} 个，是“想跑起来要先配什么”的清单。"
+        "标 **必填** 的没有默认值，缺了会直接报错。",
+        "",
+    ]
+
+    if required:
+        lines.append(f"## 必填（缺了会报错）：{len(required)} 个")
+        lines.append("")
+        lines.extend(
+            f"- `{v['name']}` — 在 `{v['path']}` 等 {v['count']} 处读取，必须配置。"
+            for v in vars_list
+            if v["required"]
+        )
+        lines.append("")
+
+    optional = [v for v in vars_list if not v["required"]]
+    if optional:
+        lines.append("## 可选（有默认值）")
+        lines.append("")
+        lines.extend(f"- `{v['name']}` — {v['count']} 处读取，不配会用默认值。" for v in optional)
+        lines.append("")
+
+    return "\n".join(lines).rstrip() + "\n"