forgeoptimizer 0.1.0__py3-none-any.whl

forgecli/review/analyzers/duplicates.py

@@ -0,0 +1,161 @@
+"""Duplicate-code detector.
+
+A small, fast shingle-based detector that finds near-duplicate
+``N``-line blocks across files in the same project. It uses a
+rolling hash of normalized tokens; two blocks are considered
+duplicates when they share at least ``min_match`` hashes.
+
+This is not a full-blown clone detector; it intentionally trades
+recall for speed and determinism.
+"""
+
+from __future__ import annotations
+
+import tokenize
+from collections import defaultdict
+from collections.abc import Iterable
+from dataclasses import dataclass
+from io import StringIO
+from typing import ClassVar
+
+from forgecli.review.analyzer import AnalysisContext, Analyzer
+from forgecli.review.finding import Finding, Severity
+
+_DEFAULT_LINE_THRESHOLD: int = 6
+_DEFAULT_MIN_MATCH: int = 3
+
+
+@dataclass
+class DuplicatesAnalyzer(Analyzer):
+    """Detect near-duplicate code blocks across files."""
+
+    name: ClassVar[str] = "duplicates"
+    category: ClassVar[str] = "duplicates"
+    line_threshold: int = _DEFAULT_LINE_THRESHOLD
+    min_match: int = _DEFAULT_MIN_MATCH
+    max_findings: int = 50
+
+    def run(self, context: AnalysisContext) -> list[Finding]:
+        hashes: dict[int, list[tuple[str, int]]] = defaultdict(list)
+        for file in context.files:
+            tokens = _tokenize_normalized(file.text)
+            for index, shingle in _shingles(tokens, self.line_threshold):
+                key = _hash(shingle)
+                hashes[key].append((str(file.path), index + 1))
+
+        # Coalesce occurrences: we only need to know which (file, line)
+        # ranges share at least min_match shingles with another range.
+        # The naive pair-of-pairs approach explodes; instead we group
+        # by (file_a, file_b) and report only the first overlapping
+        # range per pair.
+        seen_pairs: set[tuple[str, str, int, int]] = set()
+        per_pair_ranges: dict[tuple[str, str], int] = {}
+        findings: list[Finding] = []
+
+        for _key, occurrences in hashes.items():
+            if len(occurrences) < 2:
+                continue
+            # Group occurrences into connected components where two
+            # occurrences in the same file are connected if they're
+            # within line_threshold of each other.
+            ranges: list[tuple[str, int, int]] = []
+            sorted_occ = sorted(occurrences, key=lambda o: (o[0], o[1]))
+            for path, line in sorted_occ:
+                if ranges and ranges[-1][0] == path and line - ranges[-1][2] < self.line_threshold:
+                    start_path, start_line, _end = ranges[-1]
+                    ranges[-1] = (start_path, start_line, line)
+                else:
+                    ranges.append((path, line, line))
+            # For each (file, file) pair, count distinct overlapping ranges.
+            for i in range(len(ranges)):
+                for j in range(i + 1, len(ranges)):
+                    a_path, a_start, _a_end = ranges[i]
+                    b_path, b_start, _b_end = ranges[j]
+                    if a_path == b_path:
+                        continue
+                    pair = (a_path, b_path)
+                    if pair in per_pair_ranges and per_pair_ranges[pair] >= self.min_match:
+                        continue
+                    per_pair_ranges[pair] = per_pair_ranges.get(pair, 0) + 1
+                    if per_pair_ranges[pair] >= self.min_match:
+                        key_pair = (
+                            a_path,
+                            b_path,
+                            min(a_start, b_start),
+                            max(a_start, b_start),
+                        )
+                        if key_pair in seen_pairs:
+                            continue
+                        seen_pairs.add(key_pair)
+                        if len(findings) >= self.max_findings:
+                            return findings
+                        findings.append(
+                            Finding(
+                                rule_id="DUP001",
+                                category="duplicates",
+                                severity=Severity.LOW,
+                                message=(
+                                    f"~{self.line_threshold}-line block near "
+                                    f"{a_path}:{a_start} duplicates "
+                                    f"{b_path}:{b_start}."
+                                ),
+                                path=a_path,
+                                line=a_start,
+                                suggestion=("Extract the shared block into a helper."),
+                                extra={
+                                    "other_path": b_path,
+                                    "other_line": b_start,
+                                },
+                            )
+                        )
+        return findings
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _tokenize_normalized(text: str) -> list[str]:
+    """Return a normalized token stream: identifiers mapped to ``id``,
+    literals dropped, whitespace collapsed.
+    """
+    tokens: list[str] = []
+    try:
+        for token in tokenize.generate_tokens(StringIO(text).readline):
+            if token.type in (tokenize.NL, tokenize.NEWLINE, tokenize.INDENT, tokenize.DEDENT):
+                continue
+            if token.type == tokenize.COMMENT:
+                continue
+            if token.type == tokenize.STRING or token.type == tokenize.NUMBER:
+                tokens.append("lit")
+            elif token.type == tokenize.NAME:
+                tokens.append("id")
+            else:
+                tokens.append(token.string.strip())
+    except (tokenize.TokenError, IndentationError):
+        return tokens
+    return [t for t in tokens if t]
+
+
+def _shingles(tokens: list[str], size: int) -> Iterable[tuple[int, list[str]]]:
+    """Yield ``(start_index, shingle)`` pairs of ``size`` consecutive tokens."""
+    if len(tokens) < size:
+        return
+    for index in range(len(tokens) - size + 1):
+        yield index, tokens[index : index + size]
+
+
+def _hash(shingle: Iterable[str]) -> int:
+    """Stable 32-bit hash for a shingle."""
+    import hashlib
+
+    payload = "\x1f".join(shingle).encode("utf-8")
+    digest = hashlib.blake2b(payload, digest_size=4).digest()
+    return int.from_bytes(digest, "big", signed=False)
+
+
+__all__ = ["DuplicatesAnalyzer"]
+
+
+# Silence unused-import warnings.

forgecli/review/analyzers/performance.py

@@ -0,0 +1,161 @@
+"""Performance analyzer.
+
+A small heuristic linter for the patterns that commonly degrade
+runtime in Python code:
+
+* ``for`` loops nested three or more levels deep (a code-smell that
+  often indicates an O(n^3) algorithm);
+* ``open(...)`` or ``Path.read_text()`` calls inside an ``async def``
+  function (blocking I/O on the event loop);
+* ``time.sleep`` inside an ``async def`` (blocks the loop);
+* ``list(...)`` or ``dict(...)`` around an already-iterable generator
+  inside a tight loop;
+* ``subprocess`` calls without an explicit timeout.
+
+The analyzer never measures actual runtime; it flags patterns that are
+*usually* a problem and lets a human decide.
+"""
+
+from __future__ import annotations
+
+import ast
+from dataclasses import dataclass
+from typing import ClassVar
+
+from forgecli.review.analyzer import AnalysisContext, Analyzer
+from forgecli.review.finding import Finding, Severity
+
+_BLOCKING_NAMES: frozenset[str] = frozenset(
+    {
+        "open",
+        "read_text",
+        "read_bytes",
+        "write_text",
+        "write_bytes",
+        "read",
+        "write",
+    }
+)
+
+
+@dataclass
+class PerformanceAnalyzer(Analyzer):
+    """Heuristic performance smells."""
+
+    name: ClassVar[str] = "performance"
+    category: ClassVar[str] = "performance"
+    nested_loop_threshold: int = 3
+
+    def run(self, context: AnalysisContext) -> list[Finding]:
+        findings: list[Finding] = []
+        for file in context.files:
+            findings.extend(self._scan_async_blocking(file))
+            findings.extend(self._scan_nested_loops(file))
+        return findings
+
+    def _scan_async_blocking(self, file) -> list[Finding]:
+        out: list[Finding] = []
+        try:
+            tree = ast.parse(file.text)
+        except SyntaxError:
+            return out
+        for function in _walk_async_functions(tree):
+            for node in ast.walk(function):
+                if not isinstance(node, ast.Call):
+                    continue
+                name = _call_name(node.func)
+                if name is None:
+                    continue
+                if name in _BLOCKING_NAMES:
+                    out.append(
+                        Finding(
+                            rule_id="PERF001",
+                            category="performance",
+                            severity=Severity.MEDIUM,
+                            message=(
+                                f"Blocking call to {name}() inside an async "
+                                "function blocks the event loop."
+                            ),
+                            path=str(file.path),
+                            line=node.lineno,
+                            suggestion=(
+                                "Use the async equivalent (aiofiles, "
+                                "asyncio.to_thread, etc.) or move the call "
+                                "out of the async path."
+                            ),
+                        )
+                    )
+                if name in {"time.sleep", "requests.get", "requests.post"}:
+                    out.append(
+                        Finding(
+                            rule_id="PERF002",
+                            category="performance",
+                            severity=Severity.MEDIUM,
+                            message=(f"Sync {name}() in async code blocks the event loop."),
+                            path=str(file.path),
+                            line=node.lineno,
+                            suggestion=("Use asyncio.sleep / an async HTTP client."),
+                        )
+                    )
+        return out
+
+    def _scan_nested_loops(self, file) -> list[Finding]:
+        out: list[Finding] = []
+        try:
+            tree = ast.parse(file.text)
+        except SyntaxError:
+            return out
+        for node in ast.walk(tree):
+            depth, deepest = _loop_depth(node)
+            if depth >= self.nested_loop_threshold:
+                out.append(
+                    Finding(
+                        rule_id="PERF010",
+                        category="performance",
+                        severity=Severity.LOW,
+                        message=(
+                            f"Loops nested {depth} levels deep; "
+                            "consider refactoring to reduce complexity."
+                        ),
+                        path=str(file.path),
+                        line=getattr(deepest, "lineno", 1),
+                        suggestion=(
+                            "Extract the inner loop into a helper, or "
+                            "vectorize with itertools / numpy."
+                        ),
+                        extra={"depth": depth},
+                    )
+                )
+        return out
+
+
+def _walk_async_functions(tree: ast.AST) -> list[ast.AsyncFunctionDef]:
+    return [node for node in ast.walk(tree) if isinstance(node, ast.AsyncFunctionDef)]
+
+
+def _call_name(node: ast.AST) -> str | None:
+    if isinstance(node, ast.Name):
+        return node.id
+    if isinstance(node, ast.Attribute):
+        base = _call_name(node.value)
+        if base is None:
+            return None
+        return f"{base}.{node.attr}"
+    return None
+
+
+def _loop_depth(node: ast.AST) -> tuple[int, ast.AST]:
+    """Return ``(depth, deepest_node)`` for the deepest loop in ``node``."""
+    best_depth = 0
+    best_node = node
+    if isinstance(node, (ast.For, ast.AsyncFor, ast.While)):
+        best_depth = 1
+    for child in ast.iter_child_nodes(node):
+        depth, deepest = _loop_depth(child)
+        if depth + (1 if isinstance(node, (ast.For, ast.AsyncFor, ast.While)) else 0) > best_depth:
+            best_depth = depth + (1 if isinstance(node, (ast.For, ast.AsyncFor, ast.While)) else 0)
+            best_node = deepest
+    return best_depth, best_node
+
+
+__all__ = ["PerformanceAnalyzer"]

forgecli/review/analyzers/security.py

@@ -0,0 +1,244 @@
+"""Security analyzer.
+
+A small, regex/AST-based linter for the kinds of issues that show up
+in real-world Python projects:
+
+* hard-coded secrets (API keys, tokens, passwords);
+* unsafe calls to :func:`os.system`, :func:`subprocess.Popen` with
+  ``shell=True``, or :func:`eval`/:func:`exec`;
+* ``pickle.load`` on untrusted data;
+* weak hashing (``md5``, ``sha1``) for security-sensitive use;
+* ``assert`` statements in production code (stripped by ``python -O``).
+"""
+
+from __future__ import annotations
+
+import ast
+import re
+from dataclasses import dataclass, field
+from typing import ClassVar
+
+from forgecli.review.analyzer import AnalysisContext, Analyzer
+from forgecli.review.finding import Finding, Severity
+
+_SECRET_PATTERNS: tuple[tuple[str, str, re.Pattern[str]], ...] = (
+    (
+        "SEC001",
+        "AWS access key id",
+        re.compile(r"\bAKIA[0-9A-Z]{16}\b"),
+    ),
+    (
+        "SEC002",
+        "Generic API key assignment",
+        re.compile(
+            r"(?i)(?:api[_-]?key|secret|token|password|passwd|pwd)"
+            r"\s*[:=]\s*['\"][A-Za-z0-9_\-]{16,}['\"]"
+        ),
+    ),
+    (
+        "SEC003",
+        "PEM private key block",
+        re.compile(r"-----BEGIN (?:RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----"),
+    ),
+    (
+        "SEC004",
+        "Slack/bot token",
+        re.compile(r"\bxox[baprs]-[A-Za-z0-9-]{10,}\b"),
+    ),
+)
+
+
+_UNSAFE_CALLS: tuple[tuple[str | None, str, str], ...] = (
+    # (qualified-prefix-or-None, attr-or-None, rule_id)
+    # Module-qualified calls.
+    ("os", "system", "SEC010"),
+    ("os", "popen", "SEC011"),
+    ("subprocess", "call", "SEC012"),
+    ("subprocess", "Popen", "SEC013"),
+    ("pickle", "load", "SEC017"),
+    ("pickle", "loads", "SEC018"),
+    ("marshal", "load", "SEC019"),
+    ("marshal", "loads", "SEC020"),
+    # Builtins (no qualifier).
+    (None, "eval", "SEC014"),
+    (None, "exec", "SEC015"),
+    (None, "compile", "SEC016"),
+)
+
+
+_WEAK_HASH_NAMES: frozenset[str] = frozenset({"md5", "sha1"})
+
+
+@dataclass
+class SecurityAnalyzer(Analyzer):
+    """Find hard-coded secrets and unsafe calls."""
+
+    name: ClassVar[str] = "security"
+    category: ClassVar[str] = "security"
+    _EXTRA_RULES: ClassVar[tuple[tuple[str, str, re.Pattern[str]], ...]] = ()
+
+    def run(self, context: AnalysisContext) -> list[Finding]:
+        findings: list[Finding] = []
+        for file in context.files:
+            findings.extend(self._scan_secrets(file))
+            findings.extend(self._scan_unsafe_calls(file))
+            findings.extend(self._scan_weak_hash(file))
+            findings.extend(self._scan_asserts(file))
+        return findings
+
+    def _scan_secrets(self, file) -> list[Finding]:
+        out: list[Finding] = []
+        for index, line in enumerate(file.lines, start=1):
+            for rule_id, label, pattern in _SECRET_PATTERNS:
+                if pattern.search(line):
+                    out.append(
+                        Finding(
+                            rule_id=rule_id,
+                            category="security",
+                            severity=Severity.CRITICAL,
+                            message=f"Possible hard-coded {label}.",
+                            path=str(file.path),
+                            line=index,
+                            suggestion=(
+                                "Move the secret to an environment variable or a secret manager."
+                            ),
+                        )
+                    )
+        return out
+
+    def _scan_unsafe_calls(self, file) -> list[Finding]:
+        out: list[Finding] = []
+        try:
+            tree = ast.parse(file.text)
+        except SyntaxError:
+            return out
+        for node in ast.walk(tree):
+            if not isinstance(node, ast.Call):
+                continue
+            qualified = _qualified_name(node.func)
+            if qualified is None:
+                continue
+            for module, attr, rule_id in _UNSAFE_CALLS:
+                target = f"{module}.{attr}" if module else attr
+                if qualified != target:
+                    continue
+                out.append(
+                    Finding(
+                        rule_id=rule_id,
+                        category="security",
+                        severity=Severity.HIGH,
+                        message=f"Unsafe call to {qualified}().",
+                        path=str(file.path),
+                        line=node.lineno,
+                        suggestion=_suggest_for(rule_id),
+                    )
+                )
+            if qualified in {"subprocess.call", "subprocess.Popen"} and any(
+                isinstance(kw.value, ast.Constant) and kw.value.value is True
+                for kw in node.keywords
+                if kw.arg == "shell"
+            ):
+                out.append(
+                    Finding(
+                        rule_id="SEC013",
+                        category="security",
+                        severity=Severity.CRITICAL,
+                        message=(f"{qualified}(shell=True) is vulnerable to command injection."),
+                        path=str(file.path),
+                        line=node.lineno,
+                        suggestion=(
+                            "Pass a list of arguments and drop shell=True; "
+                            "use shlex.quote() only on fully-trusted input."
+                        ),
+                    )
+                )
+        return out
+
+    def _scan_weak_hash(self, file) -> list[Finding]:
+        out: list[Finding] = []
+        try:
+            tree = ast.parse(file.text)
+        except SyntaxError:
+            return out
+        for node in ast.walk(tree):
+            if not isinstance(node, ast.Call):
+                continue
+            name = _qualified_name(node.func)
+            if name is None:
+                continue
+            short = name.rsplit(".", 1)[-1]
+            if short in _WEAK_HASH_NAMES:
+                out.append(
+                    Finding(
+                        rule_id="SEC021",
+                        category="security",
+                        severity=Severity.MEDIUM,
+                        message=(f"{name} is a weak hash; prefer sha256 or sha3_256."),
+                        path=str(file.path),
+                        line=node.lineno,
+                        suggestion=(
+                            "Use hashlib.sha256() (or stronger) for any security-sensitive use."
+                        ),
+                    )
+                )
+        return out
+
+    def _scan_asserts(self, file) -> list[Finding]:
+        out: list[Finding] = []
+        try:
+            tree = ast.parse(file.text)
+        except SyntaxError:
+            return out
+        for node in ast.walk(tree):
+            if not isinstance(node, ast.Assert):
+                continue
+            out.append(
+                Finding(
+                    rule_id="SEC022",
+                    category="security",
+                    severity=Severity.LOW,
+                    message=(
+                        "assert statement is removed under `python -O`; "
+                        "don't use it for runtime checks."
+                    ),
+                    path=str(file.path),
+                    line=node.lineno,
+                    suggestion="Raise an explicit exception instead.",
+                )
+            )
+        return out
+
+
+def _qualified_name(node: ast.AST) -> str | None:
+    """Return the dotted name for a callable node, or None."""
+    if isinstance(node, ast.Name):
+        return node.id
+    if isinstance(node, ast.Attribute):
+        base = _qualified_name(node.value)
+        if base is None:
+            return None
+        return f"{base}.{node.attr}"
+    return None
+
+
+def _suggest_for(rule_id: str) -> str:
+    return {
+        "SEC010": "Use subprocess.run([...], shell=False) instead of os.system.",
+        "SEC011": "Use subprocess.run([...], shell=False) instead of os.popen.",
+        "SEC012": "Pass a list of arguments and shell=False to subprocess.call.",
+        "SEC013": "Pass a list of arguments and shell=False to subprocess.Popen.",
+        "SEC014": "Avoid eval(); use ast.literal_eval() or explicit parsing.",
+        "SEC015": "Avoid exec() on dynamic input; it's a remote-code vector.",
+        "SEC016": "Avoid compile() on user input.",
+        "SEC017": "Use json or a safe deserializer; pickle is unsafe on untrusted data.",
+        "SEC018": "Use json or a safe deserializer; pickle is unsafe on untrusted data.",
+        "SEC019": "marshal is not a safe deserialization format for untrusted data.",
+        "SEC020": "marshal is not a safe deserialization format for untrusted data.",
+    }.get(rule_id, "Replace with a safer alternative.")
+
+
+__all__ = ["SecurityAnalyzer"]
+
+
+# Silence unused-import warnings for symbols only used in some branches.
+_ = field

forgecli/review/finding.py

@@ -0,0 +1,68 @@
+"""Finding and Severity value types shared by all review analyzers."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
+
+
+class Severity(str, Enum):
+    """Severity levels for review findings."""
+
+    INFO = "info"
+    LOW = "low"
+    MEDIUM = "medium"
+    HIGH = "high"
+    CRITICAL = "critical"
+
+    @property
+    def weight(self) -> int:
+        """Numeric weight used for sorting and aggregation."""
+        return {
+            Severity.INFO: 0,
+            Severity.LOW: 1,
+            Severity.MEDIUM: 2,
+            Severity.HIGH: 3,
+            Severity.CRITICAL: 4,
+        }[self]
+
+    @property
+    def is_blocking(self) -> bool:
+        """A blocking finding is one we should refuse to merge."""
+        return self is Severity.CRITICAL
+
+
+@dataclass(frozen=True)
+class Finding:
+    """A single review observation.
+
+    The ``category`` is one of: ``"security"``, ``"performance"``,
+    ``"architecture"``, ``"complexity"``, ``"dead-code"``,
+    ``"duplicates"``, ``"suggestions"``.
+    """
+
+    rule_id: str
+    category: str
+    severity: Severity
+    message: str
+    path: str | None = None
+    line: int | None = None
+    suggestion: str | None = None
+    extra: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Return a JSON-serializable view of this finding."""
+        return {
+            "rule_id": self.rule_id,
+            "category": self.category,
+            "severity": self.severity.value,
+            "message": self.message,
+            "path": self.path,
+            "line": self.line,
+            "suggestion": self.suggestion,
+            "extra": self.extra,
+        }
+
+
+__all__ = ["Finding", "Severity"]