codeclone 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

codeclone/normalize.py

@@ -1,6 +1,17 @@
+"""
+CodeClone — AST and CFG-based code clone detector for Python
+focused on architectural duplication.
+
+Copyright (c) 2026 Den Rozhnovskiy
+Licensed under the MIT License.
+"""
+
 from __future__ import annotations
 
 import ast
+import copy
+from ast import AST
+from collections.abc import Sequence
 from dataclasses import dataclass
 
 
@@ -8,15 +19,15 @@ from dataclasses import dataclass
 class NormalizationConfig:
     ignore_docstrings: bool = True
     ignore_type_annotations: bool = True
-    normalize_attributes: bool = True  # obj.foo -> obj._ATTR_
-    normalize_constants: bool = True  # 123/"x"/None -> _CONST_
-    normalize_names: bool = True  # x,y,z -> _VAR_
+    normalize_attributes: bool = True
+    normalize_constants: bool = True
+    normalize_names: bool = True
 
 
 class AstNormalizer(ast.NodeTransformer):
     def __init__(self, cfg: NormalizationConfig):
-        self.cfg = cfg
         super().__init__()
+        self.cfg = cfg
 
     def visit_FunctionDef(self, node: ast.FunctionDef):
         return self._visit_func(node)
@@ -24,28 +35,30 @@ class AstNormalizer(ast.NodeTransformer):
     def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef):
         return self._visit_func(node)
 
-    def _visit_func(self, node):
-        # Drop docstring (first Expr(Constant(str)))
+    def _visit_func(self, node: ast.FunctionDef | ast.AsyncFunctionDef):
+        # Drop docstring
         if self.cfg.ignore_docstrings and node.body:
             first = node.body[0]
-            if isinstance(first, ast.Expr) and isinstance(getattr(first, "value", None), ast.Constant):
-                if isinstance(first.value.value, str):
-                    node.body = node.body[1:]
+            if (
+                isinstance(first, ast.Expr)
+                and isinstance(first.value, ast.Constant)
+                and isinstance(first.value.value, str)
+            ):
+                node.body = node.body[1:]
 
         if self.cfg.ignore_type_annotations:
-            # Remove annotations in args + returns
-            if hasattr(node, "returns"):
-                node.returns = None
+            node.returns = None
             args = node.args
+
             for a in getattr(args, "posonlyargs", []):
                 a.annotation = None
-            for a in getattr(args, "args", []):
+            for a in args.args:
                 a.annotation = None
-            for a in getattr(args, "kwonlyargs", []):
+            for a in args.kwonlyargs:
                 a.annotation = None
-            if getattr(args, "vararg", None):
+            if args.vararg:
                 args.vararg.annotation = None
-            if getattr(args, "kwarg", None):
+            if args.kwarg:
                 args.kwarg.annotation = None
 
         return self.generic_visit(node)
@@ -60,24 +73,58 @@ class AstNormalizer(ast.NodeTransformer):
             node.id = "_VAR_"
         return node
 
-    def visit_Attribute(self, node: ast.Attribute):
-        node = self.generic_visit(node)
+    def visit_Attribute(self, node: ast.Attribute) -> ast.Attribute:
+        new_node = self.generic_visit(node)
+        assert isinstance(new_node, ast.Attribute)
         if self.cfg.normalize_attributes:
-            node.attr = "_ATTR_"
-        return node
+            new_node.attr = "_ATTR_"
+        return new_node
 
     def visit_Constant(self, node: ast.Constant):
         if self.cfg.normalize_constants:
-            # Preserve booleans? up to you; default: normalize everything
             node.value = "_CONST_"
         return node
 
+    def visit_AugAssign(self, node: ast.AugAssign) -> AST:
+        # Normalize x += 1 to x = x + 1
+        # This allows detecting clones where one uses += and another uses = +
+        # We transform AugAssign(target, op, value) to Assign([target], BinOp(target, op, value))
+
+        # Deepcopy target to avoid reuse issues in the AST
+        target_load = copy.deepcopy(node.target)
+        # Ensure context is Load() for the right-hand side usage
+        if hasattr(target_load, "ctx"):
+            target_load.ctx = ast.Load()
+
+        new_node = ast.Assign(
+            targets=[node.target],
+            value=ast.BinOp(left=target_load, op=node.op, right=node.value),
+            lineno=node.lineno,
+            col_offset=node.col_offset,
+            end_lineno=getattr(node, "end_lineno", None),
+            end_col_offset=getattr(node, "end_col_offset", None),
+        )
+        return self.generic_visit(new_node)
+
 
 def normalized_ast_dump(func_node: ast.AST, cfg: NormalizationConfig) -> str:
-    """
-    Returns stable string representation of normalized AST.
-    """
     normalizer = AstNormalizer(cfg)
-    new_node = ast.fix_missing_locations(normalizer.visit(ast.copy_location(func_node, func_node)))
-    # include_attributes=False => more stable; annotate_fields=True => default
+    # Deepcopy to prevent side effects on the original AST
+    node_copy = copy.deepcopy(func_node)
+    new_node = ast.fix_missing_locations(normalizer.visit(node_copy))
     return ast.dump(new_node, annotate_fields=True, include_attributes=False)
+
+
+def normalized_ast_dump_from_list(
+    nodes: Sequence[ast.AST], cfg: NormalizationConfig
+) -> str:
+    normalizer = AstNormalizer(cfg)
+    dumps: list[str] = []
+
+    for node in nodes:
+        # Deepcopy to prevent side effects
+        node_copy = copy.deepcopy(node)
+        new_node = ast.fix_missing_locations(normalizer.visit(node_copy))
+        dumps.append(ast.dump(new_node, annotate_fields=True, include_attributes=False))
+
+    return ";".join(dumps)

codeclone/report.py

@@ -1,23 +1,33 @@
+"""
+CodeClone — AST and CFG-based code clone detector for Python
+focused on architectural duplication.
+
+Copyright (c) 2026 Den Rozhnovskiy
+Licensed under the MIT License.
+"""
+
 from __future__ import annotations
 
 import json
 from typing import Any
 
 
-def build_groups(units: list[dict[str, Any]]) -> dict[str, list[dict]]:
-    groups: dict[str, list[dict]] = {}
+def build_groups(units: list[dict[str, Any]]) -> dict[str, list[dict[str, Any]]]:
+    groups: dict[str, list[dict[str, Any]]] = {}
     for u in units:
         key = f"{u['fingerprint']}|{u['loc_bucket']}"
         groups.setdefault(key, []).append(u)
     return {k: v for k, v in groups.items() if len(v) > 1}
 
 
-def build_block_groups(blocks: list[dict], min_functions: int = 2) -> dict[str, list[dict]]:
-    groups: dict[str, list[dict]] = {}
+def build_block_groups(
+    blocks: list[dict[str, Any]], min_functions: int = 2
+) -> dict[str, list[dict[str, Any]]]:
+    groups: dict[str, list[dict[str, Any]]] = {}
     for b in blocks:
         groups.setdefault(b["block_hash"], []).append(b)
 
-    filtered: dict[str, list[dict]] = {}
+    filtered: dict[str, list[dict[str, Any]]] = {}
     for h, items in groups.items():
         functions = {i["qualname"] for i in items}
         if len(functions) >= min_functions:
@@ -27,19 +37,25 @@ def build_block_groups(blocks: list[dict], min_functions: int = 2) -> dict[str,
 
 
 def to_json(groups: dict) -> str:
-    return json.dumps({
-        "group_count": len(groups),
-        "groups": [
-            {"key": k, "count": len(v), "items": v}
-            for k, v in sorted(groups.items(), key=lambda kv: len(kv[1]), reverse=True)
-        ],
-    }, ensure_ascii=False, indent=2)
+    return json.dumps(
+        {
+            "group_count": len(groups),
+            "groups": [
+                {"key": k, "count": len(v), "items": v}
+                for k, v in sorted(
+                    groups.items(), key=lambda kv: len(kv[1]), reverse=True
+                )
+            ],
+        },
+        ensure_ascii=False,
+        indent=2,
+    )
 
 
 def to_text(groups: dict) -> str:
     lines: list[str] = []
     for i, (_, v) in enumerate(
-            sorted(groups.items(), key=lambda kv: len(kv[1]), reverse=True)
+        sorted(groups.items(), key=lambda kv: len(kv[1]), reverse=True)
     ):
         lines.append(f"\n=== Clone group #{i + 1} (count={len(v)}) ===")
         for item in v:

codeclone/scanner.py

@@ -1,14 +1,33 @@
+"""
+CodeClone — AST and CFG-based code clone detector for Python
+focused on architectural duplication.
+
+Copyright (c) 2026 Den Rozhnovskiy
+Licensed under the MIT License.
+"""
+
 from __future__ import annotations
 
 from pathlib import Path
 from typing import Iterable
 
 DEFAULT_EXCLUDES = (
-    ".git", ".venv", "venv", "__pycache__", "site-packages",
-    "migrations", "alembic", "dist", "build", ".tox",
+    ".git",
+    ".venv",
+    "venv",
+    "__pycache__",
+    "site-packages",
+    "migrations",
+    "alembic",
+    "dist",
+    "build",
+    ".tox",
 )
 
-def iter_py_files(root: str, excludes: tuple[str, ...] = DEFAULT_EXCLUDES) -> Iterable[str]:
+
+def iter_py_files(
+    root: str, excludes: tuple[str, ...] = DEFAULT_EXCLUDES
+) -> Iterable[str]:
     rootp = Path(root)
     for p in rootp.rglob("*.py"):
         parts = set(p.parts)
@@ -16,6 +35,7 @@ def iter_py_files(root: str, excludes: tuple[str, ...] = DEFAULT_EXCLUDES) -> It
             continue
         yield str(p)
 
+
 def module_name_from_path(root: str, filepath: str) -> str:
     rootp = Path(root).resolve()
     fp = Path(filepath).resolve()
@@ -25,4 +45,4 @@ def module_name_from_path(root: str, filepath: str) -> str:
     # __init__.py -> package name
     if stem.name == "__init__":
         stem = stem.parent
-    return ".".join(stem.parts)
+    return ".".join(stem.parts)

codeclone-1.2.0.dist-info/METADATA

@@ -0,0 +1,264 @@
+Metadata-Version: 2.4
+Name: codeclone
+Version: 1.2.0
+Summary: AST and CFG-based code clone detector for Python focused on architectural duplication
+Author-email: Den Rozhnovskiy <pytelemonbot@mail.ru>
+Maintainer-email: Den Rozhnovskiy <pytelemonbot@mail.ru>
+License: MIT
+Project-URL: Homepage, https://github.com/orenlab/codeclone
+Project-URL: Repository, https://github.com/orenlab/codeclone
+Project-URL: Issues, https://github.com/orenlab/codeclone/issues
+Project-URL: Changelog, https://github.com/orenlab/codeclone/releases
+Project-URL: Documentation, https://github.com/orenlab/codeclone/tree/main/docs
+Keywords: python,ast,code-clone,duplication,static-analysis,ci,architecture
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Software Development :: Code Generators
+Classifier: Topic :: Software Development :: Testing
+Classifier: Typing :: Typed
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pygments>=2.19.2
+Requires-Dist: rich>=14.3.2
+Provides-Extra: dev
+Requires-Dist: pytest>=9.0.0; extra == "dev"
+Requires-Dist: build>=1.2.0; extra == "dev"
+Requires-Dist: twine>=5.0.0; extra == "dev"
+Requires-Dist: mypy>=1.19.1; extra == "dev"
+Dynamic: license-file
+
+# CodeClone
+
+[![PyPI](https://img.shields.io/pypi/v/codeclone.svg)](https://pypi.org/project/codeclone/)
+[![Downloads](https://img.shields.io/pypi/dm/codeclone.svg)](https://pypi.org/project/codeclone/)
+[![Python](https://img.shields.io/pypi/pyversions/codeclone.svg)](https://pypi.org/project/codeclone/)
+[![License](https://img.shields.io/pypi/l/codeclone.svg)](LICENSE)
+
+**CodeClone** is a Python code clone detector based on **normalized AST and control-flow graphs (CFG)**.
+It helps teams discover architectural duplication and prevent new copy-paste from entering the codebase via CI.
+
+CodeClone is designed to help teams:
+
+- discover **structural and control-flow duplication**,
+- identify architectural hotspots,
+- prevent *new* duplication via CI and pre-commit hooks.
+
+Unlike token- or text-based tools, CodeClone operates on **normalized Python AST and CFG**, making it robust against renaming,
+formatting, and minor refactoring.
+
+---
+
+## Why CodeClone?
+
+Most existing tools detect *textual* duplication.
+CodeClone detects **structural and block-level duplication**, which usually signals missing abstractions or architectural drift.
+
+Typical use cases:
+
+- duplicated service or orchestration logic across layers (API ↔ application),
+- repeated validation or guard blocks,
+- copy-pasted request / handler flows,
+- duplicated control-flow logic in routers, handlers, or services.
+
+---
+
+## Features
+
+### Function-level clone detection (Type-2, CFG-based)
+
+- Detects functions and methods with identical **control-flow structure**.
+- Based on **Control Flow Graph (CFG)** fingerprinting.
+- Robust to:
+  - variable renaming,
+  - constant changes,
+  - attribute renaming,
+  - formatting differences,
+  - docstrings and type annotations.
+- Ideal for spotting architectural duplication across layers.
+
+### Block-level clone detection (Type-3-lite)
+
+- Detects repeated **statement blocks** inside larger functions.
+- Uses sliding windows over CFG-normalized statement sequences.
+- Targets:
+  - validation blocks,
+  - guard clauses,
+  - repeated orchestration logic.
+- Carefully filtered to reduce noise:
+  - no overlapping windows,
+  - no clones inside the same function,
+  - no `__init__` noise,
+  - size and statement-count thresholds.
+
+### Control-Flow Awareness (CFG v1)
+
+- Each function is converted into a **Control Flow Graph**.
+- CFG nodes contain normalized AST statements.
+- CFG edges represent structural control flow:
+  - `if` / `else`
+  - `for` / `async for` / `while`
+  - `try` / `except` / `finally`
+  - `with` / `async with`
+  - `match` / `case` (Python 3.10+)
+- Current CFG semantics (v1):
+  - `break` and `continue` are treated as statements (no jump targets),
+  - after-blocks are explicit and always present,
+  - focus is on **structural similarity**, not precise runtime semantics.
+
+This design keeps clone detection **stable, deterministic, and low-noise**.
+
+### Low-noise by design
+
+- AST + CFG normalization instead of token matching.
+- Conservative defaults tuned for real-world Python projects.
+- Explicit thresholds for size and statement count.
+- Focus on *architectural duplication*, not micro-similarities.
+
+### CI-friendly baseline mode
+
+- Establish a baseline of existing clones.
+- Fail CI **only when new clones are introduced**.
+- Safe for legacy codebases and incremental refactoring.
+
+---
+
+## Installation
+
+```bash
+pip install codeclone
+```
+
+Python **3.10+** is required.
+
+---
+
+## Quick Start
+
+Run on a project:
+
+```bash
+codeclone .
+```
+
+This will:
+
+- scan Python files,
+- build CFGs for functions,
+- detect function-level and block-level clones,
+- print a summary to stdout.
+
+Generate reports:
+
+```bash
+codeclone . \
+  --json .cache/codeclone/report.json \
+  --text .cache/codeclone/report.txt
+```
+
+Generate an HTML report:
+
+```bash
+codeclone . --html .cache/codeclone/report.html
+```
+
+---
+
+## Baseline Workflow (Recommended)
+
+### 1. Create a baseline
+
+Run once on your current codebase:
+
+```bash
+codeclone . --update-baseline
+```
+
+Commit the generated baseline file to the repository.
+
+### 2. Use in CI
+
+```bash
+codeclone . --fail-on-new
+```
+
+Behavior:
+
+- ✅ existing clones are allowed,
+- ❌ build fails if *new* clones appear,
+- ✅ refactoring that removes duplication is always allowed.
+
+---
+
+## Using with pre-commit
+
+```yaml
+repos:
+-   repo: local
+    hooks:
+    -   id: codeclone
+        name: CodeClone
+        entry: codeclone
+        language: python
+        args: [".", "--fail-on-new"]
+        types: [python]
+```
+
+---
+
+## What CodeClone Is (and Is Not)
+
+### CodeClone **is**
+
+- an architectural analysis tool,
+- a duplication radar,
+- a CI guard against copy-paste,
+- a control-flow-aware clone detector.
+
+### CodeClone **is not**
+
+- a linter,
+- a formatter,
+- a semantic equivalence prover,
+- a runtime analyzer.
+
+---
+
+## How It Works (High Level)
+
+1. Parse Python source into AST.
+2. Normalize AST (names, constants, attributes, annotations).
+3. Build a **Control Flow Graph (CFG)** per function.
+4. Compute stable CFG fingerprints.
+5. Detect function-level and block-level clones.
+6. Apply conservative filters to suppress noise.
+
+See the architectural overview:
+- [docs/architecture.md](docs/architecture.md)
+
+---
+
+## Control Flow Graph (CFG)
+
+Starting from **version 1.1.0**, CodeClone uses a **Control Flow Graph (CFG)**
+to improve structural clone detection robustness.
+
+The CFG is a **structural abstraction**, not a runtime execution model.
+
+See full design and semantics:
+- [docs/cfg.md](docs/cfg.md)
+
+---
+
+## License
+
+MIT License

codeclone-1.2.0.dist-info/RECORD

@@ -0,0 +1,19 @@
+codeclone/__init__.py,sha256=_MZuqgioYGn49qw6_OluedgcRvuu8IhAwVgHkM4Kj7s,364
+codeclone/baseline.py,sha256=tV-vgCpyQi-g1AlWJwuUFDeQwUZuhj4tX-BxFDv-LWo,1719
+codeclone/blockhash.py,sha256=jOswW9jqe9ww3y4r-gLTUZjMmn0CHXpU5qTtFndKQ10,594
+codeclone/blocks.py,sha256=Y75BSpzf-zyyeD9-vKnQfJ3QwIjYxMwiN9DbEqnbONg,1735
+codeclone/cache.py,sha256=DexslhZfxj79fnoPna8r_oBCqmsstN2ICRA_o6ZVpGo,1559
+codeclone/cfg.py,sha256=op503zRnew2ZIz5coK_HUaKt1VG1ucJBacXAZ1rBJYQ,11587
+codeclone/cli.py,sha256=c_61l3wSF2nbTZgd6LmvoUvejXatydPAPmJyRknHx78,13229
+codeclone/extractor.py,sha256=crCxgkK1n2fl5FUz6HtbaEZUH5CO8S0zqM0Xj0RSE6E,4558
+codeclone/fingerprint.py,sha256=H0YY209sfGf02VeLlxHNDp7n6es0vLiMmq3TBWDm3SE,545
+codeclone/html_report.py,sha256=e7gYxEHk5ezJtGUYIYsQlcxu_0fP3hmd9jj-zMWMfJY,25574
+codeclone/normalize.py,sha256=bvMoY3VDiZsnFiD50h5XUgwnMUKOyUEx6lJXCBiembg,4290
+codeclone/report.py,sha256=IwblTgZe4liTq3gHagnw6O4ZUkRqJ448XqzhLhJZoWM,1972
+codeclone/scanner.py,sha256=BdJFyaLv1xvimVkJyvvTN0FcG2RQZzRlTlHWi2fRQnU,1050
+codeclone-1.2.0.dist-info/licenses/LICENSE,sha256=ndXAbunvN-jCQjgCaoBOF5AN4FcRlAa0R7hK1lWDuBU,1073
+codeclone-1.2.0.dist-info/METADATA,sha256=RekVw29wHrLX0fHNLBYS9Ky6Ee6oFnntQxeCOcRbMTM,7255
+codeclone-1.2.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+codeclone-1.2.0.dist-info/entry_points.txt,sha256=_MI9DVTLOmv3OlxpyogdOmMAduiLVIdHeOlZ_KBsrIY,49
+codeclone-1.2.0.dist-info/top_level.txt,sha256=4tQa_d-4Zle-qV9KmNDkWq0WHYgZsW9vdaeF30rNntg,10
+codeclone-1.2.0.dist-info/RECORD,,

{codeclone-1.0.0.dist-info → codeclone-1.2.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any