codeclone 1.0.0__tar.gz

codeclone-1.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Denis Rozhnovskiy
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

codeclone-1.0.0/PKG-INFO

@@ -0,0 +1,211 @@
+Metadata-Version: 2.4
+Name: codeclone
+Version: 1.0.0
+Summary: AST-based code clone detector for Python focused on architectural duplication
+Author-email: Den Rozhnovskiy <pytelemonbot@mail.ru>
+Maintainer-email: Den Rozhnovskiy <pytelemonbot@mail.ru>
+License: MIT
+Project-URL: Homepage, https://github.com/orenlab/codeclone
+Project-URL: Repository, https://github.com/orenlab/codeclone
+Project-URL: Issues, https://github.com/orenlab/codeclone/issues
+Project-URL: Changelog, https://github.com/orenlab/codeclone/releases
+Keywords: python,ast,code-clone,duplication,static-analysis,ci,architecture
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Software Development :: Code Generators
+Classifier: Topic :: Software Development :: Testing
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: dev
+Requires-Dist: pytest>=9.0.0; extra == "dev"
+Requires-Dist: build>=1.2.0; extra == "dev"
+Requires-Dist: twine>=5.0.0; extra == "dev"
+Dynamic: license-file
+
+# CodeClone
+
+**CodeClone** is an AST-based code clone detector for Python, focused on **architectural duplication**, not simple
+copy-paste.
+
+It is designed to help teams:
+
+- discover structural and logical code duplication,
+- understand architectural hotspots,
+- and prevent *new* duplication from entering the codebase via CI.
+
+Unlike token- or text-based tools, CodeClone works on **normalized Python AST**, which makes it robust against renaming,
+formatting, and minor refactoring.
+
+---
+
+## Why CodeClone?
+
+Most existing tools detect *textual* duplication.
+CodeClone detects **structural and block-level duplication** that usually indicates missing abstractions or
+architectural drift.
+
+Typical use cases:
+
+- duplicated service logic across layers (API ↔ application),
+- repeated validation or guard blocks,
+- copy-pasted request/handler flows,
+- duplicated orchestration logic in routers, handlers, or services.
+
+---
+
+## Features
+
+### Function-level clone detection (Type-2)
+
+- Detects functions and methods with identical structure.
+- Robust to:
+    - variable renaming,
+    - constant changes,
+    - formatting differences.
+- Ideal for spotting architectural duplication between layers.
+
+### Block-level clone detection (Type-3-lite)
+
+- Detects repeated **statement blocks** inside larger functions.
+- Targets:
+    - validation blocks,
+    - guard clauses,
+    - repeated orchestration logic.
+- Carefully filtered to avoid noise:
+    - no overlapping windows,
+    - no clones inside the same function,
+    - no `__init__` noise.
+
+### Low-noise by design
+
+- AST normalization instead of token matching.
+- Size and statement-count thresholds.
+- Conservative defaults tuned for real-world Python projects.
+
+### CI-friendly baseline mode
+
+- Establish a baseline of existing clones.
+- Fail CI **only when new clones are introduced**.
+- Safe for legacy codebases.
+
+---
+
+## Installation
+
+```bash
+pip install codeclone
+```
+
+Python 3.10+ is required.
+
+⸻
+
+Quick Start
+
+Run on a project:
+
+```bash
+codeclone .
+```
+
+This will:
+
+* scan Python files,
+* detect function-level and block-level clones,
+* print a summary to stdout.
+
+Generate reports:
+
+```bash
+codeclone . \
+  --json-out .cache/codeclone/report.json \
+  --text-out .cache/codeclone/report.txt
+```
+
+⸻
+
+Baseline Workflow (Recommended)
+
+1. Create a baseline
+
+Run once on your current codebase:
+
+```bash
+codeclone . --update-baseline
+```
+
+This creates a file:
+
+```bash
+.codeclone-baseline.json
+```
+
+Commit this file to the repository.
+
+⸻
+
+2. Use in CI
+
+In CI, run:
+
+```bash
+codeclone . --fail-on-new
+```
+
+Behavior:
+
+* ✅ existing clones are allowed,
+* ❌ build fails if new function or block clones appear,
+* ✅ refactoring that removes duplication is always allowed.
+
+This enables gradual improvement without breaking existing development flow.
+
+⸻
+
+What CodeClone Is (and Is Not)
+
+CodeClone is
+
+* an architectural analysis tool,
+* a duplication radar,
+* a CI guard against copy-paste.
+
+CodeClone is not
+
+* a linter,
+* a formatter,
+* a replacement for SonarQube or static analyzers,
+* a semantic equivalence prover.
+
+It intentionally focuses on high-signal duplication.
+
+⸻
+
+How It Works (High Level)
+
+* Parses Python source into AST.
+* Normalizes:
+    - variable names,
+    - constants,
+    - attributes,
+    - docstrings and annotations.
+* Computes stable structural fingerprints.
+* Detects:
+    - identical function structures,
+    - repeated statement blocks across functions.
+* Applies filters to suppress noise.
+
+⸻
+
+License
+
+MIT License

codeclone-1.0.0/README.md

@@ -0,0 +1,178 @@
+# CodeClone
+
+**CodeClone** is an AST-based code clone detector for Python, focused on **architectural duplication**, not simple
+copy-paste.
+
+It is designed to help teams:
+
+- discover structural and logical code duplication,
+- understand architectural hotspots,
+- and prevent *new* duplication from entering the codebase via CI.
+
+Unlike token- or text-based tools, CodeClone works on **normalized Python AST**, which makes it robust against renaming,
+formatting, and minor refactoring.
+
+---
+
+## Why CodeClone?
+
+Most existing tools detect *textual* duplication.
+CodeClone detects **structural and block-level duplication** that usually indicates missing abstractions or
+architectural drift.
+
+Typical use cases:
+
+- duplicated service logic across layers (API ↔ application),
+- repeated validation or guard blocks,
+- copy-pasted request/handler flows,
+- duplicated orchestration logic in routers, handlers, or services.
+
+---
+
+## Features
+
+### Function-level clone detection (Type-2)
+
+- Detects functions and methods with identical structure.
+- Robust to:
+    - variable renaming,
+    - constant changes,
+    - formatting differences.
+- Ideal for spotting architectural duplication between layers.
+
+### Block-level clone detection (Type-3-lite)
+
+- Detects repeated **statement blocks** inside larger functions.
+- Targets:
+    - validation blocks,
+    - guard clauses,
+    - repeated orchestration logic.
+- Carefully filtered to avoid noise:
+    - no overlapping windows,
+    - no clones inside the same function,
+    - no `__init__` noise.
+
+### Low-noise by design
+
+- AST normalization instead of token matching.
+- Size and statement-count thresholds.
+- Conservative defaults tuned for real-world Python projects.
+
+### CI-friendly baseline mode
+
+- Establish a baseline of existing clones.
+- Fail CI **only when new clones are introduced**.
+- Safe for legacy codebases.
+
+---
+
+## Installation
+
+```bash
+pip install codeclone
+```
+
+Python 3.10+ is required.
+
+⸻
+
+Quick Start
+
+Run on a project:
+
+```bash
+codeclone .
+```
+
+This will:
+
+* scan Python files,
+* detect function-level and block-level clones,
+* print a summary to stdout.
+
+Generate reports:
+
+```bash
+codeclone . \
+  --json-out .cache/codeclone/report.json \
+  --text-out .cache/codeclone/report.txt
+```
+
+⸻
+
+Baseline Workflow (Recommended)
+
+1. Create a baseline
+
+Run once on your current codebase:
+
+```bash
+codeclone . --update-baseline
+```
+
+This creates a file:
+
+```bash
+.codeclone-baseline.json
+```
+
+Commit this file to the repository.
+
+⸻
+
+2. Use in CI
+
+In CI, run:
+
+```bash
+codeclone . --fail-on-new
+```
+
+Behavior:
+
+* ✅ existing clones are allowed,
+* ❌ build fails if new function or block clones appear,
+* ✅ refactoring that removes duplication is always allowed.
+
+This enables gradual improvement without breaking existing development flow.
+
+⸻
+
+What CodeClone Is (and Is Not)
+
+CodeClone is
+
+* an architectural analysis tool,
+* a duplication radar,
+* a CI guard against copy-paste.
+
+CodeClone is not
+
+* a linter,
+* a formatter,
+* a replacement for SonarQube or static analyzers,
+* a semantic equivalence prover.
+
+It intentionally focuses on high-signal duplication.
+
+⸻
+
+How It Works (High Level)
+
+* Parses Python source into AST.
+* Normalizes:
+    - variable names,
+    - constants,
+    - attributes,
+    - docstrings and annotations.
+* Computes stable structural fingerprints.
+* Detects:
+    - identical function structures,
+    - repeated statement blocks across functions.
+* Applies filters to suppress noise.
+
+⸻
+
+License
+
+MIT License

codeclone-1.0.0/codeclone/baseline.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Set
+
+
+class Baseline:
+    def __init__(self, path: str):
+        self.path = Path(path)
+        self.functions: Set[str] = set()
+        self.blocks: Set[str] = set()
+
+    def load(self) -> None:
+        if not self.path.exists():
+            return
+
+        data = json.loads(self.path.read_text("utf-8"))
+        self.functions = set(data.get("functions", []))
+        self.blocks = set(data.get("blocks", []))
+
+    def save(self) -> None:
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self.path.write_text(
+            json.dumps(
+                {
+                    "functions": sorted(self.functions),
+                    "blocks": sorted(self.blocks),
+                },
+                indent=2,
+                ensure_ascii=False,
+            ),
+            "utf-8",
+        )
+
+    @staticmethod
+    def from_groups(func_groups: dict, block_groups: dict) -> "Baseline":
+        bl = Baseline("")
+        bl.functions = set(func_groups.keys())
+        bl.blocks = set(block_groups.keys())
+        return bl
+
+    def diff(self, func_groups: dict, block_groups: dict) -> tuple[set, set]:
+        new_funcs = set(func_groups.keys()) - self.functions
+        new_blocks = set(block_groups.keys()) - self.blocks
+        return new_funcs, new_blocks

codeclone-1.0.0/codeclone/blockhash.py

@@ -0,0 +1,12 @@
+from __future__ import annotations
+
+import ast
+import hashlib
+
+from .normalize import NormalizationConfig, AstNormalizer
+
+def stmt_hash(stmt: ast.stmt, cfg: NormalizationConfig) -> str:
+    normalizer = AstNormalizer(cfg)
+    stmt = ast.fix_missing_locations(normalizer.visit(stmt))
+    dump = ast.dump(stmt, annotate_fields=True, include_attributes=False)
+    return hashlib.sha1(dump.encode("utf-8")).hexdigest()

codeclone-1.0.0/codeclone/blocks.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+import ast
+from dataclasses import dataclass
+
+from .blockhash import stmt_hash
+from .normalize import NormalizationConfig
+
+
+@dataclass(frozen=True)
+class BlockUnit:
+    block_hash: str
+    filepath: str
+    qualname: str
+    start_line: int
+    end_line: int
+    size: int
+
+
+def extract_blocks(
+        func_node: ast.AST,
+        *,
+        filepath: str,
+        qualname: str,
+        cfg: NormalizationConfig,
+        block_size: int,
+        max_blocks: int,
+) -> list[BlockUnit]:
+    body = getattr(func_node, "body", None)
+    if not isinstance(body, list) or len(body) < block_size:
+        return []
+
+    stmt_hashes = [stmt_hash(stmt, cfg) for stmt in body]
+
+    blocks: list[BlockUnit] = []
+    last_start: int | None = None
+    MIN_LINE_DISTANCE = 5  # suppress overlapping windows
+
+    for i in range(len(stmt_hashes) - block_size + 1):
+        start = getattr(body[i], "lineno", None)
+        end = getattr(body[i + block_size - 1], "end_lineno", None)
+        if not start or not end:
+            continue
+
+        if last_start is not None and start - last_start < MIN_LINE_DISTANCE:
+            continue
+
+        bh = "|".join(stmt_hashes[i:i + block_size])
+
+        blocks.append(BlockUnit(
+            block_hash=bh,
+            filepath=filepath,
+            qualname=qualname,
+            start_line=start,
+            end_line=end,
+            size=block_size,
+        ))
+
+        last_start = start
+        if len(blocks) >= max_blocks:
+            break
+
+    return blocks

codeclone-1.0.0/codeclone/cache.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import asdict
+from pathlib import Path
+from typing import Optional
+
+
+class Cache:
+    def __init__(self, path: str):
+        self.path = Path(path)
+        self.data: dict = {"files": {}}
+
+    def load(self) -> None:
+        if self.path.exists():
+            self.data = json.loads(self.path.read_text("utf-8"))
+
+    def save(self) -> None:
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self.path.write_text(
+            json.dumps(self.data, ensure_ascii=False, indent=2),
+            "utf-8",
+        )
+
+    def get_file_entry(self, filepath: str) -> Optional[dict]:
+        return self.data.get("files", {}).get(filepath)
+
+    def put_file_entry(self, filepath: str, stat_sig: dict, units, blocks) -> None:
+        self.data.setdefault("files", {})[filepath] = {
+            "stat": stat_sig,
+            "units": [asdict(u) for u in units],
+            "blocks": [asdict(b) for b in blocks],
+        }
+
+
+def file_stat_signature(path: str) -> dict:
+    st = os.stat(path)
+    return {
+        "mtime_ns": st.st_mtime_ns,
+        "size": st.st_size,
+    }