codeecho 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codeecho/.ignore +41 -0
- codeecho/__init__.py +22 -0
- codeecho/__main__.py +289 -0
- codeecho/db.py +289 -0
- codeecho/detector.py +246 -0
- codeecho/extractor.py +139 -0
- codeecho/fingerprint.py +38 -0
- codeecho/logging.ini +28 -0
- codeecho/models.py +52 -0
- codeecho/normalizer.py +519 -0
- codeecho/parser.py +94 -0
- codeecho/reporter/__init__.py +6 -0
- codeecho/reporter/html_reporter.py +212 -0
- codeecho/reporter/json_reporter.py +82 -0
- codeecho/scanner.py +109 -0
- codeecho-1.0.0.dist-info/METADATA +208 -0
- codeecho-1.0.0.dist-info/RECORD +20 -0
- codeecho-1.0.0.dist-info/WHEEL +4 -0
- codeecho-1.0.0.dist-info/entry_points.txt +3 -0
- codeecho-1.0.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
"""
|
|
2
|
+
HTML reporter: renders an inline self-contained report from session database results.
|
|
3
|
+
|
|
4
|
+
The HTML is generated via a Jinja2 template embedded as a module-level string — no
|
|
5
|
+
external files required. The report includes a summary panel, per-clone-type stats,
|
|
6
|
+
and expandable code-snippet cards for every clone group.
|
|
7
|
+
|
|
8
|
+
:author: Ron Webb
|
|
9
|
+
:since: 1.0.0
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import logging
|
|
13
|
+
from datetime import datetime, timezone
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
from jinja2 import BaseLoader, Environment
|
|
17
|
+
|
|
18
|
+
from codeecho.db import SessionDB
|
|
19
|
+
from codeecho.models import ScanResult
|
|
20
|
+
|
|
21
|
+
_logger = logging.getLogger("codeecho.reporter.html")
|
|
22
|
+
|
|
23
|
+
# ── Jinja2 template ───────────────────────────────────────────────────────────
|
|
24
|
+
|
|
25
|
+
_TEMPLATE: str = """<!DOCTYPE html>
|
|
26
|
+
<html lang="en">
|
|
27
|
+
<head>
|
|
28
|
+
<meta charset="utf-8"/>
|
|
29
|
+
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
|
30
|
+
<title>CodeEcho — Clone Detection Report</title>
|
|
31
|
+
<style>
|
|
32
|
+
:root{--bg:#0f1117;--surface:#1a1d27;--border:#2e3248;--accent:#7c6af7;
|
|
33
|
+
--t1:#ef4444;--t2:#f59e0b;--t3:#22c55e;--text:#e2e8f0;--muted:#94a3b8}
|
|
34
|
+
*{box-sizing:border-box;margin:0;padding:0}
|
|
35
|
+
body{background:var(--bg);color:var(--text);font-family:system-ui,sans-serif;
|
|
36
|
+
font-size:14px;line-height:1.6;padding:2rem}
|
|
37
|
+
h1{font-size:1.6rem;font-weight:700;margin-bottom:.25rem}
|
|
38
|
+
.subtitle{color:var(--muted);margin-bottom:1rem}
|
|
39
|
+
.toolbar{display:flex;align-items:center;gap:.5rem;margin-bottom:1.5rem}
|
|
40
|
+
.btn{background:var(--surface);border:1px solid var(--border);border-radius:6px;
|
|
41
|
+
color:var(--text);cursor:pointer;font-size:.8rem;padding:.35rem .9rem;
|
|
42
|
+
font-family:inherit}
|
|
43
|
+
.btn:hover{border-color:var(--accent);color:var(--accent)}
|
|
44
|
+
#show-all-btn{display:none;margin-left:auto}
|
|
45
|
+
.stats{display:flex;gap:1rem;flex-wrap:wrap;margin-bottom:1.5rem}
|
|
46
|
+
.stat{background:var(--surface);border:1px solid var(--border);border-radius:8px;
|
|
47
|
+
padding:1rem 1.5rem;min-width:130px}
|
|
48
|
+
.stat.clickable{cursor:pointer;transition:border-color .15s}
|
|
49
|
+
.stat.clickable:hover{border-color:var(--accent)}
|
|
50
|
+
.stat.active{border-color:var(--accent)}
|
|
51
|
+
.stat-val{font-size:2rem;font-weight:700}
|
|
52
|
+
.stat-lbl{color:var(--muted);font-size:.8rem;text-transform:uppercase;letter-spacing:.05em}
|
|
53
|
+
.type-badge{display:inline-block;border-radius:4px;padding:2px 8px;font-size:.75rem;
|
|
54
|
+
font-weight:600;color:#fff}
|
|
55
|
+
.t1{background:var(--t1)}.t2{background:var(--t2)}.t3{background:var(--t3)}
|
|
56
|
+
.type-section{margin-bottom:.5rem}
|
|
57
|
+
.type-section>summary{display:flex;align-items:center;gap:.5rem;list-style:none;
|
|
58
|
+
font-size:1.1rem;font-weight:600;padding:.6rem .5rem .4rem;
|
|
59
|
+
border-bottom:1px solid var(--border);cursor:pointer;
|
|
60
|
+
user-select:none;margin:1.5rem 0 .75rem}
|
|
61
|
+
.type-section>summary::-webkit-details-marker{display:none}
|
|
62
|
+
.section-chevron{color:var(--muted);transition:transform .2s;margin-left:auto;font-style:normal}
|
|
63
|
+
details.type-section[open]>summary .section-chevron{transform:rotate(90deg)}
|
|
64
|
+
.group{background:var(--surface);border:1px solid var(--border);border-radius:8px;
|
|
65
|
+
margin-bottom:1rem;overflow:hidden}
|
|
66
|
+
.group-header{display:flex;align-items:center;gap:.75rem;padding:.75rem 1rem;
|
|
67
|
+
cursor:pointer;user-select:none}
|
|
68
|
+
.group-title{flex:1;font-weight:600}
|
|
69
|
+
.group-meta{color:var(--muted);font-size:.8rem}
|
|
70
|
+
.chevron{color:var(--muted);transition:transform .2s;font-style:normal}
|
|
71
|
+
details.group[open]>summary .chevron{transform:rotate(90deg)}
|
|
72
|
+
.member{border-top:1px solid var(--border);padding:.75rem 1rem}
|
|
73
|
+
.member-meta{color:var(--muted);font-size:.8rem;margin-bottom:.5rem}
|
|
74
|
+
.member-meta strong{color:var(--text)}
|
|
75
|
+
pre{background:#0a0c12;border-radius:6px;padding:.75rem;overflow-x:auto;
|
|
76
|
+
font-size:.8rem;line-height:1.5;max-height:320px}
|
|
77
|
+
.empty{color:var(--muted);font-style:italic;padding:.5rem 0}
|
|
78
|
+
footer{margin-top:2rem;color:var(--muted);font-size:.8rem}
|
|
79
|
+
</style>
|
|
80
|
+
</head>
|
|
81
|
+
<body>
|
|
82
|
+
<h1>CodeEcho</h1>
|
|
83
|
+
<div class="subtitle">Clone Detection Report — {{ generated_at }}</div>
|
|
84
|
+
|
|
85
|
+
<div class="toolbar">
|
|
86
|
+
<button class="btn" onclick="toggleAll(true)">Expand All</button>
|
|
87
|
+
<button class="btn" onclick="toggleAll(false)">Collapse All</button>
|
|
88
|
+
<button id="show-all-btn" class="btn" onclick="filterType(null)">← Show All Types</button>
|
|
89
|
+
</div>
|
|
90
|
+
|
|
91
|
+
<div class="stats">
|
|
92
|
+
<div class="stat">
|
|
93
|
+
<div class="stat-val">{{ result.files_scanned }}</div>
|
|
94
|
+
<div class="stat-lbl">Files Scanned</div>
|
|
95
|
+
</div>
|
|
96
|
+
<div class="stat">
|
|
97
|
+
<div class="stat-val">{{ result.fragments_extracted }}</div>
|
|
98
|
+
<div class="stat-lbl">Fragments</div>
|
|
99
|
+
</div>
|
|
100
|
+
<div class="stat clickable" data-type="1" onclick="filterType(1)">
|
|
101
|
+
<div class="stat-val" style="color:var(--t1)">{{ result.type1_groups }}</div>
|
|
102
|
+
<div class="stat-lbl">Type-1 Groups</div>
|
|
103
|
+
</div>
|
|
104
|
+
<div class="stat clickable" data-type="2" onclick="filterType(2)">
|
|
105
|
+
<div class="stat-val" style="color:var(--t2)">{{ result.type2_groups }}</div>
|
|
106
|
+
<div class="stat-lbl">Type-2 Groups</div>
|
|
107
|
+
</div>
|
|
108
|
+
<div class="stat clickable" data-type="3" onclick="filterType(3)">
|
|
109
|
+
<div class="stat-val" style="color:var(--t3)">{{ result.type3_groups }}</div>
|
|
110
|
+
<div class="stat-lbl">Type-3 Groups</div>
|
|
111
|
+
</div>
|
|
112
|
+
</div>
|
|
113
|
+
|
|
114
|
+
{% for type_num, type_label, css_class in [(1,'Type-1 — Exact Clones','t1'),
|
|
115
|
+
(2,'Type-2 — Structural Clones','t2'),
|
|
116
|
+
(3,'Type-3 — Near Duplicates','t3')] %}
|
|
117
|
+
{% set type_groups = groups_by_type[type_num] %}
|
|
118
|
+
<details class="type-section" data-type="{{ type_num }}" open>
|
|
119
|
+
<summary>
|
|
120
|
+
<span class="type-badge {{ css_class }}">Type-{{ type_num }}</span>
|
|
121
|
+
{{ type_label }}
|
|
122
|
+
<i class="section-chevron">►</i>
|
|
123
|
+
</summary>
|
|
124
|
+
{% if type_groups %}
|
|
125
|
+
{% for entry in type_groups %}
|
|
126
|
+
<details class="group">
|
|
127
|
+
<summary class="group-header">
|
|
128
|
+
<i class="chevron">►</i>
|
|
129
|
+
<span class="group-title">Group {{ loop.index }} — {{ entry.members | length }} members</span>
|
|
130
|
+
{% if entry.group.similarity_score is not none %}
|
|
131
|
+
<span class="group-meta">similarity {{ "%.0f"|format(entry.group.similarity_score * 100) }}%</span>
|
|
132
|
+
{% endif %}
|
|
133
|
+
</summary>
|
|
134
|
+
{% for frag in entry.members %}
|
|
135
|
+
<div class="member">
|
|
136
|
+
<div class="member-meta">
|
|
137
|
+
<strong>{{ frag.language }}</strong> {{ frag.fragment_type }} •
|
|
138
|
+
<strong>{{ frag.file_path }}</strong>
|
|
139
|
+
lines {{ frag.start_line }}–{{ frag.end_line }}
|
|
140
|
+
</div>
|
|
141
|
+
<pre>{{ frag.source_text }}</pre>
|
|
142
|
+
</div>
|
|
143
|
+
{% endfor %}
|
|
144
|
+
</details>
|
|
145
|
+
{% endfor %}
|
|
146
|
+
{% else %}
|
|
147
|
+
<div class="empty">No {{ type_label }} found.</div>
|
|
148
|
+
{% endif %}
|
|
149
|
+
</details>
|
|
150
|
+
{% endfor %}
|
|
151
|
+
|
|
152
|
+
<footer>Scan path: {{ result.scan_path }} • Session: {{ result.session_id }}</footer>
|
|
153
|
+
|
|
154
|
+
<script>
|
|
155
|
+
function toggleAll(open) {
|
|
156
|
+
document.querySelectorAll('details').forEach(function(d) { d.open = open; });
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
function filterType(type) {
|
|
160
|
+
var sections = document.querySelectorAll('.type-section');
|
|
161
|
+
var showAllBtn = document.getElementById('show-all-btn');
|
|
162
|
+
var statCards = document.querySelectorAll('.stat.clickable');
|
|
163
|
+
if (type === null) {
|
|
164
|
+
sections.forEach(function(s) { s.style.display = ''; });
|
|
165
|
+
showAllBtn.style.display = 'none';
|
|
166
|
+
statCards.forEach(function(c) { c.classList.remove('active'); });
|
|
167
|
+
} else {
|
|
168
|
+
sections.forEach(function(s) {
|
|
169
|
+
s.style.display = (parseInt(s.dataset.type) === type) ? '' : 'none';
|
|
170
|
+
});
|
|
171
|
+
showAllBtn.style.display = 'inline-block';
|
|
172
|
+
statCards.forEach(function(c) {
|
|
173
|
+
c.classList.toggle('active', parseInt(c.dataset.type) === type);
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
</script>
|
|
178
|
+
</body>
|
|
179
|
+
</html>"""
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def write(
|
|
183
|
+
session_db: SessionDB,
|
|
184
|
+
result: ScanResult,
|
|
185
|
+
output_path: Path,
|
|
186
|
+
) -> Path:
|
|
187
|
+
"""Render all clone groups for *result.session_id* to an HTML file at *output_path*.
|
|
188
|
+
|
|
189
|
+
:param session_db: Open :class:`~codeecho.db.SessionDB` context.
|
|
190
|
+
:param result: Summary statistics from the scan.
|
|
191
|
+
:param output_path: Destination HTML file path.
|
|
192
|
+
:returns: The resolved path of the written file.
|
|
193
|
+
"""
|
|
194
|
+
groups = session_db.get_clone_groups(result.session_id)
|
|
195
|
+
groups_by_type: dict[int, list[dict]] = {1: [], 2: [], 3: []}
|
|
196
|
+
for group in groups:
|
|
197
|
+
members = session_db.get_fragments_for_group(group)
|
|
198
|
+
entry = {"group": group, "members": members}
|
|
199
|
+
groups_by_type[group.clone_type].append(entry)
|
|
200
|
+
|
|
201
|
+
env = Environment(loader=BaseLoader(), autoescape=True) # type: ignore[call-arg]
|
|
202
|
+
tmpl = env.from_string(_TEMPLATE)
|
|
203
|
+
html = tmpl.render(
|
|
204
|
+
result=result,
|
|
205
|
+
groups_by_type=groups_by_type,
|
|
206
|
+
generated_at=datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC"),
|
|
207
|
+
)
|
|
208
|
+
|
|
209
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
210
|
+
output_path.write_text(html, encoding="utf-8")
|
|
211
|
+
_logger.debug("HTML report written to %s", output_path)
|
|
212
|
+
return output_path.resolve()
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""
|
|
2
|
+
JSON reporter: serialises clone detection results from the session database to a JSON file.
|
|
3
|
+
|
|
4
|
+
:author: Ron Webb
|
|
5
|
+
:since: 1.0.0
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
import logging
|
|
10
|
+
from datetime import datetime, timezone
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from codeecho.db import SessionDB
|
|
15
|
+
from codeecho.models import CloneGroup, Fragment, ScanResult
|
|
16
|
+
|
|
17
|
+
_logger = logging.getLogger("codeecho.reporter.json")
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _fragment_to_dict(frag: Fragment) -> dict[str, Any]:
|
|
21
|
+
return {
|
|
22
|
+
"fragment_id": frag.fragment_id,
|
|
23
|
+
"file": frag.file_path,
|
|
24
|
+
"language": frag.language,
|
|
25
|
+
"fragment_type": frag.fragment_type,
|
|
26
|
+
"start_line": frag.start_line,
|
|
27
|
+
"end_line": frag.end_line,
|
|
28
|
+
"token_count": frag.token_count,
|
|
29
|
+
"source_text": frag.source_text,
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _group_to_dict(group: CloneGroup, members: list[Fragment]) -> dict[str, Any]:
|
|
34
|
+
result: dict[str, Any] = {
|
|
35
|
+
"group_id": group.group_id,
|
|
36
|
+
"clone_type": group.clone_type,
|
|
37
|
+
"representative_hash": group.representative_hash,
|
|
38
|
+
"members": [_fragment_to_dict(f) for f in members],
|
|
39
|
+
}
|
|
40
|
+
if group.similarity_score is not None:
|
|
41
|
+
result["similarity_score"] = round(group.similarity_score, 4)
|
|
42
|
+
return result
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def write(
|
|
46
|
+
session_db: SessionDB,
|
|
47
|
+
result: ScanResult,
|
|
48
|
+
output_path: Path,
|
|
49
|
+
) -> Path:
|
|
50
|
+
"""Serialise all clone groups for *result.session_id* to *output_path*.
|
|
51
|
+
|
|
52
|
+
:param session_db: Open :class:`~codeecho.db.SessionDB` context.
|
|
53
|
+
:param result: Summary statistics from the scan.
|
|
54
|
+
:param output_path: Destination JSON file path.
|
|
55
|
+
:returns: The resolved path of the written file.
|
|
56
|
+
"""
|
|
57
|
+
groups = session_db.get_clone_groups(result.session_id)
|
|
58
|
+
groups_data: list[dict[str, Any]] = []
|
|
59
|
+
for group in groups:
|
|
60
|
+
members = session_db.get_fragments_for_group(group)
|
|
61
|
+
groups_data.append(_group_to_dict(group, members))
|
|
62
|
+
|
|
63
|
+
payload = {
|
|
64
|
+
"generated_at": datetime.now(timezone.utc).isoformat(),
|
|
65
|
+
"session_id": result.session_id,
|
|
66
|
+
"scan_path": result.scan_path,
|
|
67
|
+
"summary": {
|
|
68
|
+
"files_scanned": result.files_scanned,
|
|
69
|
+
"fragments_extracted": result.fragments_extracted,
|
|
70
|
+
"type1_groups": result.type1_groups,
|
|
71
|
+
"type2_groups": result.type2_groups,
|
|
72
|
+
"type3_groups": result.type3_groups,
|
|
73
|
+
},
|
|
74
|
+
"clone_groups": groups_data,
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
78
|
+
output_path.write_text(
|
|
79
|
+
json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8"
|
|
80
|
+
)
|
|
81
|
+
_logger.debug("JSON report written to %s", output_path)
|
|
82
|
+
return output_path.resolve()
|
codeecho/scanner.py
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"""
|
|
2
|
+
File-system scanner that discovers source files for clone detection.
|
|
3
|
+
|
|
4
|
+
:author: Ron Webb
|
|
5
|
+
:since: 1.0.0
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import fnmatch
|
|
9
|
+
import logging
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
from braincraft import IgnoreFile
|
|
13
|
+
|
|
14
|
+
_logger = logging.getLogger("codeecho.scanner")
|
|
15
|
+
|
|
16
|
+
EXTENSION_TO_LANGUAGE: dict[str, str] = {
|
|
17
|
+
".py": "Python",
|
|
18
|
+
".js": "JavaScript",
|
|
19
|
+
".mjs": "JavaScript",
|
|
20
|
+
".cjs": "JavaScript",
|
|
21
|
+
".ts": "TypeScript",
|
|
22
|
+
".tsx": "TypeScript",
|
|
23
|
+
".java": "Java",
|
|
24
|
+
".go": "Go",
|
|
25
|
+
".gs": "Gosu",
|
|
26
|
+
".gsx": "Gosu",
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
_DEFAULT_EXCLUDE_DIRS: frozenset[str] = frozenset(
|
|
30
|
+
{
|
|
31
|
+
".git",
|
|
32
|
+
".svn",
|
|
33
|
+
".hg",
|
|
34
|
+
".venv",
|
|
35
|
+
"venv",
|
|
36
|
+
"env",
|
|
37
|
+
".env",
|
|
38
|
+
"__pycache__",
|
|
39
|
+
"node_modules",
|
|
40
|
+
"build",
|
|
41
|
+
"dist",
|
|
42
|
+
"target",
|
|
43
|
+
"out",
|
|
44
|
+
".tox",
|
|
45
|
+
".pytest_cache",
|
|
46
|
+
"htmlcov",
|
|
47
|
+
".mypy_cache",
|
|
48
|
+
".ruff_cache",
|
|
49
|
+
}
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def scan(
|
|
54
|
+
root: Path,
|
|
55
|
+
exclude_patterns: tuple[str, ...] = (),
|
|
56
|
+
ignore_file: IgnoreFile | None = None,
|
|
57
|
+
) -> list[tuple[Path, str]]:
|
|
58
|
+
"""Walk *root* recursively and return ``(path, language)`` pairs for supported files.
|
|
59
|
+
|
|
60
|
+
:param root: Root directory to scan.
|
|
61
|
+
:param exclude_patterns: Glob patterns (fnmatch-style) whose matching paths are skipped.
|
|
62
|
+
:param ignore_file: Optional gitignore-style :class:`~braincraft.IgnoreFile`; matched paths are skipped.
|
|
63
|
+
:returns: List of ``(absolute_path, language_name)`` tuples.
|
|
64
|
+
"""
|
|
65
|
+
results: list[tuple[Path, str]] = []
|
|
66
|
+
root = root.resolve()
|
|
67
|
+
_logger.debug("Scanning root: %s", root)
|
|
68
|
+
|
|
69
|
+
for entry in _walk(root, exclude_patterns, ignore_file):
|
|
70
|
+
suffix = entry.suffix.lower()
|
|
71
|
+
if language := EXTENSION_TO_LANGUAGE.get(suffix):
|
|
72
|
+
results.append((entry, language))
|
|
73
|
+
|
|
74
|
+
_logger.debug("Found %d source files under %s", len(results), root)
|
|
75
|
+
return results
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _walk(
|
|
79
|
+
root: Path,
|
|
80
|
+
exclude_patterns: tuple[str, ...],
|
|
81
|
+
ignore_file: IgnoreFile | None,
|
|
82
|
+
):
|
|
83
|
+
"""Recursively collect file paths, skipping excluded directories and glob-matched files."""
|
|
84
|
+
try:
|
|
85
|
+
for child in sorted(root.iterdir()):
|
|
86
|
+
if child.is_dir():
|
|
87
|
+
if child.name in _DEFAULT_EXCLUDE_DIRS:
|
|
88
|
+
continue
|
|
89
|
+
if _matches_any(child, exclude_patterns):
|
|
90
|
+
continue
|
|
91
|
+
if ignore_file is not None and ignore_file.is_ignored(child):
|
|
92
|
+
_logger.debug("Ignored (ignore file): %s", child)
|
|
93
|
+
continue
|
|
94
|
+
yield from _walk(child, exclude_patterns, ignore_file)
|
|
95
|
+
elif child.is_file():
|
|
96
|
+
if _matches_any(child, exclude_patterns):
|
|
97
|
+
continue
|
|
98
|
+
if ignore_file is not None and ignore_file.is_ignored(child):
|
|
99
|
+
_logger.debug("Ignored (ignore file): %s", child)
|
|
100
|
+
continue
|
|
101
|
+
yield child
|
|
102
|
+
except PermissionError as exc:
|
|
103
|
+
_logger.warning("Permission denied reading %s: %s", root, exc)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _matches_any(path: Path, patterns: tuple[str, ...]) -> bool:
|
|
107
|
+
"""Return True if *path* matches any of the given fnmatch-style *patterns*."""
|
|
108
|
+
path_str = str(path)
|
|
109
|
+
return any(fnmatch.fnmatch(path_str, p) for p in patterns)
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: codeecho
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: A developer tool that scans your codebase to detect and highlight ECHOES of duplicated or near-duplicated code so you can refactor toward cleaner, more maintainable designs.
|
|
5
|
+
License: MIT License
|
|
6
|
+
|
|
7
|
+
Copyright (c) 2026 Ron Webb
|
|
8
|
+
|
|
9
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
10
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
11
|
+
in the Software without restriction, including without limitation the rights
|
|
12
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
13
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
14
|
+
furnished to do so, subject to the following conditions:
|
|
15
|
+
|
|
16
|
+
The above copyright notice and this permission notice shall be included in all
|
|
17
|
+
copies or substantial portions of the Software.
|
|
18
|
+
|
|
19
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
20
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
21
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
22
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
23
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
24
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
25
|
+
SOFTWARE.
|
|
26
|
+
License-File: LICENSE
|
|
27
|
+
Author: Ron Webb
|
|
28
|
+
Requires-Python: >=3.14
|
|
29
|
+
Classifier: License :: Other/Proprietary License
|
|
30
|
+
Classifier: Programming Language :: Python :: 3
|
|
31
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
32
|
+
Requires-Dist: braincraft (>=1.2.0,<2.0.0)
|
|
33
|
+
Requires-Dist: click (>=8.4.2,<9.0.0)
|
|
34
|
+
Requires-Dist: env-dir-bootstrap (>=1.1.0,<2.0.0)
|
|
35
|
+
Requires-Dist: jinja2 (>=3.1)
|
|
36
|
+
Requires-Dist: logenrich (>=1.0.1,<2.0.0)
|
|
37
|
+
Requires-Dist: rich (>=15.0.0,<16.0.0)
|
|
38
|
+
Requires-Dist: tree-sitter (>=0.26.0,<0.27.0)
|
|
39
|
+
Requires-Dist: tree-sitter-go (>=0.25.0,<0.26.0)
|
|
40
|
+
Requires-Dist: tree-sitter-java (>=0.23.5,<0.24.0)
|
|
41
|
+
Requires-Dist: tree-sitter-javascript (>=0.25.0,<0.26.0)
|
|
42
|
+
Requires-Dist: tree-sitter-python (>=0.25.0,<0.26.0)
|
|
43
|
+
Requires-Dist: tree-sitter-typescript (>=0.23.2,<0.24.0)
|
|
44
|
+
Description-Content-Type: text/markdown
|
|
45
|
+
|
|
46
|
+
# codeecho 1.0.0
|
|
47
|
+
|
|
48
|
+
> A developer tool that scans your codebase to detect and highlight **echoes** of duplicated or near-duplicated code, so you can refactor toward cleaner, more maintainable designs.
|
|
49
|
+
|
|
50
|
+
[](LICENSE)
|
|
51
|
+
[](CHANGELOG.md)
|
|
52
|
+
|
|
53
|
+
## Prerequisites
|
|
54
|
+
|
|
55
|
+
- Python `>=3.14`
|
|
56
|
+
|
|
57
|
+
## Installation
|
|
58
|
+
|
|
59
|
+
```powershell
|
|
60
|
+
pip install codeecho
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
## Usage
|
|
64
|
+
|
|
65
|
+
```powershell
|
|
66
|
+
python -m codeecho <path> [options]
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
### Clone detection levels
|
|
70
|
+
|
|
71
|
+
| Type | Detection strategy | Example |
|
|
72
|
+
|------|--------------------|---------|
|
|
73
|
+
| **Type-1** | Exact copy-paste — identical token sequences | Two functions with the same code copied verbatim |
|
|
74
|
+
| **Type-2** | Structural clone — same structure, renamed identifiers / literals | Same logic with different variable names |
|
|
75
|
+
| **Type-3** | Near-duplicate — high Jaccard similarity on token sets | Nearly identical functions with a few extra lines |
|
|
76
|
+
|
|
77
|
+
### Supported languages
|
|
78
|
+
|
|
79
|
+
| Language | Extensions |
|
|
80
|
+
|----------|-----------|
|
|
81
|
+
| Python | `.py` |
|
|
82
|
+
| JavaScript | `.js`, `.mjs`, `.cjs` |
|
|
83
|
+
| TypeScript | `.ts`, `.tsx` |
|
|
84
|
+
| Java | `.java` |
|
|
85
|
+
| Go | `.go` |
|
|
86
|
+
| Gosu | `.gs`, `.gsx` |
|
|
87
|
+
|
|
88
|
+
### Arguments
|
|
89
|
+
|
|
90
|
+
| Argument | Description |
|
|
91
|
+
|----------|-------------|
|
|
92
|
+
| `path` | Root directory to scan. |
|
|
93
|
+
|
|
94
|
+
### Options
|
|
95
|
+
|
|
96
|
+
| Option | Default | Description |
|
|
97
|
+
|--------|---------|-------------|
|
|
98
|
+
| `--types <types>` | `all` | Clone types to detect: comma-separated (`1`, `2`, `3`) or `all`. |
|
|
99
|
+
| `--threshold <float>` | `0.8` | Jaccard similarity threshold for Type-3 detection (`0.0`–`1.0`). |
|
|
100
|
+
| `--output <name>` | `codeecho-output` | Base name (without extension) for output file(s). |
|
|
101
|
+
| `--output-dir <dir>` | `<cwd>/reports` | Directory where output file(s) will be written. |
|
|
102
|
+
| `--db-dir <dir>` | `~/.codeecho` | Directory for the SQLite scratch database (`codeecho.db`). Session records are removed after the report is written. |
|
|
103
|
+
| `--format <fmt>` | `both` | Output format: `json`, `html`, or `both`. |
|
|
104
|
+
| `--min-tokens <n>` | `10` | Minimum token count for a code fragment to be included. |
|
|
105
|
+
| `--exclude <pattern>` | _(none)_ | Glob pattern(s) to exclude from scanning (repeatable). |
|
|
106
|
+
| `--version` | | Print the version and exit. |
|
|
107
|
+
| `-h, --help` | | Show help and exit. |
|
|
108
|
+
|
|
109
|
+
### Examples
|
|
110
|
+
|
|
111
|
+
Scan the current directory and write both JSON and HTML reports:
|
|
112
|
+
|
|
113
|
+
```powershell
|
|
114
|
+
python -m codeecho .
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
Detect only Type-1 and Type-2 clones in a `src/` tree:
|
|
118
|
+
|
|
119
|
+
```powershell
|
|
120
|
+
python -m codeecho src --types 1,2
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
Scan with a custom output name and directory:
|
|
124
|
+
|
|
125
|
+
```powershell
|
|
126
|
+
python -m codeecho . --output my-scan --output-dir audit/reports
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
Lower the Type-3 threshold to catch more near-duplicates:
|
|
130
|
+
|
|
131
|
+
```powershell
|
|
132
|
+
python -m codeecho . --threshold 0.6
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
Exclude test and vendor directories:
|
|
136
|
+
|
|
137
|
+
```powershell
|
|
138
|
+
python -m codeecho . --exclude "*/tests/*" --exclude "*/vendor/*"
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
## Configuration
|
|
142
|
+
|
|
143
|
+
| Environment variable | Description |
|
|
144
|
+
|----------------------|-------------|
|
|
145
|
+
| `CODEECHO_CONFIG_DIR` | Directory where `logging.ini` and `.ignore` are seeded on first run. When unset, the bundled copies inside the package are used directly. |
|
|
146
|
+
|
|
147
|
+
### `.ignore` file
|
|
148
|
+
|
|
149
|
+
On first run, a `.ignore` file is seeded into `CODEECHO_CONFIG_DIR` (or the package directory when unset). It follows gitignore syntax and is applied during file scanning to exclude paths in addition to any `--exclude` patterns passed on the command line. Edit this file to permanently suppress paths you never want scanned.
|
|
150
|
+
|
|
151
|
+
## Development
|
|
152
|
+
|
|
153
|
+
### Prerequisites
|
|
154
|
+
|
|
155
|
+
- Poetry `2.2+`
|
|
156
|
+
|
|
157
|
+
### Setup
|
|
158
|
+
|
|
159
|
+
```powershell
|
|
160
|
+
poetry install
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
### Run
|
|
164
|
+
|
|
165
|
+
```powershell
|
|
166
|
+
poetry run python -m codeecho <path> [options]
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
### Architecture
|
|
170
|
+
|
|
171
|
+
```mermaid
|
|
172
|
+
flowchart TD
|
|
173
|
+
CLI["__main__.py\n(Click CLI)"] --> ScannerM["scanner.py\nFile discovery"]
|
|
174
|
+
ScannerM --> Parser["parser.py\nTree-sitter parsing"]
|
|
175
|
+
Parser --> Extractor["extractor.py\nFragment extraction\n(functions · classes · files)"]
|
|
176
|
+
Extractor --> Normalizer["normalizer.py\nRegex tokeniser\nType-2 normalisation"]
|
|
177
|
+
Normalizer --> Fingerprint["fingerprint.py\nSHA-256 hashing"]
|
|
178
|
+
Fingerprint --> DB["db.py\nSQLite session store"]
|
|
179
|
+
DB --> Detector["detector.py\nType-1 / 2 hash grouping\nType-3 Jaccard similarity"]
|
|
180
|
+
Detector --> DB
|
|
181
|
+
DB --> JSON["reporter/json_reporter.py\nJSON report"]
|
|
182
|
+
DB --> HTML["reporter/html_reporter.py\nHTML report"]
|
|
183
|
+
DB -->|delete session| Cleanup["Session cleanup"]
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
### Format and Lint
|
|
187
|
+
|
|
188
|
+
```powershell
|
|
189
|
+
poetry run black codeecho
|
|
190
|
+
poetry run pylint codeecho
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
### Run Tests with Coverage
|
|
194
|
+
|
|
195
|
+
```powershell
|
|
196
|
+
poetry run pytest --cov=codeecho tests --cov-report html
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
## [Changelog](CHANGELOG.md)
|
|
200
|
+
|
|
201
|
+
## License
|
|
202
|
+
|
|
203
|
+
This project is licensed under the [MIT License](LICENSE).
|
|
204
|
+
|
|
205
|
+
## Author
|
|
206
|
+
|
|
207
|
+
Ron Webb
|
|
208
|
+
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
codeecho/.ignore,sha256=nFtm2m82jdMWoOE-zN1Blkl5zPzvj-RTp1RBVUX0Rkk,890
|
|
2
|
+
codeecho/__init__.py,sha256=Zn5pJRqT5mi4SKNXA_Om6vsbRrdEMsqutUX1zdy_Y1E,566
|
|
3
|
+
codeecho/__main__.py,sha256=Q8TyKYoVVdM5sRr4S1GTbNGhgVkads4JZKSPpjxxwsM,9904
|
|
4
|
+
codeecho/db.py,sha256=kYKWzgK43vVus9Yd7GU5Pq1Gpyzm-Cv-EePtz-t8uHc,10846
|
|
5
|
+
codeecho/detector.py,sha256=SXVWizPkF4vioW1YLauT5X4vCX7aeNyRUlH0e7nQ9V0,9189
|
|
6
|
+
codeecho/extractor.py,sha256=Q1xAmm2uxaCl79JrMKRLbMv_8he91ux4l3XzrBiHC3E,5942
|
|
7
|
+
codeecho/fingerprint.py,sha256=1XiUbOu_o-f2BmIgKtoY7R82MPvRlP6h3_Mhjcykwgo,1153
|
|
8
|
+
codeecho/logging.ini,sha256=RS__BOZiTPN8NPq_uXsAKiSnoieotWk0eUm12hYyR5k,532
|
|
9
|
+
codeecho/models.py,sha256=9jF8iv8nr27gLvLU5xejqL5VHaCp5Hu0yiN3Jbhjkhk,1372
|
|
10
|
+
codeecho/normalizer.py,sha256=2LkqTVHQ5pIVELd4U-BUwZInS_0W_A30n4PrWjYoNR4,14287
|
|
11
|
+
codeecho/parser.py,sha256=FMIpdNuYZ1KNfXknmNIgpaB8vBjJZcJqmnfSma8bbH4,3521
|
|
12
|
+
codeecho/reporter/__init__.py,sha256=aKypjtT_gmODQrdTQ8vPqiKAN117Ge9AgdR63WbggRg,134
|
|
13
|
+
codeecho/reporter/html_reporter.py,sha256=tuOK2GzAJ7EePhqN37XEiXkucLf2ovx1HdGXiNZSuZ0,9222
|
|
14
|
+
codeecho/reporter/json_reporter.py,sha256=d-xyQBmXWSsrtHK1FQhpa3s1hhFKfVRC7bZ2pmrV814,2731
|
|
15
|
+
codeecho/scanner.py,sha256=NnU_hWwgX5Ih75dpnlkXo8keTEAWJBsMY-ZNAIo0m-s,3363
|
|
16
|
+
codeecho-1.0.0.dist-info/entry_points.txt,sha256=xFm7C6gbwIifpmqvbD9yxlFEt6yhKYOQUs-THG3lzGU,51
|
|
17
|
+
codeecho-1.0.0.dist-info/licenses/LICENSE,sha256=2cmui6TBSfJF5E2Qr0xNS8x4ZY0grz8qLEBEW1CgpjA,1086
|
|
18
|
+
codeecho-1.0.0.dist-info/METADATA,sha256=lk33QRK9aOC-G6dtlT7jSsop-l8A0jQOLoS2HQbKhds,7060
|
|
19
|
+
codeecho-1.0.0.dist-info/WHEEL,sha256=Vz2fHgx6HFtSwhs8KvkHLqH5Ea4w1_rner5uNVGCeIE,88
|
|
20
|
+
codeecho-1.0.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ron Webb
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|