codeecho 1.1.0__tar.gz → 1.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codeecho-1.2.0/CHANGELOG.md +41 -0
- {codeecho-1.1.0 → codeecho-1.2.0}/PKG-INFO +32 -8
- {codeecho-1.1.0 → codeecho-1.2.0}/README.md +31 -7
- {codeecho-1.1.0 → codeecho-1.2.0}/codeecho/__init__.py +6 -2
- {codeecho-1.1.0 → codeecho-1.2.0}/codeecho/__main__.py +235 -41
- codeecho-1.2.0/codeecho/basis.py +101 -0
- codeecho-1.2.0/codeecho/config.ini +2 -0
- codeecho-1.2.0/codeecho/config.py +41 -0
- {codeecho-1.1.0 → codeecho-1.2.0}/codeecho/db.py +42 -11
- {codeecho-1.1.0 → codeecho-1.2.0}/codeecho/detector.py +44 -10
- {codeecho-1.1.0 → codeecho-1.2.0}/codeecho/extractor.py +5 -1
- {codeecho-1.1.0 → codeecho-1.2.0}/codeecho/fingerprint.py +9 -2
- {codeecho-1.1.0 → codeecho-1.2.0}/codeecho/models.py +4 -0
- {codeecho-1.1.0 → codeecho-1.2.0}/codeecho/normalizer.py +30 -6
- {codeecho-1.1.0 → codeecho-1.2.0}/codeecho/parser.py +11 -2
- {codeecho-1.1.0 → codeecho-1.2.0}/codeecho/reporter/html_reporter.py +42 -10
- {codeecho-1.1.0 → codeecho-1.2.0}/codeecho/reporter/json_reporter.py +45 -10
- {codeecho-1.1.0 → codeecho-1.2.0}/codeecho/scanner.py +9 -2
- {codeecho-1.1.0 → codeecho-1.2.0}/pyproject.toml +2 -2
- codeecho-1.1.0/CHANGELOG.md +0 -25
- {codeecho-1.1.0 → codeecho-1.2.0}/LICENSE +0 -0
- {codeecho-1.1.0 → codeecho-1.2.0}/codeecho/.ignore +0 -0
- {codeecho-1.1.0 → codeecho-1.2.0}/codeecho/logging.ini +0 -0
- {codeecho-1.1.0 → codeecho-1.2.0}/codeecho/reporter/__init__.py +0 -0
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 1.2.0 - 2026-09-18
|
|
4
|
+
|
|
5
|
+
### Added
|
|
6
|
+
|
|
7
|
+
- `config.ini` (seeded into `CODEECHO_CONFIG_DIR`) with an `[override]` section that lets you rename
|
|
8
|
+
the `.ignore` file via the `ignore-file` key. Falls back to the bundled `.ignore` when the
|
|
9
|
+
configured file is missing.
|
|
10
|
+
- `--target-list` flag: treat `PATH` as a single file listing scan targets, one per line (blank
|
|
11
|
+
lines and `#`-prefixed comments are skipped), instead of passing `PATH` arguments directly.
|
|
12
|
+
- `--basis` flag: file listing basis target paths, one per line — same format as
|
|
13
|
+
`--target-list`. Absolute file/directory entries are merged into the scan automatically
|
|
14
|
+
and matched exactly; relative entries (e.g. a bare filename) are matched by filename/suffix
|
|
15
|
+
against any file discovered in the scan. The report is filtered to only clone groups
|
|
16
|
+
touching at least one basis file, and groups duplicated purely among basis files are
|
|
17
|
+
flagged (`basis_internal` in JSON, a "Basis-to-Basis" badge in HTML, `is_basis` per member).
|
|
18
|
+
|
|
19
|
+
## 1.1.0 - 2026-08-15
|
|
20
|
+
|
|
21
|
+
### Added
|
|
22
|
+
|
|
23
|
+
- Gosu (`.gs`, `.gsx`) is now parsed with a dedicated [tree-sitter-gosu](https://github.com/rcw3bb/tree-sitter-gosu)
|
|
24
|
+
grammar instead of the Java fallback, enabling accurate fragment extraction using Gosu-native node types
|
|
25
|
+
(`compilation_unit`, `function_declaration`, `constructor_declaration`, `class_declaration`).
|
|
26
|
+
|
|
27
|
+
## 1.0.1 - 2026-07-10
|
|
28
|
+
|
|
29
|
+
### Fixed
|
|
30
|
+
|
|
31
|
+
- Type-3 clone detection no longer flags fragments as clones when one is nested inside the other
|
|
32
|
+
within the same file (e.g. a `function` inside its enclosing `class`).
|
|
33
|
+
- Type-3 clone groups now drop outer/container fragments (e.g. a whole-`file` fragment) when a
|
|
34
|
+
more specific nested fragment from the same file is already a member of that group, preventing
|
|
35
|
+
false positives caused by union-find transitivity.
|
|
36
|
+
|
|
37
|
+
## 1.0.0 - 2026-07-10
|
|
38
|
+
|
|
39
|
+
### Added
|
|
40
|
+
|
|
41
|
+
- Initial release of codeecho.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: codeecho
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.2.0
|
|
4
4
|
Summary: A developer tool that scans your codebase to detect and highlight ECHOES of duplicated or near-duplicated code so you can refactor toward cleaner, more maintainable designs.
|
|
5
5
|
License: MIT License
|
|
6
6
|
|
|
@@ -44,12 +44,11 @@ Requires-Dist: tree-sitter-python (>=0.25.0,<0.26.0)
|
|
|
44
44
|
Requires-Dist: tree-sitter-typescript (>=0.23.2,<0.24.0)
|
|
45
45
|
Description-Content-Type: text/markdown
|
|
46
46
|
|
|
47
|
-
# codeecho 1.
|
|
47
|
+
# codeecho 1.2.0
|
|
48
48
|
|
|
49
|
-
|
|
49
|
+
[](https://github.com/rcw3bb/codeecho/blob/main/LICENSE) [](https://github.com/rcw3bb/codeecho/blob/main/CHANGELOG.md) [](https://www.python.org/) [](https://pypi.org/project/codeecho/)
|
|
50
50
|
|
|
51
|
-
|
|
52
|
-
[](CHANGELOG.md)
|
|
51
|
+
> A developer tool that scans your codebase to detect and highlight **echoes** of duplicated or near-duplicated code, so you can refactor toward cleaner, more maintainable designs.
|
|
53
52
|
|
|
54
53
|
## Prerequisites
|
|
55
54
|
|
|
@@ -104,6 +103,8 @@ python -m codeecho <path> [options]
|
|
|
104
103
|
| `--format <fmt>` | `both` | Output format: `json`, `html`, or `both`. |
|
|
105
104
|
| `--min-tokens <n>` | `10` | Minimum token count for a code fragment to be included. |
|
|
106
105
|
| `--exclude <pattern>` | _(none)_ | Glob pattern(s) to exclude from scanning (repeatable). |
|
|
106
|
+
| `--target-list` | `false` | Treat `PATH` as a single existing file listing target paths (files and/or directories), one per line, instead of individual `PATH` arguments. Blank lines and lines starting with `#` are skipped. |
|
|
107
|
+
| `--basis <file>` | _(none)_ | File listing basis target paths, one per line — same format as `--target-list`. Absolute file/directory entries are merged into the scan automatically and matched exactly; relative entries (e.g. a bare filename like `Foo.gs`) are matched by filename/suffix against any file discovered in the scan. The report is filtered to only clone groups touching at least one basis file, and groups duplicated purely among basis files are flagged (`basis_internal` in JSON, a "Basis-to-Basis" badge in HTML). |
|
|
107
108
|
| `--version` | | Print the version and exit. |
|
|
108
109
|
| `-h, --help` | | Show help and exit. |
|
|
109
110
|
|
|
@@ -139,16 +140,39 @@ Exclude test and vendor directories:
|
|
|
139
140
|
python -m codeecho . --exclude "*/tests/*" --exclude "*/vendor/*"
|
|
140
141
|
```
|
|
141
142
|
|
|
143
|
+
Scan targets listed in a file, one path per line:
|
|
144
|
+
|
|
145
|
+
```powershell
|
|
146
|
+
python -m codeecho targets.txt --target-list
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
Find where code from a reference/basis set of files is duplicated elsewhere in the codebase:
|
|
150
|
+
|
|
151
|
+
```powershell
|
|
152
|
+
python -m codeecho src --basis basis.txt
|
|
153
|
+
```
|
|
154
|
+
|
|
142
155
|
## Configuration
|
|
143
156
|
|
|
144
157
|
| Environment variable | Description |
|
|
145
158
|
|----------------------|-------------|
|
|
146
|
-
| `CODEECHO_CONFIG_DIR` | Directory where `logging.ini
|
|
159
|
+
| `CODEECHO_CONFIG_DIR` | Directory where `logging.ini`, `.ignore`, and `config.ini` are seeded on first run. When unset, the bundled copies inside the package are used directly. |
|
|
147
160
|
|
|
148
161
|
### `.ignore` file
|
|
149
162
|
|
|
150
163
|
On first run, a `.ignore` file is seeded into `CODEECHO_CONFIG_DIR` (or the package directory when unset). It follows gitignore syntax and is applied during file scanning to exclude paths in addition to any `--exclude` patterns passed on the command line. Edit this file to permanently suppress paths you never want scanned.
|
|
151
164
|
|
|
165
|
+
### Overriding the ignore filename
|
|
166
|
+
|
|
167
|
+
`config.ini` (also seeded into `CODEECHO_CONFIG_DIR` on first run) contains an `[override]` section with an `ignore-file` key, which names the file used in place of `.ignore`, resolved relative to `CODEECHO_CONFIG_DIR`:
|
|
168
|
+
|
|
169
|
+
```ini
|
|
170
|
+
[override]
|
|
171
|
+
ignore-file = .ignore
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
Point `ignore-file` at a different filename to use an alternate ignore file (also placed inside `CODEECHO_CONFIG_DIR`). If the configured file is missing, codeecho logs a warning and falls back to the bundled `.ignore`.
|
|
175
|
+
|
|
152
176
|
## Development
|
|
153
177
|
|
|
154
178
|
### Prerequisites
|
|
@@ -197,11 +221,11 @@ poetry run pylint codeecho
|
|
|
197
221
|
poetry run pytest --cov=codeecho tests --cov-report html
|
|
198
222
|
```
|
|
199
223
|
|
|
200
|
-
## [Changelog](CHANGELOG.md)
|
|
224
|
+
## [Changelog](https://github.com/rcw3bb/codeecho/blob/main/CHANGELOG.md)
|
|
201
225
|
|
|
202
226
|
## License
|
|
203
227
|
|
|
204
|
-
This project is licensed under the [MIT License](LICENSE).
|
|
228
|
+
This project is licensed under the [MIT License](https://github.com/rcw3bb/codeecho/blob/main/LICENSE).
|
|
205
229
|
|
|
206
230
|
## Author
|
|
207
231
|
|
|
@@ -1,9 +1,8 @@
|
|
|
1
|
-
# codeecho 1.
|
|
1
|
+
# codeecho 1.2.0
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
[](https://github.com/rcw3bb/codeecho/blob/main/LICENSE) [](https://github.com/rcw3bb/codeecho/blob/main/CHANGELOG.md) [](https://www.python.org/) [](https://pypi.org/project/codeecho/)
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
[](CHANGELOG.md)
|
|
5
|
+
> A developer tool that scans your codebase to detect and highlight **echoes** of duplicated or near-duplicated code, so you can refactor toward cleaner, more maintainable designs.
|
|
7
6
|
|
|
8
7
|
## Prerequisites
|
|
9
8
|
|
|
@@ -58,6 +57,8 @@ python -m codeecho <path> [options]
|
|
|
58
57
|
| `--format <fmt>` | `both` | Output format: `json`, `html`, or `both`. |
|
|
59
58
|
| `--min-tokens <n>` | `10` | Minimum token count for a code fragment to be included. |
|
|
60
59
|
| `--exclude <pattern>` | _(none)_ | Glob pattern(s) to exclude from scanning (repeatable). |
|
|
60
|
+
| `--target-list` | `false` | Treat `PATH` as a single existing file listing target paths (files and/or directories), one per line, instead of individual `PATH` arguments. Blank lines and lines starting with `#` are skipped. |
|
|
61
|
+
| `--basis <file>` | _(none)_ | File listing basis target paths, one per line — same format as `--target-list`. Absolute file/directory entries are merged into the scan automatically and matched exactly; relative entries (e.g. a bare filename like `Foo.gs`) are matched by filename/suffix against any file discovered in the scan. The report is filtered to only clone groups touching at least one basis file, and groups duplicated purely among basis files are flagged (`basis_internal` in JSON, a "Basis-to-Basis" badge in HTML). |
|
|
61
62
|
| `--version` | | Print the version and exit. |
|
|
62
63
|
| `-h, --help` | | Show help and exit. |
|
|
63
64
|
|
|
@@ -93,16 +94,39 @@ Exclude test and vendor directories:
|
|
|
93
94
|
python -m codeecho . --exclude "*/tests/*" --exclude "*/vendor/*"
|
|
94
95
|
```
|
|
95
96
|
|
|
97
|
+
Scan targets listed in a file, one path per line:
|
|
98
|
+
|
|
99
|
+
```powershell
|
|
100
|
+
python -m codeecho targets.txt --target-list
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
Find where code from a reference/basis set of files is duplicated elsewhere in the codebase:
|
|
104
|
+
|
|
105
|
+
```powershell
|
|
106
|
+
python -m codeecho src --basis basis.txt
|
|
107
|
+
```
|
|
108
|
+
|
|
96
109
|
## Configuration
|
|
97
110
|
|
|
98
111
|
| Environment variable | Description |
|
|
99
112
|
|----------------------|-------------|
|
|
100
|
-
| `CODEECHO_CONFIG_DIR` | Directory where `logging.ini
|
|
113
|
+
| `CODEECHO_CONFIG_DIR` | Directory where `logging.ini`, `.ignore`, and `config.ini` are seeded on first run. When unset, the bundled copies inside the package are used directly. |
|
|
101
114
|
|
|
102
115
|
### `.ignore` file
|
|
103
116
|
|
|
104
117
|
On first run, a `.ignore` file is seeded into `CODEECHO_CONFIG_DIR` (or the package directory when unset). It follows gitignore syntax and is applied during file scanning to exclude paths in addition to any `--exclude` patterns passed on the command line. Edit this file to permanently suppress paths you never want scanned.
|
|
105
118
|
|
|
119
|
+
### Overriding the ignore filename
|
|
120
|
+
|
|
121
|
+
`config.ini` (also seeded into `CODEECHO_CONFIG_DIR` on first run) contains an `[override]` section with an `ignore-file` key, which names the file used in place of `.ignore`, resolved relative to `CODEECHO_CONFIG_DIR`:
|
|
122
|
+
|
|
123
|
+
```ini
|
|
124
|
+
[override]
|
|
125
|
+
ignore-file = .ignore
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
Point `ignore-file` at a different filename to use an alternate ignore file (also placed inside `CODEECHO_CONFIG_DIR`). If the configured file is missing, codeecho logs a warning and falls back to the bundled `.ignore`.
|
|
129
|
+
|
|
106
130
|
## Development
|
|
107
131
|
|
|
108
132
|
### Prerequisites
|
|
@@ -151,11 +175,11 @@ poetry run pylint codeecho
|
|
|
151
175
|
poetry run pytest --cov=codeecho tests --cov-report html
|
|
152
176
|
```
|
|
153
177
|
|
|
154
|
-
## [Changelog](CHANGELOG.md)
|
|
178
|
+
## [Changelog](https://github.com/rcw3bb/codeecho/blob/main/CHANGELOG.md)
|
|
155
179
|
|
|
156
180
|
## License
|
|
157
181
|
|
|
158
|
-
This project is licensed under the [MIT License](LICENSE).
|
|
182
|
+
This project is licensed under the [MIT License](https://github.com/rcw3bb/codeecho/blob/main/LICENSE).
|
|
159
183
|
|
|
160
184
|
## Author
|
|
161
185
|
|
|
@@ -2,21 +2,25 @@
|
|
|
2
2
|
codeecho - A developer tool that scans your codebase to detect and highlight ECHOES
|
|
3
3
|
of duplicated or near-duplicated code so you can refactor toward cleaner, more
|
|
4
4
|
maintainable designs.
|
|
5
|
+
|
|
6
|
+
:author: Ron Webb
|
|
7
|
+
:since: 1.0.0
|
|
5
8
|
"""
|
|
6
9
|
|
|
7
10
|
from env_dir_bootstrap import EnvDirBootstrap
|
|
8
11
|
from logenrich import setup_logger
|
|
9
12
|
|
|
10
|
-
__version__ = "1.
|
|
13
|
+
__version__ = "1.2.0"
|
|
11
14
|
|
|
12
15
|
_bootstrapper = EnvDirBootstrap(
|
|
13
16
|
env_var="CODEECHO_CONFIG_DIR",
|
|
14
|
-
resources=["logging.ini", ".ignore"],
|
|
17
|
+
resources=["logging.ini", ".ignore", "config.ini"],
|
|
15
18
|
package="codeecho",
|
|
16
19
|
)
|
|
17
20
|
|
|
18
21
|
_bootstrapper.setup()
|
|
19
22
|
|
|
20
23
|
CONF_DIR = str(_bootstrapper.get_dir())
|
|
24
|
+
DEFAULT_IGNORE_PATH = _bootstrapper.resolve(".ignore")
|
|
21
25
|
|
|
22
26
|
setup_logger("codeecho", conf_dir=CONF_DIR)
|
|
@@ -4,9 +4,13 @@ CLI entry point for codeecho.
|
|
|
4
4
|
Invoked via::
|
|
5
5
|
|
|
6
6
|
poetry run python -m codeecho [OPTIONS] PATH
|
|
7
|
+
|
|
8
|
+
:author: Ron Webb
|
|
9
|
+
:since: 1.0.0
|
|
7
10
|
"""
|
|
8
11
|
|
|
9
12
|
import json
|
|
13
|
+
import logging
|
|
10
14
|
import os
|
|
11
15
|
import uuid
|
|
12
16
|
from pathlib import Path
|
|
@@ -24,20 +28,27 @@ from rich.progress import (
|
|
|
24
28
|
)
|
|
25
29
|
from rich.table import Table
|
|
26
30
|
|
|
27
|
-
from
|
|
28
|
-
from
|
|
29
|
-
from
|
|
30
|
-
from
|
|
31
|
-
from
|
|
32
|
-
from
|
|
31
|
+
from . import __version__, CONF_DIR, DEFAULT_IGNORE_PATH
|
|
32
|
+
from . import basis as basis_module
|
|
33
|
+
from . import extractor, fingerprint, parser as ts_parser, scanner
|
|
34
|
+
from .config import Config
|
|
35
|
+
from .db import SessionDB, get_db_path
|
|
36
|
+
from .detector import detect
|
|
37
|
+
from .models import ScanResult
|
|
38
|
+
from .reporter import html_reporter, json_reporter
|
|
33
39
|
|
|
34
40
|
_console = Console()
|
|
41
|
+
_config = Config()
|
|
42
|
+
_logger = logging.getLogger("codeecho.__main__")
|
|
35
43
|
|
|
36
44
|
_FORMAT_CHOICES = click.Choice(["json", "html", "both"])
|
|
37
45
|
|
|
38
46
|
|
|
39
47
|
def _parse_types(types_str: str) -> set[int]:
|
|
40
|
-
"""Parse ``--types`` value into a set of integers.
|
|
48
|
+
"""Parse ``--types`` value into a set of integers.
|
|
49
|
+
|
|
50
|
+
:since: 1.0.0
|
|
51
|
+
"""
|
|
41
52
|
if types_str.strip().lower() == "all":
|
|
42
53
|
return {1, 2, 3}
|
|
43
54
|
result: set[int] = set()
|
|
@@ -52,6 +63,120 @@ def _parse_types(types_str: str) -> set[int]:
|
|
|
52
63
|
return result
|
|
53
64
|
|
|
54
65
|
|
|
66
|
+
def _read_target_list(list_file: Path) -> list[Path]:
|
|
67
|
+
"""Read one target path per line from *list_file*.
|
|
68
|
+
|
|
69
|
+
Blank lines and lines starting with ``#`` are skipped.
|
|
70
|
+
|
|
71
|
+
:since: 1.2.0
|
|
72
|
+
"""
|
|
73
|
+
result: list[Path] = []
|
|
74
|
+
for line in list_file.read_text(encoding="utf-8").splitlines():
|
|
75
|
+
stripped = line.strip()
|
|
76
|
+
if stripped and not stripped.startswith("#"):
|
|
77
|
+
result.append(Path(stripped))
|
|
78
|
+
return result
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _load_ignore_file(ignore_path: Path, base_dir: Path) -> IgnoreFile | None:
|
|
82
|
+
"""Load an :class:`IgnoreFile` from *ignore_path*, logging failures.
|
|
83
|
+
|
|
84
|
+
Returns ``None`` on a missing or non-UTF-8 file instead of raising.
|
|
85
|
+
|
|
86
|
+
:since: 1.2.0
|
|
87
|
+
"""
|
|
88
|
+
try:
|
|
89
|
+
return IgnoreFile(ignore_path, base_dir=base_dir)
|
|
90
|
+
except FileNotFoundError:
|
|
91
|
+
_logger.warning("Ignore file not found at %s", ignore_path)
|
|
92
|
+
return None
|
|
93
|
+
except UnicodeDecodeError as exc:
|
|
94
|
+
_logger.warning("Ignore file at %s is not valid UTF-8: %s", ignore_path, exc)
|
|
95
|
+
return None
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _build_ignore(base_dir: Path) -> IgnoreFile | None:
|
|
99
|
+
"""Build a path-ignore matcher anchored at *base_dir*.
|
|
100
|
+
|
|
101
|
+
The ignore filename is resolved from ``config.ini``'s ``[override]
|
|
102
|
+
ignore-file`` setting under :data:`CONF_DIR`; when that file is missing,
|
|
103
|
+
the bundled default ``.ignore`` is tried instead.
|
|
104
|
+
|
|
105
|
+
:since: 1.2.0
|
|
106
|
+
"""
|
|
107
|
+
custom_path = Path(CONF_DIR) / _config.get_ignore_file()
|
|
108
|
+
ignore = _load_ignore_file(custom_path, base_dir)
|
|
109
|
+
if ignore is not None:
|
|
110
|
+
return ignore
|
|
111
|
+
if custom_path == Path(DEFAULT_IGNORE_PATH):
|
|
112
|
+
return None
|
|
113
|
+
return _load_ignore_file(Path(DEFAULT_IGNORE_PATH), base_dir)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _resolve_target_paths(
|
|
117
|
+
paths: tuple[Path, ...], target_list: bool
|
|
118
|
+
) -> tuple[Path, ...]:
|
|
119
|
+
"""Validate *paths* and expand ``--target-list`` into concrete target paths.
|
|
120
|
+
|
|
121
|
+
:since: 1.2.0
|
|
122
|
+
"""
|
|
123
|
+
if not paths:
|
|
124
|
+
raise click.UsageError("At least one PATH is required.")
|
|
125
|
+
|
|
126
|
+
if not target_list:
|
|
127
|
+
return paths
|
|
128
|
+
|
|
129
|
+
if len(paths) != 1 or not paths[0].is_file():
|
|
130
|
+
raise click.UsageError(
|
|
131
|
+
"--target-list requires PATH to be a single existing file."
|
|
132
|
+
)
|
|
133
|
+
expanded = tuple(_read_target_list(paths[0]))
|
|
134
|
+
if not expanded:
|
|
135
|
+
raise click.UsageError("--target-list file contains no target paths.")
|
|
136
|
+
return expanded
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _read_basis_targets(basis_path: Path | None) -> list[Path]:
|
|
140
|
+
"""Read the basis path(s) listed in *basis_path*, if given.
|
|
141
|
+
|
|
142
|
+
Absolute entries are resolved (normalising case/symlinks); relative entries
|
|
143
|
+
(e.g. a bare filename) are left as-is so they can be matched by filename/suffix
|
|
144
|
+
against the scanned files instead of being resolved against the current
|
|
145
|
+
working directory.
|
|
146
|
+
|
|
147
|
+
:since: 1.2.0
|
|
148
|
+
"""
|
|
149
|
+
if basis_path is None:
|
|
150
|
+
return []
|
|
151
|
+
targets = [
|
|
152
|
+
p.resolve() if p.is_absolute() else p for p in _read_target_list(basis_path)
|
|
153
|
+
]
|
|
154
|
+
if not targets:
|
|
155
|
+
raise click.UsageError("--basis file contains no target paths.")
|
|
156
|
+
return targets
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _discover_files(
|
|
160
|
+
resolved: list[Path], exclude: tuple[str, ...]
|
|
161
|
+
) -> list[tuple[Path, str]]:
|
|
162
|
+
"""Print scan targets and return the discovered ``(file, language)`` pairs.
|
|
163
|
+
|
|
164
|
+
:since: 1.2.0
|
|
165
|
+
"""
|
|
166
|
+
for target_path in resolved:
|
|
167
|
+
_console.print(f"[dim]Scanning:[/dim] [bold]{target_path}[/bold]")
|
|
168
|
+
try:
|
|
169
|
+
base_dir = Path(os.path.commonpath(resolved))
|
|
170
|
+
if not base_dir.is_dir():
|
|
171
|
+
base_dir = base_dir.parent
|
|
172
|
+
except ValueError:
|
|
173
|
+
base_dir = Path.cwd()
|
|
174
|
+
ignore_file = _build_ignore(base_dir)
|
|
175
|
+
return scanner.scan(
|
|
176
|
+
tuple(resolved), exclude_patterns=exclude, ignore_file=ignore_file
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
|
|
55
180
|
@click.command(
|
|
56
181
|
name="codeecho", context_settings={"help_option_names": ["-h", "--help"]}
|
|
57
182
|
)
|
|
@@ -120,6 +245,33 @@ def _parse_types(types_str: str) -> set[int]:
|
|
|
120
245
|
metavar="PATTERN",
|
|
121
246
|
help="Glob pattern(s) to exclude from scanning (repeatable).",
|
|
122
247
|
)
|
|
248
|
+
@click.option(
|
|
249
|
+
"--target-list",
|
|
250
|
+
"target_list",
|
|
251
|
+
is_flag=True,
|
|
252
|
+
default=False,
|
|
253
|
+
help=(
|
|
254
|
+
"Treat PATH as a single existing file listing target paths (files "
|
|
255
|
+
"and/or directories), one per line, instead of individual PATH "
|
|
256
|
+
"arguments. Blank lines and lines starting with '#' are skipped."
|
|
257
|
+
),
|
|
258
|
+
)
|
|
259
|
+
@click.option(
|
|
260
|
+
"--basis",
|
|
261
|
+
"basis_path",
|
|
262
|
+
default=None,
|
|
263
|
+
type=click.Path(exists=True, dir_okay=False, path_type=Path),
|
|
264
|
+
metavar="FILE",
|
|
265
|
+
help=(
|
|
266
|
+
"File listing basis target paths, one per line, same format as "
|
|
267
|
+
"--target-list. Absolute file/directory entries are added to the scan "
|
|
268
|
+
"automatically and matched exactly; relative entries (e.g. a bare "
|
|
269
|
+
"filename) are matched by filename/suffix against any file discovered "
|
|
270
|
+
"in the scan. The report is filtered to only clone groups that touch "
|
|
271
|
+
"at least one basis file; groups duplicated purely among basis files "
|
|
272
|
+
"are flagged as such."
|
|
273
|
+
),
|
|
274
|
+
)
|
|
123
275
|
def main( # pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals,invalid-name
|
|
124
276
|
paths: tuple[Path, ...],
|
|
125
277
|
types: str,
|
|
@@ -130,11 +282,15 @@ def main( # pylint: disable=too-many-arguments,too-many-positional-arguments,to
|
|
|
130
282
|
fmt: str,
|
|
131
283
|
min_tokens: int,
|
|
132
284
|
exclude: tuple[str, ...],
|
|
285
|
+
target_list: bool,
|
|
286
|
+
basis_path: Path | None,
|
|
133
287
|
) -> None:
|
|
134
288
|
"""Scan one or more PATH(s) for duplicate and near-duplicate code.
|
|
135
289
|
|
|
136
290
|
Each PATH may be a file or a directory. Generates JSON and/or HTML reports,
|
|
137
291
|
then removes the intermediate session data from the embedded database.
|
|
292
|
+
|
|
293
|
+
:since: 1.0.0
|
|
138
294
|
"""
|
|
139
295
|
_console.print(
|
|
140
296
|
Panel(
|
|
@@ -143,23 +299,26 @@ def main( # pylint: disable=too-many-arguments,too-many-positional-arguments,to
|
|
|
143
299
|
)
|
|
144
300
|
)
|
|
145
301
|
|
|
146
|
-
|
|
147
|
-
raise click.UsageError("At least one PATH is required.")
|
|
148
|
-
|
|
302
|
+
paths = _resolve_target_paths(paths, target_list)
|
|
149
303
|
detect_types = _parse_types(types)
|
|
150
304
|
if output_dir is None:
|
|
151
305
|
output_dir = Path.cwd() / "reports"
|
|
306
|
+
output_dir.mkdir(parents=True, exist_ok=True)
|
|
307
|
+
|
|
308
|
+
basis_targets = _read_basis_targets(basis_path)
|
|
309
|
+
|
|
152
310
|
session_id = str(uuid.uuid4())
|
|
153
311
|
config = {
|
|
154
312
|
"types": types,
|
|
155
313
|
"threshold": threshold,
|
|
156
314
|
"min_tokens": min_tokens,
|
|
157
315
|
"exclude": list(exclude),
|
|
316
|
+
"basis": [str(p) for p in basis_targets],
|
|
158
317
|
}
|
|
159
|
-
|
|
160
|
-
output_dir.mkdir(parents=True, exist_ok=True)
|
|
161
|
-
|
|
162
318
|
resolved = [p.resolve() for p in paths]
|
|
319
|
+
for target in basis_targets:
|
|
320
|
+
if target.is_absolute() and target not in resolved:
|
|
321
|
+
resolved.append(target)
|
|
163
322
|
|
|
164
323
|
db_path = Path(get_db_path(str(db_dir) if db_dir else None))
|
|
165
324
|
with SessionDB(db_path=db_path) as session_db:
|
|
@@ -167,34 +326,37 @@ def main( # pylint: disable=too-many-arguments,too-many-positional-arguments,to
|
|
|
167
326
|
session_id, json.dumps([str(p) for p in resolved]), config
|
|
168
327
|
)
|
|
169
328
|
|
|
170
|
-
|
|
171
|
-
for p in resolved:
|
|
172
|
-
_console.print(f"[dim]Scanning:[/dim] [bold]{p}[/bold]")
|
|
173
|
-
try:
|
|
174
|
-
base_dir = Path(os.path.commonpath(resolved))
|
|
175
|
-
if not base_dir.is_dir():
|
|
176
|
-
base_dir = base_dir.parent
|
|
177
|
-
except ValueError:
|
|
178
|
-
base_dir = Path.cwd()
|
|
179
|
-
_ignore_file = IgnoreFile(Path(CONF_DIR) / ".ignore", base_dir=base_dir)
|
|
180
|
-
files = scanner.scan(
|
|
181
|
-
tuple(resolved), exclude_patterns=exclude, ignore_file=_ignore_file
|
|
182
|
-
)
|
|
183
|
-
total_fragments = 0
|
|
329
|
+
files = _discover_files(resolved, exclude)
|
|
184
330
|
|
|
185
331
|
if not files:
|
|
186
332
|
_console.print("[yellow]No supported source files found.[/yellow]")
|
|
187
333
|
session_db.delete_session(session_id)
|
|
188
334
|
return
|
|
189
335
|
|
|
190
|
-
|
|
336
|
+
basis_files = None
|
|
337
|
+
if basis_targets:
|
|
338
|
+
basis_files = basis_module.resolve_basis_files(files, basis_targets)
|
|
339
|
+
if not basis_files:
|
|
340
|
+
_console.print(
|
|
341
|
+
"[yellow]Warning: none of the --basis paths matched any scanned files.[/yellow]"
|
|
342
|
+
)
|
|
343
|
+
|
|
191
344
|
total_fragments = _process_files(files, session_db, session_id, min_tokens)
|
|
192
345
|
|
|
193
|
-
# ── Phase 3: Clone detection ────────────────────────────────────────
|
|
194
346
|
cnt1, cnt2, cnt3 = _detect_clones(
|
|
195
347
|
session_db, session_id, detect_types, threshold
|
|
196
348
|
)
|
|
197
349
|
|
|
350
|
+
basis_counts = {1: 0, 2: 0, 3: 0}
|
|
351
|
+
if basis_files is not None:
|
|
352
|
+
groups = session_db.get_clone_groups(session_id)
|
|
353
|
+
groups_with_members = [
|
|
354
|
+
(g, session_db.get_fragments_for_group(g)) for g in groups
|
|
355
|
+
]
|
|
356
|
+
basis_counts = basis_module.count_basis_groups_by_type(
|
|
357
|
+
groups_with_members, basis_files
|
|
358
|
+
)
|
|
359
|
+
|
|
198
360
|
result = ScanResult(
|
|
199
361
|
session_id=session_id,
|
|
200
362
|
version=__version__,
|
|
@@ -204,12 +366,16 @@ def main( # pylint: disable=too-many-arguments,too-many-positional-arguments,to
|
|
|
204
366
|
type1_groups=cnt1,
|
|
205
367
|
type2_groups=cnt2,
|
|
206
368
|
type3_groups=cnt3,
|
|
369
|
+
basis_paths=[str(p) for p in basis_targets],
|
|
370
|
+
basis_type1_groups=basis_counts[1],
|
|
371
|
+
basis_type2_groups=basis_counts[2],
|
|
372
|
+
basis_type3_groups=basis_counts[3],
|
|
207
373
|
)
|
|
208
374
|
|
|
209
|
-
|
|
210
|
-
|
|
375
|
+
written = _write_reports(
|
|
376
|
+
session_db, result, output_dir, output, fmt, basis_files
|
|
377
|
+
)
|
|
211
378
|
|
|
212
|
-
# ── Phase 5: Clean up session ───────────────────────────────────────
|
|
213
379
|
session_db.delete_session(session_id)
|
|
214
380
|
|
|
215
381
|
# ── Summary table (printed after DB is closed) ──────────────────────────
|
|
@@ -222,46 +388,72 @@ def _detect_clones(
|
|
|
222
388
|
detect_types: set[int],
|
|
223
389
|
threshold: float,
|
|
224
390
|
) -> tuple[int, int, int]:
|
|
225
|
-
"""Run clone detection and return (type1_count, type2_count, type3_count).
|
|
391
|
+
"""Run clone detection and return (type1_count, type2_count, type3_count).
|
|
392
|
+
|
|
393
|
+
:since: 1.0.0
|
|
394
|
+
"""
|
|
226
395
|
_console.print("[dim]Detecting clones…[/dim]")
|
|
227
396
|
return detect(session_db, session_id, detect_types, threshold)
|
|
228
397
|
|
|
229
398
|
|
|
230
|
-
def _write_reports(
|
|
399
|
+
def _write_reports( # pylint: disable=too-many-arguments,too-many-positional-arguments
|
|
231
400
|
session_db: SessionDB,
|
|
232
401
|
result: ScanResult,
|
|
233
402
|
output_dir: Path,
|
|
234
403
|
output: str,
|
|
235
404
|
fmt: str,
|
|
405
|
+
basis_files: frozenset[str] | None = None,
|
|
236
406
|
) -> list[Path]:
|
|
237
|
-
"""Write the requested report formats and return a list of written paths.
|
|
407
|
+
"""Write the requested report formats and return a list of written paths.
|
|
408
|
+
|
|
409
|
+
:since: 1.0.0
|
|
410
|
+
"""
|
|
238
411
|
written: list[Path] = []
|
|
239
412
|
if fmt in ("json", "both"):
|
|
240
413
|
written.append(
|
|
241
|
-
json_reporter.write(
|
|
414
|
+
json_reporter.write(
|
|
415
|
+
session_db, result, output_dir / f"{output}.json", basis_files
|
|
416
|
+
)
|
|
242
417
|
)
|
|
243
418
|
if fmt in ("html", "both"):
|
|
244
419
|
written.append(
|
|
245
|
-
html_reporter.write(
|
|
420
|
+
html_reporter.write(
|
|
421
|
+
session_db, result, output_dir / f"{output}.html", basis_files
|
|
422
|
+
)
|
|
246
423
|
)
|
|
247
424
|
return written
|
|
248
425
|
|
|
249
426
|
|
|
250
427
|
def _print_summary(result: ScanResult, written: list[Path]) -> None:
|
|
251
|
-
"""Print the scan summary table and list of saved report paths.
|
|
428
|
+
"""Print the scan summary table and list of saved report paths.
|
|
429
|
+
|
|
430
|
+
:since: 1.0.0
|
|
431
|
+
"""
|
|
432
|
+
has_basis = bool(result.basis_paths)
|
|
433
|
+
|
|
434
|
+
def _value(count: int, basis_count: int) -> str:
|
|
435
|
+
return f"{count} ({basis_count})" if has_basis else str(count)
|
|
436
|
+
|
|
252
437
|
table = Table(title="Scan Summary", show_header=True, header_style="bold magenta")
|
|
253
438
|
table.add_column("Metric", style="dim", min_width=26)
|
|
254
439
|
table.add_column("Value", justify="right", style="bold")
|
|
255
440
|
table.add_row("Files scanned", str(result.files_scanned))
|
|
256
441
|
table.add_row("Fragments extracted", str(result.fragments_extracted))
|
|
257
|
-
table.add_row("[red]Type-1[/red] clone groups (exact)", str(result.type1_groups))
|
|
258
442
|
table.add_row(
|
|
259
|
-
"[
|
|
443
|
+
"[red]Type-1[/red] clone groups (exact)",
|
|
444
|
+
_value(result.type1_groups, result.basis_type1_groups),
|
|
445
|
+
)
|
|
446
|
+
table.add_row(
|
|
447
|
+
"[yellow]Type-2[/yellow] clone groups (structural)",
|
|
448
|
+
_value(result.type2_groups, result.basis_type2_groups),
|
|
260
449
|
)
|
|
261
450
|
table.add_row(
|
|
262
|
-
"[green]Type-3[/green] clone groups (near-duplicate)",
|
|
451
|
+
"[green]Type-3[/green] clone groups (near-duplicate)",
|
|
452
|
+
_value(result.type3_groups, result.basis_type3_groups),
|
|
263
453
|
)
|
|
264
454
|
_console.print(table)
|
|
455
|
+
if has_basis:
|
|
456
|
+
_console.print("[dim]Value in parentheses = basis-touching groups.[/dim]")
|
|
265
457
|
_console.print("\n[bold]Reports saved:[/bold]")
|
|
266
458
|
for dest in written:
|
|
267
459
|
_console.print(f" [cyan]•[/cyan] {dest}")
|
|
@@ -277,6 +469,8 @@ def _process_files(
|
|
|
277
469
|
|
|
278
470
|
Returns:
|
|
279
471
|
Total number of fragments extracted.
|
|
472
|
+
|
|
473
|
+
:since: 1.0.0
|
|
280
474
|
"""
|
|
281
475
|
total = 0
|
|
282
476
|
with Progress(
|