redup 0.3.20__tar.gz → 0.3.22__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {redup-0.3.20 → redup-0.3.22}/PKG-INFO +2 -2
- {redup-0.3.20 → redup-0.3.22}/README.md +1 -1
- {redup-0.3.20 → redup-0.3.22}/pyproject.toml +1 -1
- {redup-0.3.20 → redup-0.3.22}/src/redup/__init__.py +1 -1
- redup-0.3.22/src/redup/cli_app/config_builder.py +51 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup/cli_app/main.py +220 -8
- redup-0.3.22/src/redup/cli_app/output_writer.py +78 -0
- redup-0.3.22/src/redup/cli_app/scan_commands.py +163 -0
- redup-0.3.22/src/redup/cli_app/scan_helpers.py +34 -0
- redup-0.3.22/src/redup/core/fuzzy_similarity.py +408 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup/core/models.py +8 -3
- redup-0.3.22/src/redup/core/universal_fuzzy.py +454 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup.egg-info/PKG-INFO +2 -2
- {redup-0.3.20 → redup-0.3.22}/src/redup.egg-info/SOURCES.txt +6 -0
- {redup-0.3.20 → redup-0.3.22}/LICENSE +0 -0
- {redup-0.3.20 → redup-0.3.22}/setup.cfg +0 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup/__main__.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup/cli_app/__init__.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup/core/__init__.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup/core/cache.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup/core/config.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup/core/differ.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup/core/hash_cache.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup/core/hasher.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup/core/lazy_grouper.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup/core/lsh_matcher.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup/core/matcher.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup/core/memory_scanner.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup/core/parallel_scanner.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup/core/pipeline.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup/core/planner.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup/core/scanner.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup/core/ts_extractor.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup/core/ultra_fast_scanner.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup/core/utils/__init__.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup/core/utils/duplicate_finders.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup/core/utils/function_extractor.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup/core/utils/hash_utils.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup/core/utils/language_dispatcher.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup/reporters/__init__.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup/reporters/code2llm_reporter.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup/reporters/enhanced_reporter.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup/reporters/json_reporter.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup/reporters/markdown_reporter.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup/reporters/toon_reporter.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup/reporters/yaml_reporter.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup.egg-info/dependency_links.txt +0 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup.egg-info/entry_points.txt +0 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup.egg-info/requires.txt +0 -0
- {redup-0.3.20 → redup-0.3.22}/src/redup.egg-info/top_level.txt +0 -0
- {redup-0.3.20 → redup-0.3.22}/tests/test_e2e.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/tests/test_hasher.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/tests/test_matcher.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/tests/test_models.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/tests/test_pipeline.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/tests/test_planner.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/tests/test_reporters.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/tests/test_scanner.py +0 -0
- {redup-0.3.20 → redup-0.3.22}/tests/test_ts_extractor.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: redup
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.22
|
|
4
4
|
Summary: Code duplication analyzer and refactoring planner for LLMs
|
|
5
5
|
Author-email: Tom Sapletta <tom@sapletta.com>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -83,7 +83,7 @@ Dynamic: license-file
|
|
|
83
83
|
[](https://pypi.org/project/redup/)
|
|
84
84
|
[](https://opensource.org/licenses/Apache-2.0)
|
|
85
85
|
[](https://python.org)
|
|
86
|
-
[](https://pypi.org/project/redup/)
|
|
87
87
|
|
|
88
88
|
reDUP scans codebases for duplicated functions, blocks, and structural patterns — then builds a prioritized refactoring map that LLMs can consume to eliminate redundancy systematically.
|
|
89
89
|
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
[](https://pypi.org/project/redup/)
|
|
6
6
|
[](https://opensource.org/licenses/Apache-2.0)
|
|
7
7
|
[](https://python.org)
|
|
8
|
-
[](https://pypi.org/project/redup/)
|
|
9
9
|
|
|
10
10
|
reDUP scans codebases for duplicated functions, blocks, and structural patterns — then builds a prioritized refactoring map that LLMs can consume to eliminate redundancy systematically.
|
|
11
11
|
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""Configuration builders for reDUP CLI."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from redup.core.config import config_to_scan_config, load_config
|
|
7
|
+
from redup.core.models import ScanConfig
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def build_config(
|
|
11
|
+
path: Path,
|
|
12
|
+
extensions: str,
|
|
13
|
+
min_lines: int,
|
|
14
|
+
min_similarity: float,
|
|
15
|
+
include_tests: bool
|
|
16
|
+
) -> ScanConfig:
|
|
17
|
+
"""Build basic scan configuration."""
|
|
18
|
+
config = load_config()
|
|
19
|
+
return config_to_scan_config(config, path)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def build_config_with_file_support(
|
|
23
|
+
path: Path,
|
|
24
|
+
extensions: Any,
|
|
25
|
+
min_lines: Any,
|
|
26
|
+
min_similarity: Any,
|
|
27
|
+
include_tests: Any,
|
|
28
|
+
parallel: bool = False,
|
|
29
|
+
max_workers: Any = None,
|
|
30
|
+
incremental: bool = False,
|
|
31
|
+
memory_cache: bool = False,
|
|
32
|
+
max_cache_mb: int = 512,
|
|
33
|
+
functions_only: bool = False
|
|
34
|
+
) -> ScanConfig:
|
|
35
|
+
"""Build scan configuration with advanced options."""
|
|
36
|
+
config = load_config()
|
|
37
|
+
scan_config = config_to_scan_config(config, path)
|
|
38
|
+
|
|
39
|
+
# Override with CLI parameters if provided
|
|
40
|
+
if extensions is not None:
|
|
41
|
+
scan_config.extensions = [e.strip() for e in extensions.split(',')]
|
|
42
|
+
if min_lines is not None:
|
|
43
|
+
scan_config.min_lines = min_lines
|
|
44
|
+
if min_similarity is not None:
|
|
45
|
+
scan_config.min_similarity = min_similarity
|
|
46
|
+
if include_tests is not None:
|
|
47
|
+
scan_config.include_tests = include_tests
|
|
48
|
+
if functions_only:
|
|
49
|
+
scan_config.function_level_only = functions_only
|
|
50
|
+
|
|
51
|
+
return scan_config
|
|
@@ -13,15 +13,17 @@ import typer
|
|
|
13
13
|
warnings.filterwarnings("ignore", category=SyntaxWarning)
|
|
14
14
|
|
|
15
15
|
import redup # noqa: E402
|
|
16
|
-
from redup.core.config import
|
|
16
|
+
from redup.core.config import create_sample_redup_toml, load_config # noqa: E402
|
|
17
17
|
from redup.core.models import DuplicationMap, ScanConfig # noqa: E402
|
|
18
|
-
from redup.core.
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
from redup.
|
|
22
|
-
from redup.
|
|
23
|
-
from redup.
|
|
24
|
-
|
|
18
|
+
from redup.core.differ import compare_scans, format_diff_result # noqa: E402
|
|
19
|
+
|
|
20
|
+
# Import refactored modules
|
|
21
|
+
from redup.cli_app.config_builder import build_config, build_config_with_file_support # noqa: E402
|
|
22
|
+
from redup.cli_app.output_writer import write_output, write_results # noqa: E402
|
|
23
|
+
from redup.cli_app.scan_commands import ( # noqa: E402
|
|
24
|
+
scan_command, diff_command, check_command, config_command, info_command
|
|
25
|
+
)
|
|
26
|
+
from redup.cli_app.scan_helpers import print_scan_header, print_scan_summary # noqa: E402
|
|
25
27
|
|
|
26
28
|
app = typer.Typer(
|
|
27
29
|
name="redup",
|
|
@@ -142,6 +144,160 @@ def _print_scan_summary(dup_map: DuplicationMap) -> None:
|
|
|
142
144
|
typer.echo("")
|
|
143
145
|
|
|
144
146
|
|
|
147
|
+
def _apply_fuzzy_similarity(dup_map: DuplicationMap, threshold: float) -> DuplicationMap:
|
|
148
|
+
"""Apply fuzzy similarity detection to find similar components across all languages."""
|
|
149
|
+
validation = _validate_fuzzy_input(dup_map, threshold)
|
|
150
|
+
if not validation.is_valid:
|
|
151
|
+
typer.echo(validation.message)
|
|
152
|
+
return dup_map
|
|
153
|
+
|
|
154
|
+
all_blocks = _extract_all_blocks(dup_map)
|
|
155
|
+
if not all_blocks:
|
|
156
|
+
typer.echo("🔍 No code blocks found for fuzzy analysis")
|
|
157
|
+
return dup_map
|
|
158
|
+
|
|
159
|
+
typer.echo(f"🔍 Analyzing {len(all_blocks)} code blocks for universal fuzzy similarity...")
|
|
160
|
+
|
|
161
|
+
html_css_blocks, other_blocks = _separate_blocks_by_type(all_blocks)
|
|
162
|
+
similar_pairs = _analyze_blocks_with_detectors(html_css_blocks, other_blocks, threshold)
|
|
163
|
+
|
|
164
|
+
_report_fuzzy_results(similar_pairs)
|
|
165
|
+
return dup_map
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _validate_fuzzy_input(dup_map: DuplicationMap, threshold: float) -> 'FuzzyValidationResult':
|
|
169
|
+
"""Validate input parameters for fuzzy similarity analysis."""
|
|
170
|
+
from dataclasses import dataclass
|
|
171
|
+
|
|
172
|
+
@dataclass
|
|
173
|
+
class FuzzyValidationResult:
|
|
174
|
+
is_valid: bool
|
|
175
|
+
message: str = ""
|
|
176
|
+
|
|
177
|
+
if threshold <= 0 or threshold > 1:
|
|
178
|
+
return FuzzyValidationResult(False, "❌ Invalid threshold. Must be between 0 and 1.")
|
|
179
|
+
|
|
180
|
+
if not dup_map.groups:
|
|
181
|
+
return FuzzyValidationResult(False, "❌ No duplicate groups found.")
|
|
182
|
+
|
|
183
|
+
return FuzzyValidationResult(True)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _extract_all_blocks(dup_map: DuplicationMap) -> list:
|
|
187
|
+
"""Extract all code blocks from duplicate fragments."""
|
|
188
|
+
from redup.core.scanner import CodeBlock
|
|
189
|
+
|
|
190
|
+
all_blocks = []
|
|
191
|
+
for group in dup_map.groups:
|
|
192
|
+
for fragment in group.fragments:
|
|
193
|
+
# Create a CodeBlock from fragment data
|
|
194
|
+
block = CodeBlock(
|
|
195
|
+
file=fragment.file,
|
|
196
|
+
line_start=fragment.line_start,
|
|
197
|
+
line_end=fragment.line_end,
|
|
198
|
+
text=fragment.text,
|
|
199
|
+
function_name=fragment.function_name,
|
|
200
|
+
class_name=fragment.class_name
|
|
201
|
+
)
|
|
202
|
+
all_blocks.append(block)
|
|
203
|
+
return all_blocks
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def _separate_blocks_by_type(all_blocks: list) -> tuple[list, list]:
|
|
207
|
+
"""Separate blocks into HTML/CSS and other language categories."""
|
|
208
|
+
html_css_blocks = [block for block in all_blocks
|
|
209
|
+
if Path(block.file).suffix.lower() in ['.html', '.htm', '.css', '.scss', '.sass', '.less']]
|
|
210
|
+
|
|
211
|
+
other_blocks = [block for block in all_blocks if block not in html_css_blocks]
|
|
212
|
+
|
|
213
|
+
return html_css_blocks, other_blocks
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _analyze_blocks_with_detectors(html_css_blocks: list, other_blocks: list, threshold: float) -> list:
|
|
217
|
+
"""Analyze blocks using appropriate fuzzy similarity detectors."""
|
|
218
|
+
similar_pairs = []
|
|
219
|
+
|
|
220
|
+
# Analyze HTML/CSS with specialized detector
|
|
221
|
+
if html_css_blocks:
|
|
222
|
+
typer.echo(f" 📝 Analyzing {len(html_css_blocks)} HTML/CSS blocks...")
|
|
223
|
+
html_css_similar = _analyze_html_css_blocks(html_css_blocks, threshold)
|
|
224
|
+
similar_pairs.extend(html_css_similar)
|
|
225
|
+
typer.echo(f" ✨ Found {len(html_css_similar)} similar HTML/CSS component pairs")
|
|
226
|
+
|
|
227
|
+
# Analyze other languages with universal detector
|
|
228
|
+
if other_blocks:
|
|
229
|
+
typer.echo(f" 🔧 Analyzing {len(other_blocks)} blocks from other languages...")
|
|
230
|
+
universal_similar = _analyze_other_language_blocks(other_blocks, threshold)
|
|
231
|
+
similar_pairs.extend(universal_similar)
|
|
232
|
+
typer.echo(f" ✨ Found {len(universal_similar)} similar component pairs from other languages")
|
|
233
|
+
|
|
234
|
+
return similar_pairs
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _analyze_html_css_blocks(html_css_blocks: list, threshold: float) -> list:
|
|
238
|
+
"""Analyze HTML/CSS blocks using specialized fuzzy similarity detector."""
|
|
239
|
+
from redup.core.fuzzy_similarity import FuzzySimilarityDetector
|
|
240
|
+
|
|
241
|
+
html_css_detector = FuzzySimilarityDetector(similarity_threshold=threshold)
|
|
242
|
+
return html_css_detector.find_similar_components(html_css_blocks)
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def _analyze_other_language_blocks(other_blocks: list, threshold: float) -> list:
|
|
246
|
+
"""Analyze non-HTML/CSS blocks using universal fuzzy similarity detector."""
|
|
247
|
+
from redup.core.universal_fuzzy import UniversalFuzzyDetector
|
|
248
|
+
|
|
249
|
+
universal_detector = UniversalFuzzyDetector(similarity_threshold=threshold)
|
|
250
|
+
return universal_detector.find_similar_components(other_blocks)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def _report_fuzzy_results(similar_pairs: list) -> None:
|
|
254
|
+
"""Report fuzzy similarity analysis results."""
|
|
255
|
+
if similar_pairs:
|
|
256
|
+
typer.echo(f"🎯 Total fuzzy similar components found: {len(similar_pairs)}")
|
|
257
|
+
|
|
258
|
+
# Simplified reporting to reduce complexity
|
|
259
|
+
typer.echo(f" 📊 Language groups: {len(set(Path(b1.file).suffix + '+' + Path(b2.file).suffix for b1, b2, _ in similar_pairs))}")
|
|
260
|
+
|
|
261
|
+
# Show top examples only
|
|
262
|
+
for block1, block2, similarity in similar_pairs[:5]:
|
|
263
|
+
typer.echo(f" {block1.file} ↔ {block2.file}: {similarity:.2f}")
|
|
264
|
+
|
|
265
|
+
if len(similar_pairs) > 5:
|
|
266
|
+
typer.echo(f" ... and {len(similar_pairs) - 5} more")
|
|
267
|
+
|
|
268
|
+
# Note: In production, you'd integrate these into DuplicationMap
|
|
269
|
+
# For now, we just report the findings
|
|
270
|
+
|
|
271
|
+
else:
|
|
272
|
+
typer.echo("🔍 No fuzzy similar components found")
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def _report_similarity_by_groups(similar_pairs):
|
|
276
|
+
"""Extracted function to reduce complexity."""
|
|
277
|
+
lang_groups = {}
|
|
278
|
+
for block1, block2, similarity in similar_pairs:
|
|
279
|
+
lang1 = Path(block1.file).suffix
|
|
280
|
+
lang2 = Path(block2.file).suffix
|
|
281
|
+
comp_type = "unknown"
|
|
282
|
+
|
|
283
|
+
# Try to get component type
|
|
284
|
+
try:
|
|
285
|
+
from redup.core.universal_fuzzy import UniversalFuzzyExtractor
|
|
286
|
+
extractor = UniversalFuzzyExtractor()
|
|
287
|
+
sig1 = extractor.extract_universal_signature(block1)
|
|
288
|
+
if sig1:
|
|
289
|
+
comp_type = sig1.component_type
|
|
290
|
+
except:
|
|
291
|
+
pass
|
|
292
|
+
|
|
293
|
+
group_key = f"{lang1}+{lang2}:{comp_type}"
|
|
294
|
+
if group_key not in lang_groups:
|
|
295
|
+
lang_groups[group_key] = []
|
|
296
|
+
lang_groups[group_key].append((block1, block2, similarity))
|
|
297
|
+
|
|
298
|
+
return lang_groups
|
|
299
|
+
|
|
300
|
+
|
|
145
301
|
def _write_results(
|
|
146
302
|
dup_map: DuplicationMap,
|
|
147
303
|
format: str,
|
|
@@ -247,6 +403,44 @@ def scan(
|
|
|
247
403
|
"--parallel/--no-parallel",
|
|
248
404
|
help="Use parallel scanning for large projects (default: disabled due to issues).",
|
|
249
405
|
),
|
|
406
|
+
max_workers: int | None = typer.Option(
|
|
407
|
+
None,
|
|
408
|
+
"--max-workers",
|
|
409
|
+
help="Maximum number of parallel workers.",
|
|
410
|
+
),
|
|
411
|
+
incremental: bool = typer.Option(
|
|
412
|
+
False,
|
|
413
|
+
"--incremental",
|
|
414
|
+
help="Use incremental scanning with caching.",
|
|
415
|
+
),
|
|
416
|
+
memory_cache: bool = typer.Option(
|
|
417
|
+
True,
|
|
418
|
+
"--memory-cache/--no-memory-cache",
|
|
419
|
+
help="Use memory cache for faster scanning.",
|
|
420
|
+
),
|
|
421
|
+
max_cache_mb: int = typer.Option(
|
|
422
|
+
512,
|
|
423
|
+
"--max-cache-mb",
|
|
424
|
+
help="Maximum memory cache size in MB.",
|
|
425
|
+
),
|
|
426
|
+
fuzzy: bool = typer.Option(
|
|
427
|
+
False,
|
|
428
|
+
"--fuzzy",
|
|
429
|
+
help="Enable fuzzy similarity detection.",
|
|
430
|
+
),
|
|
431
|
+
fuzzy_threshold: float = typer.Option(
|
|
432
|
+
0.8,
|
|
433
|
+
"--fuzzy-threshold",
|
|
434
|
+
help="Fuzzy similarity threshold (0.0-1.0).",
|
|
435
|
+
),
|
|
436
|
+
) -> None:
|
|
437
|
+
"""Scan a project for code duplicates."""
|
|
438
|
+
return scan_command(
|
|
439
|
+
path, format, output, extensions, min_lines, min_similarity,
|
|
440
|
+
include_tests, functions_only, parallel, max_workers,
|
|
441
|
+
incremental, memory_cache, max_cache_mb, fuzzy, fuzzy_threshold
|
|
442
|
+
)
|
|
443
|
+
),
|
|
250
444
|
max_workers: int = typer.Option(
|
|
251
445
|
None,
|
|
252
446
|
"--max-workers",
|
|
@@ -267,6 +461,16 @@ def scan(
|
|
|
267
461
|
"--max-cache-mb",
|
|
268
462
|
help="Maximum memory for file caching in MB (default: 512).",
|
|
269
463
|
),
|
|
464
|
+
fuzzy: bool = typer.Option(
|
|
465
|
+
False,
|
|
466
|
+
"--fuzzy/--no-fuzzy",
|
|
467
|
+
help="Enable fuzzy similarity detection for HTML/CSS components (default: disabled).",
|
|
468
|
+
),
|
|
469
|
+
fuzzy_threshold: float = typer.Option(
|
|
470
|
+
0.8,
|
|
471
|
+
"--fuzzy-threshold",
|
|
472
|
+
help="Fuzzy similarity threshold (0.0-1.0, default: 0.8).",
|
|
473
|
+
),
|
|
270
474
|
) -> None:
|
|
271
475
|
"""Scan a project for code duplicates and generate a refactoring map.
|
|
272
476
|
|
|
@@ -284,6 +488,10 @@ def scan(
|
|
|
284
488
|
parallel, max_workers, incremental, memory_cache, max_cache_mb, functions_only
|
|
285
489
|
)
|
|
286
490
|
|
|
491
|
+
# Add fuzzy options to config
|
|
492
|
+
config.fuzzy_enabled = fuzzy
|
|
493
|
+
config.fuzzy_threshold = fuzzy_threshold
|
|
494
|
+
|
|
287
495
|
_print_scan_header(path, config.extensions, config.min_block_lines, config.min_similarity)
|
|
288
496
|
|
|
289
497
|
# Choose analysis method
|
|
@@ -292,6 +500,10 @@ def scan(
|
|
|
292
500
|
else:
|
|
293
501
|
dup_map = analyze(config=config, function_level_only=functions_only)
|
|
294
502
|
|
|
503
|
+
# Apply fuzzy similarity detection if enabled
|
|
504
|
+
if fuzzy:
|
|
505
|
+
dup_map = _apply_fuzzy_similarity(dup_map, fuzzy_threshold)
|
|
506
|
+
|
|
295
507
|
_print_scan_summary(dup_map)
|
|
296
508
|
_write_results(dup_map, format, output, path)
|
|
297
509
|
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""Output writers for reDUP CLI results."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from redup.core.models import DuplicationMap
|
|
7
|
+
from redup.reporters.json_reporter import to_json
|
|
8
|
+
from redup.reporters.markdown_reporter import to_markdown
|
|
9
|
+
from redup.reporters.toon_reporter import to_toon
|
|
10
|
+
from redup.reporters.yaml_reporter import to_yaml
|
|
11
|
+
from redup.reporters.code2llm_reporter import export_code2llm, to_code2llm_context, to_code2llm_toon
|
|
12
|
+
from redup.reporters.enhanced_reporter import EnhancedReporter
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def write_output(content: str, output: Path | None, suffix: str) -> None:
|
|
16
|
+
"""Write content to file or stdout."""
|
|
17
|
+
import typer
|
|
18
|
+
|
|
19
|
+
if output:
|
|
20
|
+
target = output if output.suffix else output / f"duplication.{suffix}"
|
|
21
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
22
|
+
target.write_text(content, encoding="utf-8")
|
|
23
|
+
typer.echo(f" → {target}")
|
|
24
|
+
else:
|
|
25
|
+
typer.echo(content)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def write_results(dup_map: DuplicationMap, format: str, output: Any, path: Path) -> None:
|
|
29
|
+
"""Write analysis results in specified format."""
|
|
30
|
+
import typer
|
|
31
|
+
|
|
32
|
+
if format == "json":
|
|
33
|
+
content = to_json(dup_map)
|
|
34
|
+
write_output(content, output, "json")
|
|
35
|
+
elif format == "yaml":
|
|
36
|
+
content = to_yaml(dup_map)
|
|
37
|
+
write_output(content, output, "yaml")
|
|
38
|
+
elif format == "toon":
|
|
39
|
+
content = to_toon(dup_map)
|
|
40
|
+
write_output(content, output, "toon")
|
|
41
|
+
elif format == "markdown":
|
|
42
|
+
content = to_markdown(dup_map)
|
|
43
|
+
write_output(content, output, "md")
|
|
44
|
+
elif format == "enhanced":
|
|
45
|
+
reporter = EnhancedReporter(dup_map)
|
|
46
|
+
content = reporter.generate_report()
|
|
47
|
+
write_output(content, output, "html")
|
|
48
|
+
elif format == "code2llm":
|
|
49
|
+
if output:
|
|
50
|
+
content = to_code2llm_context(dup_map)
|
|
51
|
+
write_output(content, output, "md")
|
|
52
|
+
else:
|
|
53
|
+
# For code2llm, create a directory with multiple files
|
|
54
|
+
export_code2llm(dup_map, output or path / "code2llm")
|
|
55
|
+
elif format == "all":
|
|
56
|
+
# Generate all formats
|
|
57
|
+
formats = {
|
|
58
|
+
"json": to_json(dup_map),
|
|
59
|
+
"yaml": to_yaml(dup_map),
|
|
60
|
+
"toon": to_toon(dup_map),
|
|
61
|
+
"markdown": to_markdown(dup_map),
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
if output and output.is_dir():
|
|
65
|
+
for fmt, content in formats.items():
|
|
66
|
+
target = output / f"duplication.{fmt}"
|
|
67
|
+
target.write_text(content, encoding="utf-8")
|
|
68
|
+
typer.echo(f" → {target}")
|
|
69
|
+
else:
|
|
70
|
+
# Write to separate files in the same directory
|
|
71
|
+
base_dir = output.parent if output else path
|
|
72
|
+
for fmt, content in formats.items():
|
|
73
|
+
target = base_dir / f"duplication.{fmt}"
|
|
74
|
+
target.write_text(content, encoding="utf-8")
|
|
75
|
+
typer.echo(f" → {target}")
|
|
76
|
+
else:
|
|
77
|
+
typer.echo(f"❌ Unknown format: {format}", err=True)
|
|
78
|
+
raise typer.Exit(1)
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
"""Scan command implementations for reDUP CLI."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
import typer
|
|
7
|
+
|
|
8
|
+
from redup.core.config import create_sample_redup_toml
|
|
9
|
+
from redup.core.models import DuplicationMap
|
|
10
|
+
from redup.core.pipeline import analyze, analyze_parallel, analyze_optimized
|
|
11
|
+
from redup.cli_app.config_builder import build_config, build_config_with_file_support
|
|
12
|
+
from redup.cli_app.output_writer import write_output
|
|
13
|
+
from redup.cli_app.scan_helpers import print_scan_header, print_scan_summary, apply_fuzzy_similarity
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def scan_command(
|
|
17
|
+
path: Path = typer.Argument(Path("."), help="Project root directory to scan.", exists=True, dir_okay=True, file_okay=False),
|
|
18
|
+
format: str = typer.Option("toon", "--format", "-f", help="Output format (json, yaml, toon, markdown, code2llm, all, enhanced)."),
|
|
19
|
+
output: Any = typer.Option(None, "--output", "-o", help="Output directory or file path. Defaults to stdout."),
|
|
20
|
+
extensions: Any = typer.Option(None, "--ext", "-e", help="Comma-separated file extensions to scan. Overrides config."),
|
|
21
|
+
min_lines: Any = typer.Option(None, "--min-lines", help="Minimum block size (lines). Overrides config."),
|
|
22
|
+
min_similarity: Any = typer.Option(None, "--min-sim", help="Minimum similarity score (0.0-1.0). Overrides config."),
|
|
23
|
+
include_tests: bool = typer.Option(False, "--include-tests", help="Include test files in analysis."),
|
|
24
|
+
functions_only: bool = typer.Option(False, "--functions-only", help="Only analyze function-level duplicates."),
|
|
25
|
+
parallel: bool = typer.Option(False, "--parallel", help="Use parallel processing for faster analysis."),
|
|
26
|
+
max_workers: Any = typer.Option(None, "--max-workers", help="Maximum number of parallel workers."),
|
|
27
|
+
incremental: bool = typer.Option(False, "--incremental", help="Use incremental scanning (cache-based)."),
|
|
28
|
+
memory_cache: bool = typer.Option(True, "--memory-cache/--no-memory-cache", help="Use memory cache for faster processing."),
|
|
29
|
+
max_cache_mb: int = typer.Option(512, "--max-cache-mb", help="Maximum memory cache size in MB."),
|
|
30
|
+
fuzzy: bool = typer.Option(False, "--fuzzy", help="Enable fuzzy similarity detection."),
|
|
31
|
+
fuzzy_threshold: float = typer.Option(0.8, "--fuzzy-threshold", help="Fuzzy similarity threshold (0.0-1.0)."),
|
|
32
|
+
) -> None:
|
|
33
|
+
"""Scan a project for code duplicates."""
|
|
34
|
+
|
|
35
|
+
# Build configuration
|
|
36
|
+
if any([extensions, min_lines, min_similarity, include_tests, functions_only, parallel, max_workers, incremental, memory_cache, max_cache_mb]):
|
|
37
|
+
config = build_config_with_file_support(
|
|
38
|
+
path, extensions, min_lines, min_similarity, include_tests,
|
|
39
|
+
parallel, max_workers, incremental, memory_cache, max_cache_mb, functions_only
|
|
40
|
+
)
|
|
41
|
+
else:
|
|
42
|
+
config = build_config(path, extensions, min_lines, min_similarity, include_tests)
|
|
43
|
+
|
|
44
|
+
# Print scan header
|
|
45
|
+
print_scan_header(path, config.extensions, config.min_lines, config.min_similarity)
|
|
46
|
+
|
|
47
|
+
# Run analysis
|
|
48
|
+
if parallel and max_workers and max_workers > 1:
|
|
49
|
+
dup_map = analyze_parallel(config, max_workers)
|
|
50
|
+
elif memory_cache:
|
|
51
|
+
dup_map = analyze_optimized(config, use_memory_cache=True, max_cache_mb=max_cache_mb)
|
|
52
|
+
else:
|
|
53
|
+
dup_map = analyze(config, function_level_only=functions_only)
|
|
54
|
+
|
|
55
|
+
# Apply fuzzy similarity if requested
|
|
56
|
+
if fuzzy:
|
|
57
|
+
dup_map = apply_fuzzy_similarity(dup_map, fuzzy_threshold)
|
|
58
|
+
|
|
59
|
+
# Print summary
|
|
60
|
+
print_scan_summary(dup_map)
|
|
61
|
+
|
|
62
|
+
# Write results
|
|
63
|
+
from redup.cli_app.output_writer import write_results
|
|
64
|
+
write_results(dup_map, format, output, path)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def diff_command(before: Path, after: Path) -> None:
|
|
68
|
+
"""Compare two reDUP analysis results."""
|
|
69
|
+
from redup.core.differ import compare_scans, format_diff_result
|
|
70
|
+
|
|
71
|
+
try:
|
|
72
|
+
diff = compare_scans(before, after)
|
|
73
|
+
result = format_diff_result(diff)
|
|
74
|
+
typer.echo(result)
|
|
75
|
+
except Exception as e:
|
|
76
|
+
typer.echo(f"❌ Error comparing scans: {e}", err=True)
|
|
77
|
+
raise typer.Exit(1)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def check_command(
|
|
81
|
+
path: Path = typer.Argument(Path("."), help="Project root directory to check.", exists=True, dir_okay=True, file_okay=False),
|
|
82
|
+
max_groups: int = typer.Option(10, "--max-groups", help="Maximum duplicate groups to report."),
|
|
83
|
+
max_saved_lines: int = typer.Option(100, "--max-saved-lines", help="Maximum saved lines to report."),
|
|
84
|
+
extensions: Any = typer.Option(None, "--ext", "-e", help="Comma-separated file extensions to scan."),
|
|
85
|
+
min_lines: Any = typer.Option(None, "--min-lines", help="Minimum block size (lines)."),
|
|
86
|
+
min_similarity: Any = typer.Option(None, "--min-sim", help="Minimum similarity score (0.0-1.0)."),
|
|
87
|
+
include_tests: bool = typer.Option(False, "--include-tests", help="Include test files."),
|
|
88
|
+
) -> None:
|
|
89
|
+
"""Quick check for duplicates with summary report."""
|
|
90
|
+
|
|
91
|
+
config = build_config_with_file_support(
|
|
92
|
+
path, extensions, min_lines, min_similarity, include_tests
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
print_scan_header(path, config.extensions, config.min_lines, config.min_similarity)
|
|
96
|
+
|
|
97
|
+
dup_map = analyze(config)
|
|
98
|
+
print_scan_summary(dup_map)
|
|
99
|
+
|
|
100
|
+
# Show top duplicate groups
|
|
101
|
+
if dup_map.groups:
|
|
102
|
+
typer.echo(f"\n🎯 Top {min(max_groups, len(dup_map.groups))} duplicate groups:")
|
|
103
|
+
for i, group in enumerate(dup_map.sorted_by_impact()[:max_groups]):
|
|
104
|
+
if group.saved_lines_potential <= max_saved_lines:
|
|
105
|
+
break
|
|
106
|
+
typer.echo(f" {i+1}. {group.id}: {group.occurrences} occurrences, {group.saved_lines_potential} lines recoverable")
|
|
107
|
+
else:
|
|
108
|
+
typer.echo("✅ No duplicates found!")
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def config_command(init: bool = typer.Option(False, "--init", help="Create sample redup.toml configuration."),
|
|
112
|
+
show: bool = typer.Option(False, "--show", help="Show current configuration.")) -> None:
|
|
113
|
+
"""Manage reDUP configuration."""
|
|
114
|
+
|
|
115
|
+
if init:
|
|
116
|
+
config_path = Path("redup.toml")
|
|
117
|
+
if config_path.exists():
|
|
118
|
+
typer.echo("⚠️ redup.toml already exists. Overwrite? [y/N]", nl=False)
|
|
119
|
+
response = input().strip().lower()
|
|
120
|
+
if response != 'y':
|
|
121
|
+
typer.echo("Cancelled.")
|
|
122
|
+
return
|
|
123
|
+
|
|
124
|
+
content = create_sample_redup_toml()
|
|
125
|
+
config_path.write_text(content, encoding="utf-8")
|
|
126
|
+
typer.echo(f"✅ Created sample configuration: {config_path}")
|
|
127
|
+
|
|
128
|
+
elif show:
|
|
129
|
+
from redup.core.config import load_config
|
|
130
|
+
config = load_config()
|
|
131
|
+
typer.echo("Current configuration:")
|
|
132
|
+
typer.echo(f" Extensions: {config.get('extensions', ['.py'])}")
|
|
133
|
+
typer.echo(f" Min lines: {config.get('min_lines', 3)}")
|
|
134
|
+
typer.echo(f" Min similarity: {config.get('min_similarity', 0.8)}")
|
|
135
|
+
typer.echo(f" Include tests: {config.get('include_tests', False)}")
|
|
136
|
+
|
|
137
|
+
else:
|
|
138
|
+
typer.echo("Use --init to create config or --show to display current config.")
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def info_command() -> None:
|
|
142
|
+
"""Show reDUP version and system information."""
|
|
143
|
+
import redup
|
|
144
|
+
import sys
|
|
145
|
+
|
|
146
|
+
typer.echo(f"reDUP version: {redup.__version__}")
|
|
147
|
+
typer.echo(f"Python version: {sys.version}")
|
|
148
|
+
typer.echo(f"Installation path: {Path(redup.__file__).parent}")
|
|
149
|
+
|
|
150
|
+
# Check optional dependencies
|
|
151
|
+
deps = {
|
|
152
|
+
"tree-sitter": "tree_sitter",
|
|
153
|
+
"datasketch": "datasketch",
|
|
154
|
+
"rapidfuzz": "rapidfuzz",
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
typer.echo("\nOptional dependencies:")
|
|
158
|
+
for name, module in deps.items():
|
|
159
|
+
try:
|
|
160
|
+
__import__(module)
|
|
161
|
+
typer.echo(f" ✅ {name}")
|
|
162
|
+
except ImportError:
|
|
163
|
+
typer.echo(f" ❌ {name} (install with: pip install {name})")
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""Helper functions for scan operations."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import List
|
|
5
|
+
|
|
6
|
+
import typer
|
|
7
|
+
|
|
8
|
+
from redup.core.models import DuplicationMap
|
|
9
|
+
from redup.cli_app.main import _apply_fuzzy_similarity
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def print_scan_header(path: Path, ext_list: List[str], min_lines: int, min_similarity: float) -> None:
|
|
13
|
+
"""Print scan operation header."""
|
|
14
|
+
typer.echo(f"🔍 Scanning: {path}")
|
|
15
|
+
typer.echo(f"📁 Extensions: {', '.join(ext_list)}")
|
|
16
|
+
typer.echo(f"📏 Min lines: {min_lines}")
|
|
17
|
+
typer.echo(f"🎯 Min similarity: {min_similarity}")
|
|
18
|
+
typer.echo("")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def print_scan_summary(dup_map: DuplicationMap) -> None:
|
|
22
|
+
"""Print scan operation summary."""
|
|
23
|
+
typer.echo(f"📊 Scanned {dup_map.stats.files_scanned} files "
|
|
24
|
+
f"({dup_map.stats.total_lines} lines, "
|
|
25
|
+
f"{dup_map.stats.scan_time_ms:.0f}ms)")
|
|
26
|
+
typer.echo(f"Found {dup_map.total_groups} duplicate groups "
|
|
27
|
+
f"({dup_map.total_fragments} fragments, "
|
|
28
|
+
f"{dup_map.total_saved_lines} lines recoverable)")
|
|
29
|
+
typer.echo("")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def apply_fuzzy_similarity(dup_map: DuplicationMap, threshold: float) -> DuplicationMap:
|
|
33
|
+
"""Apply fuzzy similarity detection (wrapper for main function)."""
|
|
34
|
+
return _apply_fuzzy_similarity(dup_map, threshold)
|