redup 0.3.20__tar.gz → 0.3.22__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. {redup-0.3.20 → redup-0.3.22}/PKG-INFO +2 -2
  2. {redup-0.3.20 → redup-0.3.22}/README.md +1 -1
  3. {redup-0.3.20 → redup-0.3.22}/pyproject.toml +1 -1
  4. {redup-0.3.20 → redup-0.3.22}/src/redup/__init__.py +1 -1
  5. redup-0.3.22/src/redup/cli_app/config_builder.py +51 -0
  6. {redup-0.3.20 → redup-0.3.22}/src/redup/cli_app/main.py +220 -8
  7. redup-0.3.22/src/redup/cli_app/output_writer.py +78 -0
  8. redup-0.3.22/src/redup/cli_app/scan_commands.py +163 -0
  9. redup-0.3.22/src/redup/cli_app/scan_helpers.py +34 -0
  10. redup-0.3.22/src/redup/core/fuzzy_similarity.py +408 -0
  11. {redup-0.3.20 → redup-0.3.22}/src/redup/core/models.py +8 -3
  12. redup-0.3.22/src/redup/core/universal_fuzzy.py +454 -0
  13. {redup-0.3.20 → redup-0.3.22}/src/redup.egg-info/PKG-INFO +2 -2
  14. {redup-0.3.20 → redup-0.3.22}/src/redup.egg-info/SOURCES.txt +6 -0
  15. {redup-0.3.20 → redup-0.3.22}/LICENSE +0 -0
  16. {redup-0.3.20 → redup-0.3.22}/setup.cfg +0 -0
  17. {redup-0.3.20 → redup-0.3.22}/src/redup/__main__.py +0 -0
  18. {redup-0.3.20 → redup-0.3.22}/src/redup/cli_app/__init__.py +0 -0
  19. {redup-0.3.20 → redup-0.3.22}/src/redup/core/__init__.py +0 -0
  20. {redup-0.3.20 → redup-0.3.22}/src/redup/core/cache.py +0 -0
  21. {redup-0.3.20 → redup-0.3.22}/src/redup/core/config.py +0 -0
  22. {redup-0.3.20 → redup-0.3.22}/src/redup/core/differ.py +0 -0
  23. {redup-0.3.20 → redup-0.3.22}/src/redup/core/hash_cache.py +0 -0
  24. {redup-0.3.20 → redup-0.3.22}/src/redup/core/hasher.py +0 -0
  25. {redup-0.3.20 → redup-0.3.22}/src/redup/core/lazy_grouper.py +0 -0
  26. {redup-0.3.20 → redup-0.3.22}/src/redup/core/lsh_matcher.py +0 -0
  27. {redup-0.3.20 → redup-0.3.22}/src/redup/core/matcher.py +0 -0
  28. {redup-0.3.20 → redup-0.3.22}/src/redup/core/memory_scanner.py +0 -0
  29. {redup-0.3.20 → redup-0.3.22}/src/redup/core/parallel_scanner.py +0 -0
  30. {redup-0.3.20 → redup-0.3.22}/src/redup/core/pipeline.py +0 -0
  31. {redup-0.3.20 → redup-0.3.22}/src/redup/core/planner.py +0 -0
  32. {redup-0.3.20 → redup-0.3.22}/src/redup/core/scanner.py +0 -0
  33. {redup-0.3.20 → redup-0.3.22}/src/redup/core/ts_extractor.py +0 -0
  34. {redup-0.3.20 → redup-0.3.22}/src/redup/core/ultra_fast_scanner.py +0 -0
  35. {redup-0.3.20 → redup-0.3.22}/src/redup/core/utils/__init__.py +0 -0
  36. {redup-0.3.20 → redup-0.3.22}/src/redup/core/utils/duplicate_finders.py +0 -0
  37. {redup-0.3.20 → redup-0.3.22}/src/redup/core/utils/function_extractor.py +0 -0
  38. {redup-0.3.20 → redup-0.3.22}/src/redup/core/utils/hash_utils.py +0 -0
  39. {redup-0.3.20 → redup-0.3.22}/src/redup/core/utils/language_dispatcher.py +0 -0
  40. {redup-0.3.20 → redup-0.3.22}/src/redup/reporters/__init__.py +0 -0
  41. {redup-0.3.20 → redup-0.3.22}/src/redup/reporters/code2llm_reporter.py +0 -0
  42. {redup-0.3.20 → redup-0.3.22}/src/redup/reporters/enhanced_reporter.py +0 -0
  43. {redup-0.3.20 → redup-0.3.22}/src/redup/reporters/json_reporter.py +0 -0
  44. {redup-0.3.20 → redup-0.3.22}/src/redup/reporters/markdown_reporter.py +0 -0
  45. {redup-0.3.20 → redup-0.3.22}/src/redup/reporters/toon_reporter.py +0 -0
  46. {redup-0.3.20 → redup-0.3.22}/src/redup/reporters/yaml_reporter.py +0 -0
  47. {redup-0.3.20 → redup-0.3.22}/src/redup.egg-info/dependency_links.txt +0 -0
  48. {redup-0.3.20 → redup-0.3.22}/src/redup.egg-info/entry_points.txt +0 -0
  49. {redup-0.3.20 → redup-0.3.22}/src/redup.egg-info/requires.txt +0 -0
  50. {redup-0.3.20 → redup-0.3.22}/src/redup.egg-info/top_level.txt +0 -0
  51. {redup-0.3.20 → redup-0.3.22}/tests/test_e2e.py +0 -0
  52. {redup-0.3.20 → redup-0.3.22}/tests/test_hasher.py +0 -0
  53. {redup-0.3.20 → redup-0.3.22}/tests/test_matcher.py +0 -0
  54. {redup-0.3.20 → redup-0.3.22}/tests/test_models.py +0 -0
  55. {redup-0.3.20 → redup-0.3.22}/tests/test_pipeline.py +0 -0
  56. {redup-0.3.20 → redup-0.3.22}/tests/test_planner.py +0 -0
  57. {redup-0.3.20 → redup-0.3.22}/tests/test_reporters.py +0 -0
  58. {redup-0.3.20 → redup-0.3.22}/tests/test_scanner.py +0 -0
  59. {redup-0.3.20 → redup-0.3.22}/tests/test_ts_extractor.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: redup
3
- Version: 0.3.20
3
+ Version: 0.3.22
4
4
  Summary: Code duplication analyzer and refactoring planner for LLMs
5
5
  Author-email: Tom Sapletta <tom@sapletta.com>
6
6
  License-Expression: Apache-2.0
@@ -83,7 +83,7 @@ Dynamic: license-file
83
83
  [![PyPI](https://img.shields.io/pypi/v/redup)](https://pypi.org/project/redup/)
84
84
  [![License: Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
85
85
  [![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://python.org)
86
- [![Version](https://img.shields.io/badge/version-0.3.20-green.svg)](https://pypi.org/project/redup/)
86
+ [![Version](https://img.shields.io/badge/version-0.3.22-green.svg)](https://pypi.org/project/redup/)
87
87
 
88
88
  reDUP scans codebases for duplicated functions, blocks, and structural patterns — then builds a prioritized refactoring map that LLMs can consume to eliminate redundancy systematically.
89
89
 
@@ -5,7 +5,7 @@
5
5
  [![PyPI](https://img.shields.io/pypi/v/redup)](https://pypi.org/project/redup/)
6
6
  [![License: Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
7
7
  [![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://python.org)
8
- [![Version](https://img.shields.io/badge/version-0.3.20-green.svg)](https://pypi.org/project/redup/)
8
+ [![Version](https://img.shields.io/badge/version-0.3.22-green.svg)](https://pypi.org/project/redup/)
9
9
 
10
10
  reDUP scans codebases for duplicated functions, blocks, and structural patterns — then builds a prioritized refactoring map that LLMs can consume to eliminate redundancy systematically.
11
11
 
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "redup"
7
- version = "0.3.20"
7
+ version = "0.3.22"
8
8
  description = "Code duplication analyzer and refactoring planner for LLMs"
9
9
  readme = "README.md"
10
10
  license = "Apache-2.0"
@@ -2,7 +2,7 @@
2
2
 
3
3
  from __future__ import annotations
4
4
 
5
- __version__ = "0.3.20"
5
+ __version__ = "0.3.22"
6
6
 
7
7
  from redup.core.models import (
8
8
  DuplicateFragment,
@@ -0,0 +1,51 @@
1
+ """Configuration builders for reDUP CLI."""
2
+
3
+ from pathlib import Path
4
+ from typing import Any
5
+
6
+ from redup.core.config import config_to_scan_config, load_config
7
+ from redup.core.models import ScanConfig
8
+
9
+
10
+ def build_config(
11
+ path: Path,
12
+ extensions: str,
13
+ min_lines: int,
14
+ min_similarity: float,
15
+ include_tests: bool
16
+ ) -> ScanConfig:
17
+ """Build basic scan configuration."""
18
+ config = load_config()
19
+ return config_to_scan_config(config, path)
20
+
21
+
22
+ def build_config_with_file_support(
23
+ path: Path,
24
+ extensions: Any,
25
+ min_lines: Any,
26
+ min_similarity: Any,
27
+ include_tests: Any,
28
+ parallel: bool = False,
29
+ max_workers: Any = None,
30
+ incremental: bool = False,
31
+ memory_cache: bool = False,
32
+ max_cache_mb: int = 512,
33
+ functions_only: bool = False
34
+ ) -> ScanConfig:
35
+ """Build scan configuration with advanced options."""
36
+ config = load_config()
37
+ scan_config = config_to_scan_config(config, path)
38
+
39
+ # Override with CLI parameters if provided
40
+ if extensions is not None:
41
+ scan_config.extensions = [e.strip() for e in extensions.split(',')]
42
+ if min_lines is not None:
43
+ scan_config.min_lines = min_lines
44
+ if min_similarity is not None:
45
+ scan_config.min_similarity = min_similarity
46
+ if include_tests is not None:
47
+ scan_config.include_tests = include_tests
48
+ if functions_only:
49
+ scan_config.function_level_only = functions_only
50
+
51
+ return scan_config
@@ -13,15 +13,17 @@ import typer
13
13
  warnings.filterwarnings("ignore", category=SyntaxWarning)
14
14
 
15
15
  import redup # noqa: E402
16
- from redup.core.config import config_to_scan_config, create_sample_redup_toml, load_config # noqa: E402
16
+ from redup.core.config import create_sample_redup_toml, load_config # noqa: E402
17
17
  from redup.core.models import DuplicationMap, ScanConfig # noqa: E402
18
- from redup.core.pipeline import analyze, analyze_parallel, analyze_optimized # noqa: E402
19
- from redup.reporters.json_reporter import to_json # noqa: E402
20
- from redup.reporters.markdown_reporter import to_markdown # noqa: E402
21
- from redup.reporters.toon_reporter import to_toon # noqa: E402
22
- from redup.reporters.yaml_reporter import to_yaml # noqa: E402
23
- from redup.reporters.code2llm_reporter import export_code2llm, to_code2llm_context, to_code2llm_toon # noqa: E402
24
- from redup.reporters.enhanced_reporter import EnhancedReporter # noqa: E402
18
+ from redup.core.differ import compare_scans, format_diff_result # noqa: E402
19
+
20
+ # Import refactored modules
21
+ from redup.cli_app.config_builder import build_config, build_config_with_file_support # noqa: E402
22
+ from redup.cli_app.output_writer import write_output, write_results # noqa: E402
23
+ from redup.cli_app.scan_commands import ( # noqa: E402
24
+ scan_command, diff_command, check_command, config_command, info_command
25
+ )
26
+ from redup.cli_app.scan_helpers import print_scan_header, print_scan_summary # noqa: E402
25
27
 
26
28
  app = typer.Typer(
27
29
  name="redup",
@@ -142,6 +144,160 @@ def _print_scan_summary(dup_map: DuplicationMap) -> None:
142
144
  typer.echo("")
143
145
 
144
146
 
147
+ def _apply_fuzzy_similarity(dup_map: DuplicationMap, threshold: float) -> DuplicationMap:
148
+ """Apply fuzzy similarity detection to find similar components across all languages."""
149
+ validation = _validate_fuzzy_input(dup_map, threshold)
150
+ if not validation.is_valid:
151
+ typer.echo(validation.message)
152
+ return dup_map
153
+
154
+ all_blocks = _extract_all_blocks(dup_map)
155
+ if not all_blocks:
156
+ typer.echo("🔍 No code blocks found for fuzzy analysis")
157
+ return dup_map
158
+
159
+ typer.echo(f"🔍 Analyzing {len(all_blocks)} code blocks for universal fuzzy similarity...")
160
+
161
+ html_css_blocks, other_blocks = _separate_blocks_by_type(all_blocks)
162
+ similar_pairs = _analyze_blocks_with_detectors(html_css_blocks, other_blocks, threshold)
163
+
164
+ _report_fuzzy_results(similar_pairs)
165
+ return dup_map
166
+
167
+
168
+ def _validate_fuzzy_input(dup_map: DuplicationMap, threshold: float) -> 'FuzzyValidationResult':
169
+ """Validate input parameters for fuzzy similarity analysis."""
170
+ from dataclasses import dataclass
171
+
172
+ @dataclass
173
+ class FuzzyValidationResult:
174
+ is_valid: bool
175
+ message: str = ""
176
+
177
+ if threshold <= 0 or threshold > 1:
178
+ return FuzzyValidationResult(False, "❌ Invalid threshold. Must be between 0 and 1.")
179
+
180
+ if not dup_map.groups:
181
+ return FuzzyValidationResult(False, "❌ No duplicate groups found.")
182
+
183
+ return FuzzyValidationResult(True)
184
+
185
+
186
+ def _extract_all_blocks(dup_map: DuplicationMap) -> list:
187
+ """Extract all code blocks from duplicate fragments."""
188
+ from redup.core.scanner import CodeBlock
189
+
190
+ all_blocks = []
191
+ for group in dup_map.groups:
192
+ for fragment in group.fragments:
193
+ # Create a CodeBlock from fragment data
194
+ block = CodeBlock(
195
+ file=fragment.file,
196
+ line_start=fragment.line_start,
197
+ line_end=fragment.line_end,
198
+ text=fragment.text,
199
+ function_name=fragment.function_name,
200
+ class_name=fragment.class_name
201
+ )
202
+ all_blocks.append(block)
203
+ return all_blocks
204
+
205
+
206
+ def _separate_blocks_by_type(all_blocks: list) -> tuple[list, list]:
207
+ """Separate blocks into HTML/CSS and other language categories."""
208
+ html_css_blocks = [block for block in all_blocks
209
+ if Path(block.file).suffix.lower() in ['.html', '.htm', '.css', '.scss', '.sass', '.less']]
210
+
211
+ other_blocks = [block for block in all_blocks if block not in html_css_blocks]
212
+
213
+ return html_css_blocks, other_blocks
214
+
215
+
216
+ def _analyze_blocks_with_detectors(html_css_blocks: list, other_blocks: list, threshold: float) -> list:
217
+ """Analyze blocks using appropriate fuzzy similarity detectors."""
218
+ similar_pairs = []
219
+
220
+ # Analyze HTML/CSS with specialized detector
221
+ if html_css_blocks:
222
+ typer.echo(f" 📝 Analyzing {len(html_css_blocks)} HTML/CSS blocks...")
223
+ html_css_similar = _analyze_html_css_blocks(html_css_blocks, threshold)
224
+ similar_pairs.extend(html_css_similar)
225
+ typer.echo(f" ✨ Found {len(html_css_similar)} similar HTML/CSS component pairs")
226
+
227
+ # Analyze other languages with universal detector
228
+ if other_blocks:
229
+ typer.echo(f" 🔧 Analyzing {len(other_blocks)} blocks from other languages...")
230
+ universal_similar = _analyze_other_language_blocks(other_blocks, threshold)
231
+ similar_pairs.extend(universal_similar)
232
+ typer.echo(f" ✨ Found {len(universal_similar)} similar component pairs from other languages")
233
+
234
+ return similar_pairs
235
+
236
+
237
+ def _analyze_html_css_blocks(html_css_blocks: list, threshold: float) -> list:
238
+ """Analyze HTML/CSS blocks using specialized fuzzy similarity detector."""
239
+ from redup.core.fuzzy_similarity import FuzzySimilarityDetector
240
+
241
+ html_css_detector = FuzzySimilarityDetector(similarity_threshold=threshold)
242
+ return html_css_detector.find_similar_components(html_css_blocks)
243
+
244
+
245
+ def _analyze_other_language_blocks(other_blocks: list, threshold: float) -> list:
246
+ """Analyze non-HTML/CSS blocks using universal fuzzy similarity detector."""
247
+ from redup.core.universal_fuzzy import UniversalFuzzyDetector
248
+
249
+ universal_detector = UniversalFuzzyDetector(similarity_threshold=threshold)
250
+ return universal_detector.find_similar_components(other_blocks)
251
+
252
+
253
+ def _report_fuzzy_results(similar_pairs: list) -> None:
254
+ """Report fuzzy similarity analysis results."""
255
+ if similar_pairs:
256
+ typer.echo(f"🎯 Total fuzzy similar components found: {len(similar_pairs)}")
257
+
258
+ # Simplified reporting to reduce complexity
259
+ typer.echo(f" 📊 Language groups: {len(set(Path(b1.file).suffix + '+' + Path(b2.file).suffix for b1, b2, _ in similar_pairs))}")
260
+
261
+ # Show top examples only
262
+ for block1, block2, similarity in similar_pairs[:5]:
263
+ typer.echo(f" {block1.file} ↔ {block2.file}: {similarity:.2f}")
264
+
265
+ if len(similar_pairs) > 5:
266
+ typer.echo(f" ... and {len(similar_pairs) - 5} more")
267
+
268
+ # Note: In production, you'd integrate these into DuplicationMap
269
+ # For now, we just report the findings
270
+
271
+ else:
272
+ typer.echo("🔍 No fuzzy similar components found")
273
+
274
+
275
+ def _report_similarity_by_groups(similar_pairs):
276
+ """Extracted function to reduce complexity."""
277
+ lang_groups = {}
278
+ for block1, block2, similarity in similar_pairs:
279
+ lang1 = Path(block1.file).suffix
280
+ lang2 = Path(block2.file).suffix
281
+ comp_type = "unknown"
282
+
283
+ # Try to get component type
284
+ try:
285
+ from redup.core.universal_fuzzy import UniversalFuzzyExtractor
286
+ extractor = UniversalFuzzyExtractor()
287
+ sig1 = extractor.extract_universal_signature(block1)
288
+ if sig1:
289
+ comp_type = sig1.component_type
290
+ except:
291
+ pass
292
+
293
+ group_key = f"{lang1}+{lang2}:{comp_type}"
294
+ if group_key not in lang_groups:
295
+ lang_groups[group_key] = []
296
+ lang_groups[group_key].append((block1, block2, similarity))
297
+
298
+ return lang_groups
299
+
300
+
145
301
  def _write_results(
146
302
  dup_map: DuplicationMap,
147
303
  format: str,
@@ -247,6 +403,44 @@ def scan(
247
403
  "--parallel/--no-parallel",
248
404
  help="Use parallel scanning for large projects (default: disabled due to issues).",
249
405
  ),
406
+ max_workers: int | None = typer.Option(
407
+ None,
408
+ "--max-workers",
409
+ help="Maximum number of parallel workers.",
410
+ ),
411
+ incremental: bool = typer.Option(
412
+ False,
413
+ "--incremental",
414
+ help="Use incremental scanning with caching.",
415
+ ),
416
+ memory_cache: bool = typer.Option(
417
+ True,
418
+ "--memory-cache/--no-memory-cache",
419
+ help="Use memory cache for faster scanning.",
420
+ ),
421
+ max_cache_mb: int = typer.Option(
422
+ 512,
423
+ "--max-cache-mb",
424
+ help="Maximum memory cache size in MB.",
425
+ ),
426
+ fuzzy: bool = typer.Option(
427
+ False,
428
+ "--fuzzy",
429
+ help="Enable fuzzy similarity detection.",
430
+ ),
431
+ fuzzy_threshold: float = typer.Option(
432
+ 0.8,
433
+ "--fuzzy-threshold",
434
+ help="Fuzzy similarity threshold (0.0-1.0).",
435
+ ),
436
+ ) -> None:
437
+ """Scan a project for code duplicates."""
438
+ return scan_command(
439
+ path, format, output, extensions, min_lines, min_similarity,
440
+ include_tests, functions_only, parallel, max_workers,
441
+ incremental, memory_cache, max_cache_mb, fuzzy, fuzzy_threshold
442
+ )
443
+ ),
250
444
  max_workers: int = typer.Option(
251
445
  None,
252
446
  "--max-workers",
@@ -267,6 +461,16 @@ def scan(
267
461
  "--max-cache-mb",
268
462
  help="Maximum memory for file caching in MB (default: 512).",
269
463
  ),
464
+ fuzzy: bool = typer.Option(
465
+ False,
466
+ "--fuzzy/--no-fuzzy",
467
+ help="Enable fuzzy similarity detection for HTML/CSS components (default: disabled).",
468
+ ),
469
+ fuzzy_threshold: float = typer.Option(
470
+ 0.8,
471
+ "--fuzzy-threshold",
472
+ help="Fuzzy similarity threshold (0.0-1.0, default: 0.8).",
473
+ ),
270
474
  ) -> None:
271
475
  """Scan a project for code duplicates and generate a refactoring map.
272
476
 
@@ -284,6 +488,10 @@ def scan(
284
488
  parallel, max_workers, incremental, memory_cache, max_cache_mb, functions_only
285
489
  )
286
490
 
491
+ # Add fuzzy options to config
492
+ config.fuzzy_enabled = fuzzy
493
+ config.fuzzy_threshold = fuzzy_threshold
494
+
287
495
  _print_scan_header(path, config.extensions, config.min_block_lines, config.min_similarity)
288
496
 
289
497
  # Choose analysis method
@@ -292,6 +500,10 @@ def scan(
292
500
  else:
293
501
  dup_map = analyze(config=config, function_level_only=functions_only)
294
502
 
503
+ # Apply fuzzy similarity detection if enabled
504
+ if fuzzy:
505
+ dup_map = _apply_fuzzy_similarity(dup_map, fuzzy_threshold)
506
+
295
507
  _print_scan_summary(dup_map)
296
508
  _write_results(dup_map, format, output, path)
297
509
 
@@ -0,0 +1,78 @@
1
+ """Output writers for reDUP CLI results."""
2
+
3
+ from pathlib import Path
4
+ from typing import Any
5
+
6
+ from redup.core.models import DuplicationMap
7
+ from redup.reporters.json_reporter import to_json
8
+ from redup.reporters.markdown_reporter import to_markdown
9
+ from redup.reporters.toon_reporter import to_toon
10
+ from redup.reporters.yaml_reporter import to_yaml
11
+ from redup.reporters.code2llm_reporter import export_code2llm, to_code2llm_context, to_code2llm_toon
12
+ from redup.reporters.enhanced_reporter import EnhancedReporter
13
+
14
+
15
+ def write_output(content: str, output: Path | None, suffix: str) -> None:
16
+ """Write content to file or stdout."""
17
+ import typer
18
+
19
+ if output:
20
+ target = output if output.suffix else output / f"duplication.{suffix}"
21
+ target.parent.mkdir(parents=True, exist_ok=True)
22
+ target.write_text(content, encoding="utf-8")
23
+ typer.echo(f" → {target}")
24
+ else:
25
+ typer.echo(content)
26
+
27
+
28
+ def write_results(dup_map: DuplicationMap, format: str, output: Any, path: Path) -> None:
29
+ """Write analysis results in specified format."""
30
+ import typer
31
+
32
+ if format == "json":
33
+ content = to_json(dup_map)
34
+ write_output(content, output, "json")
35
+ elif format == "yaml":
36
+ content = to_yaml(dup_map)
37
+ write_output(content, output, "yaml")
38
+ elif format == "toon":
39
+ content = to_toon(dup_map)
40
+ write_output(content, output, "toon")
41
+ elif format == "markdown":
42
+ content = to_markdown(dup_map)
43
+ write_output(content, output, "md")
44
+ elif format == "enhanced":
45
+ reporter = EnhancedReporter(dup_map)
46
+ content = reporter.generate_report()
47
+ write_output(content, output, "html")
48
+ elif format == "code2llm":
49
+ if output:
50
+ content = to_code2llm_context(dup_map)
51
+ write_output(content, output, "md")
52
+ else:
53
+ # For code2llm, create a directory with multiple files
54
+ export_code2llm(dup_map, output or path / "code2llm")
55
+ elif format == "all":
56
+ # Generate all formats
57
+ formats = {
58
+ "json": to_json(dup_map),
59
+ "yaml": to_yaml(dup_map),
60
+ "toon": to_toon(dup_map),
61
+ "markdown": to_markdown(dup_map),
62
+ }
63
+
64
+ if output and output.is_dir():
65
+ for fmt, content in formats.items():
66
+ target = output / f"duplication.{fmt}"
67
+ target.write_text(content, encoding="utf-8")
68
+ typer.echo(f" → {target}")
69
+ else:
70
+ # Write to separate files in the same directory
71
+ base_dir = output.parent if output else path
72
+ for fmt, content in formats.items():
73
+ target = base_dir / f"duplication.{fmt}"
74
+ target.write_text(content, encoding="utf-8")
75
+ typer.echo(f" → {target}")
76
+ else:
77
+ typer.echo(f"❌ Unknown format: {format}", err=True)
78
+ raise typer.Exit(1)
@@ -0,0 +1,163 @@
1
+ """Scan command implementations for reDUP CLI."""
2
+
3
+ from pathlib import Path
4
+ from typing import Any
5
+
6
+ import typer
7
+
8
+ from redup.core.config import create_sample_redup_toml
9
+ from redup.core.models import DuplicationMap
10
+ from redup.core.pipeline import analyze, analyze_parallel, analyze_optimized
11
+ from redup.cli_app.config_builder import build_config, build_config_with_file_support
12
+ from redup.cli_app.output_writer import write_output
13
+ from redup.cli_app.scan_helpers import print_scan_header, print_scan_summary, apply_fuzzy_similarity
14
+
15
+
16
+ def scan_command(
17
+ path: Path = typer.Argument(Path("."), help="Project root directory to scan.", exists=True, dir_okay=True, file_okay=False),
18
+ format: str = typer.Option("toon", "--format", "-f", help="Output format (json, yaml, toon, markdown, code2llm, all, enhanced)."),
19
+ output: Any = typer.Option(None, "--output", "-o", help="Output directory or file path. Defaults to stdout."),
20
+ extensions: Any = typer.Option(None, "--ext", "-e", help="Comma-separated file extensions to scan. Overrides config."),
21
+ min_lines: Any = typer.Option(None, "--min-lines", help="Minimum block size (lines). Overrides config."),
22
+ min_similarity: Any = typer.Option(None, "--min-sim", help="Minimum similarity score (0.0-1.0). Overrides config."),
23
+ include_tests: bool = typer.Option(False, "--include-tests", help="Include test files in analysis."),
24
+ functions_only: bool = typer.Option(False, "--functions-only", help="Only analyze function-level duplicates."),
25
+ parallel: bool = typer.Option(False, "--parallel", help="Use parallel processing for faster analysis."),
26
+ max_workers: Any = typer.Option(None, "--max-workers", help="Maximum number of parallel workers."),
27
+ incremental: bool = typer.Option(False, "--incremental", help="Use incremental scanning (cache-based)."),
28
+ memory_cache: bool = typer.Option(True, "--memory-cache/--no-memory-cache", help="Use memory cache for faster processing."),
29
+ max_cache_mb: int = typer.Option(512, "--max-cache-mb", help="Maximum memory cache size in MB."),
30
+ fuzzy: bool = typer.Option(False, "--fuzzy", help="Enable fuzzy similarity detection."),
31
+ fuzzy_threshold: float = typer.Option(0.8, "--fuzzy-threshold", help="Fuzzy similarity threshold (0.0-1.0)."),
32
+ ) -> None:
33
+ """Scan a project for code duplicates."""
34
+
35
+ # Build configuration
36
+ if any([extensions, min_lines, min_similarity, include_tests, functions_only, parallel, max_workers, incremental, memory_cache, max_cache_mb]):
37
+ config = build_config_with_file_support(
38
+ path, extensions, min_lines, min_similarity, include_tests,
39
+ parallel, max_workers, incremental, memory_cache, max_cache_mb, functions_only
40
+ )
41
+ else:
42
+ config = build_config(path, extensions, min_lines, min_similarity, include_tests)
43
+
44
+ # Print scan header
45
+ print_scan_header(path, config.extensions, config.min_lines, config.min_similarity)
46
+
47
+ # Run analysis
48
+ if parallel and max_workers and max_workers > 1:
49
+ dup_map = analyze_parallel(config, max_workers)
50
+ elif memory_cache:
51
+ dup_map = analyze_optimized(config, use_memory_cache=True, max_cache_mb=max_cache_mb)
52
+ else:
53
+ dup_map = analyze(config, function_level_only=functions_only)
54
+
55
+ # Apply fuzzy similarity if requested
56
+ if fuzzy:
57
+ dup_map = apply_fuzzy_similarity(dup_map, fuzzy_threshold)
58
+
59
+ # Print summary
60
+ print_scan_summary(dup_map)
61
+
62
+ # Write results
63
+ from redup.cli_app.output_writer import write_results
64
+ write_results(dup_map, format, output, path)
65
+
66
+
67
+ def diff_command(before: Path, after: Path) -> None:
68
+ """Compare two reDUP analysis results."""
69
+ from redup.core.differ import compare_scans, format_diff_result
70
+
71
+ try:
72
+ diff = compare_scans(before, after)
73
+ result = format_diff_result(diff)
74
+ typer.echo(result)
75
+ except Exception as e:
76
+ typer.echo(f"❌ Error comparing scans: {e}", err=True)
77
+ raise typer.Exit(1)
78
+
79
+
80
+ def check_command(
81
+ path: Path = typer.Argument(Path("."), help="Project root directory to check.", exists=True, dir_okay=True, file_okay=False),
82
+ max_groups: int = typer.Option(10, "--max-groups", help="Maximum duplicate groups to report."),
83
+ max_saved_lines: int = typer.Option(100, "--max-saved-lines", help="Maximum saved lines to report."),
84
+ extensions: Any = typer.Option(None, "--ext", "-e", help="Comma-separated file extensions to scan."),
85
+ min_lines: Any = typer.Option(None, "--min-lines", help="Minimum block size (lines)."),
86
+ min_similarity: Any = typer.Option(None, "--min-sim", help="Minimum similarity score (0.0-1.0)."),
87
+ include_tests: bool = typer.Option(False, "--include-tests", help="Include test files."),
88
+ ) -> None:
89
+ """Quick check for duplicates with summary report."""
90
+
91
+ config = build_config_with_file_support(
92
+ path, extensions, min_lines, min_similarity, include_tests
93
+ )
94
+
95
+ print_scan_header(path, config.extensions, config.min_lines, config.min_similarity)
96
+
97
+ dup_map = analyze(config)
98
+ print_scan_summary(dup_map)
99
+
100
+ # Show top duplicate groups
101
+ if dup_map.groups:
102
+ typer.echo(f"\n🎯 Top {min(max_groups, len(dup_map.groups))} duplicate groups:")
103
+ for i, group in enumerate(dup_map.sorted_by_impact()[:max_groups]):
104
+ if group.saved_lines_potential <= max_saved_lines:
105
+ break
106
+ typer.echo(f" {i+1}. {group.id}: {group.occurrences} occurrences, {group.saved_lines_potential} lines recoverable")
107
+ else:
108
+ typer.echo("✅ No duplicates found!")
109
+
110
+
111
+ def config_command(init: bool = typer.Option(False, "--init", help="Create sample redup.toml configuration."),
112
+ show: bool = typer.Option(False, "--show", help="Show current configuration.")) -> None:
113
+ """Manage reDUP configuration."""
114
+
115
+ if init:
116
+ config_path = Path("redup.toml")
117
+ if config_path.exists():
118
+ typer.echo("⚠️ redup.toml already exists. Overwrite? [y/N]", nl=False)
119
+ response = input().strip().lower()
120
+ if response != 'y':
121
+ typer.echo("Cancelled.")
122
+ return
123
+
124
+ content = create_sample_redup_toml()
125
+ config_path.write_text(content, encoding="utf-8")
126
+ typer.echo(f"✅ Created sample configuration: {config_path}")
127
+
128
+ elif show:
129
+ from redup.core.config import load_config
130
+ config = load_config()
131
+ typer.echo("Current configuration:")
132
+ typer.echo(f" Extensions: {config.get('extensions', ['.py'])}")
133
+ typer.echo(f" Min lines: {config.get('min_lines', 3)}")
134
+ typer.echo(f" Min similarity: {config.get('min_similarity', 0.8)}")
135
+ typer.echo(f" Include tests: {config.get('include_tests', False)}")
136
+
137
+ else:
138
+ typer.echo("Use --init to create config or --show to display current config.")
139
+
140
+
141
+ def info_command() -> None:
142
+ """Show reDUP version and system information."""
143
+ import redup
144
+ import sys
145
+
146
+ typer.echo(f"reDUP version: {redup.__version__}")
147
+ typer.echo(f"Python version: {sys.version}")
148
+ typer.echo(f"Installation path: {Path(redup.__file__).parent}")
149
+
150
+ # Check optional dependencies
151
+ deps = {
152
+ "tree-sitter": "tree_sitter",
153
+ "datasketch": "datasketch",
154
+ "rapidfuzz": "rapidfuzz",
155
+ }
156
+
157
+ typer.echo("\nOptional dependencies:")
158
+ for name, module in deps.items():
159
+ try:
160
+ __import__(module)
161
+ typer.echo(f" ✅ {name}")
162
+ except ImportError:
163
+ typer.echo(f" ❌ {name} (install with: pip install {name})")
@@ -0,0 +1,34 @@
1
+ """Helper functions for scan operations."""
2
+
3
+ from pathlib import Path
4
+ from typing import List
5
+
6
+ import typer
7
+
8
+ from redup.core.models import DuplicationMap
9
+ from redup.cli_app.main import _apply_fuzzy_similarity
10
+
11
+
12
+ def print_scan_header(path: Path, ext_list: List[str], min_lines: int, min_similarity: float) -> None:
13
+ """Print scan operation header."""
14
+ typer.echo(f"🔍 Scanning: {path}")
15
+ typer.echo(f"📁 Extensions: {', '.join(ext_list)}")
16
+ typer.echo(f"📏 Min lines: {min_lines}")
17
+ typer.echo(f"🎯 Min similarity: {min_similarity}")
18
+ typer.echo("")
19
+
20
+
21
+ def print_scan_summary(dup_map: DuplicationMap) -> None:
22
+ """Print scan operation summary."""
23
+ typer.echo(f"📊 Scanned {dup_map.stats.files_scanned} files "
24
+ f"({dup_map.stats.total_lines} lines, "
25
+ f"{dup_map.stats.scan_time_ms:.0f}ms)")
26
+ typer.echo(f"Found {dup_map.total_groups} duplicate groups "
27
+ f"({dup_map.total_fragments} fragments, "
28
+ f"{dup_map.total_saved_lines} lines recoverable)")
29
+ typer.echo("")
30
+
31
+
32
+ def apply_fuzzy_similarity(dup_map: DuplicationMap, threshold: float) -> DuplicationMap:
33
+ """Apply fuzzy similarity detection (wrapper for main function)."""
34
+ return _apply_fuzzy_similarity(dup_map, threshold)