treepeat 0.0.1__tar.gz

treepeat-0.0.1/PKG-INFO

@@ -0,0 +1,89 @@
+Metadata-Version: 2.4
+Name: treepeat
+Version: 0.0.1
+Summary: treepeat: a treesitter-based CLI tool to detect similar code
+Author-email: Dane Summers <dsummersl@gmail.com>
+License-Expression: Apache-2.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Typing :: Typed
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: click>=8.3.0
+Requires-Dist: datasketch>=1.6.5
+Requires-Dist: pydantic>=2.12.3
+Requires-Dist: pydantic-settings>=2.0.0
+Requires-Dist: python-magic>=0.4.27
+Requires-Dist: rich>=14.2.0
+Requires-Dist: sarif-pydantic>=0.6.2
+Requires-Dist: tdqm>=0.0.1
+Requires-Dist: tree-sitter>=0.25.2
+Requires-Dist: tree-sitter-language-pack>=0.10.0
+
+# Treepeat - finding groups of similar code
+
+`treepeat` is a CLI tool that detects similar code using treesitter AST analysis with locality-sensitive hashing:
+- Find **duplicate** code blocks meaningful to the language (classes/functions), not just lines.
+- near-duplicates
+- structurally similar text.
+
+Helpers: This is very much an proof of concept - I'm happy with it, but I haven't supported very many languages at present. PRs welcome!
+
+## Usage
+
+### detect
+
+Scan a codebase for similar or duplicate code blocks using tree-sitter AST analysis and locality-sensitive hashing.
+
+Key flags:
+- `--ruleset`: Normalization ruleset to use (`none`, `default`, `loose`) - controls how code is normalized before comparison
+- `--similarity`: Percent similarity from 1-100 (default: 100 for exact duplicates)
+- `--min-lines`: Minimum number of lines for a match (default: 5)
+- `--diff`: Show side-by-side comparisons of similar blocks
+- `--format`: Output format - `console` (default) or `sarif` for CI integration
+
+```bash
+# Find exact duplicates
+treepeat detect /path/to/codebase
+
+# Find near-duplicates with 80% similarity threshold
+treepeat detect --similarity 80 /path/to/codebase
+
+# Show diffs between similar blocks and use loose ruleset
+treepeat --ruleset loose detect --diff --min-lines 10 /path/to/codebase
+
+# Output results in SARIF format for CI tools
+treepeat detect --format sarif -o results.sarif /path/to/codebase
+```
+
+### Other sub commands
+
+#### list-ruleset
+
+List all rules in a ruleset, along with their descriptions. Use `--language` to see which rules apply to a specific language.
+
+#### treesitter
+
+Display how treepeat normalizes source code into tree-sitter tokens for similarity detection -- helpful for debugging why a certain section of a file might be similar to another. Shows the original source code side-by-side with the normalized token representation.
+
+## Dev setup
+
+```bash
+make setup
+make test
+```
+
+## Benchmarks
+
+treepeat includes a testing framework for comparing duplication detection tools against real-world codebases.
+
+```bash
+# Run all benchmark tests
+make benchmark
+
+# Compare results across tools
+make benchmark-compare
+```
+
+## ADRs
+
+Architecture Decision Records live in docs/adr.

treepeat-0.0.1/README.md

@@ -0,0 +1,68 @@
+# Treepeat - finding groups of similar code
+
+`treepeat` is a CLI tool that detects similar code using treesitter AST analysis with locality-sensitive hashing:
+- Find **duplicate** code blocks meaningful to the language (classes/functions), not just lines.
+- near-duplicates
+- structurally similar text.
+
+Helpers: This is very much an proof of concept - I'm happy with it, but I haven't supported very many languages at present. PRs welcome!
+
+## Usage
+
+### detect
+
+Scan a codebase for similar or duplicate code blocks using tree-sitter AST analysis and locality-sensitive hashing.
+
+Key flags:
+- `--ruleset`: Normalization ruleset to use (`none`, `default`, `loose`) - controls how code is normalized before comparison
+- `--similarity`: Percent similarity from 1-100 (default: 100 for exact duplicates)
+- `--min-lines`: Minimum number of lines for a match (default: 5)
+- `--diff`: Show side-by-side comparisons of similar blocks
+- `--format`: Output format - `console` (default) or `sarif` for CI integration
+
+```bash
+# Find exact duplicates
+treepeat detect /path/to/codebase
+
+# Find near-duplicates with 80% similarity threshold
+treepeat detect --similarity 80 /path/to/codebase
+
+# Show diffs between similar blocks and use loose ruleset
+treepeat --ruleset loose detect --diff --min-lines 10 /path/to/codebase
+
+# Output results in SARIF format for CI tools
+treepeat detect --format sarif -o results.sarif /path/to/codebase
+```
+
+### Other sub commands
+
+#### list-ruleset
+
+List all rules in a ruleset, along with their descriptions. Use `--language` to see which rules apply to a specific language.
+
+#### treesitter
+
+Display how treepeat normalizes source code into tree-sitter tokens for similarity detection -- helpful for debugging why a certain section of a file might be similar to another. Shows the original source code side-by-side with the normalized token representation.
+
+## Dev setup
+
+```bash
+make setup
+make test
+```
+
+## Benchmarks
+
+treepeat includes a testing framework for comparing duplication detection tools against real-world codebases.
+
+```bash
+# Run all benchmark tests
+make benchmark
+
+# Compare results across tools
+make benchmark-compare
+```
+
+## ADRs
+
+Architecture Decision Records live in docs/adr.

treepeat-0.0.1/pyproject.toml

@@ -0,0 +1,56 @@
+[project]
+name = "treepeat"
+version = "0.0.1"
+description = "treepeat: a treesitter-based CLI tool to detect similar code"
+readme = "README.md"
+requires-python = ">=3.11"
+authors = [{ name = "Dane Summers", email = "dsummersl@gmail.com" }]
+license = "Apache-2.0"
+classifiers = [
+"Programming Language :: Python :: 3",
+"Typing :: Typed",
+]
+dependencies = [
+    "click>=8.3.0",
+    "datasketch>=1.6.5",
+    "pydantic>=2.12.3",
+    "pydantic-settings>=2.0.0",
+    "python-magic>=0.4.27",
+    "rich>=14.2.0",
+    "sarif-pydantic>=0.6.2",
+    "tdqm>=0.0.1",
+    "tree-sitter>=0.25.2",
+    "tree-sitter-language-pack>=0.10.0",
+]
+
+[project.scripts]
+treepeat = "treepeat.cli:main"
+
+[dependency-groups]
+dev = [
+    "pytest>=8",
+    "pytest-cov>=6",
+    "ruff>=0.5",
+    "mypy>=1.10",
+    "pre-commit>=3.7",
+    "radon>=6.0.1",
+]
+
+[tool.uv]
+package = true
+
+[tool.ruff]
+line-length = 100
+exclude = [".venv", ".git", "build", "dist", "tests/fixtures", "benchmark-tests/codebases"]
+
+[tool.pyright]
+exclude = ["benchmark-tests"]
+
+[tool.mypy]
+python_version = "3.11"
+strict = true
+packages = ["treepeat"]
+
+[tool.pytest.ini_options]
+addopts = "--cov=treepeat  --cov-report=term-missing"
+norecursedirs = "benchmark-tests"

treepeat-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

treepeat-0.0.1/treepeat/cli/__init__.py

@@ -0,0 +1,5 @@
+"""CLI module."""
+
+from treepeat.cli.cli import main
+
+__all__ = ["main"]

treepeat-0.0.1/treepeat/cli/cli.py

@@ -0,0 +1,60 @@
+"""CLI interface for treepeat."""
+
+import logging
+
+import click
+from rich.console import Console
+from rich.logging import RichHandler
+
+from treepeat.cli.commands import detect, list_ruleset, treesitter
+
+console = Console()
+
+
+def setup_logging(log_level: str) -> None:
+    """Configure logging with rich handler."""
+    logging.basicConfig(
+        level=log_level,
+        format="%(message)s",
+        handlers=[RichHandler(console=console, rich_tracebacks=True)],
+    )
+
+
+@click.group()
+@click.pass_context
+@click.option(
+    "--log-level",
+    "-l",
+    type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], case_sensitive=False),
+    default="WARNING",
+    help="Set the logging level",
+)
+@click.option(
+    "--ruleset",
+    "-r",
+    type=click.Choice(["none", "default", "loose"], case_sensitive=False),
+    default="default",
+    help="Built-in ruleset profile to use (default: default)",
+)
+def main(
+    ctx: click.Context,
+    log_level: str,
+    ruleset: str,
+) -> None:
+    """Tree-sitter based similarity detector."""
+    setup_logging(log_level.upper())
+
+    # Store common options in context for subcommands
+    ctx.ensure_object(dict)
+    ctx.obj["log_level"] = log_level
+    ctx.obj["ruleset"] = ruleset
+
+
+# Register subcommands
+main.add_command(detect)
+main.add_command(treesitter)
+main.add_command(list_ruleset)
+
+
+if __name__ == "__main__":
+    main()

treepeat-0.0.1/treepeat/cli/commands/__init__.py

@@ -0,0 +1,7 @@
+"""CLI subcommands."""
+
+from .detect import detect
+from .list_ruleset import list_ruleset
+from .treesitter import treesitter
+
+__all__ = ["detect", "list_ruleset", "treesitter"]

treepeat-0.0.1/treepeat/cli/commands/detect.py

@@ -0,0 +1,390 @@
+"""Detect command - find similar code regions."""
+
+import sys
+from pathlib import Path
+
+import click
+from rich.console import Console
+from rich.table import Table
+
+from treepeat.config import LSHSettings, MinHashSettings, PipelineSettings, RulesSettings, ShingleSettings, set_settings
+from treepeat.formatters import format_as_sarif
+from treepeat.models.similarity import Region, RegionSignature, SimilarRegionGroup, SimilarityResult
+from treepeat.pipeline.pipeline import run_pipeline
+
+console = Console()
+
+
+def _parse_patterns(pattern_string: str) -> list[str]:
+    """Parse comma-separated pattern string into list."""
+    return [p.strip() for p in pattern_string.split(",") if p.strip()]
+
+
+def _create_rules_settings(ruleset: str) -> RulesSettings:
+    """Create RulesSettings."""
+    return RulesSettings(ruleset=ruleset)
+
+
+def _configure_settings(
+    ruleset: str,
+    similarity_percent: float,
+    min_lines: int,
+    ignore: str,
+    ignore_files: str,
+    ignore_node_types: str,
+) -> None:
+    """Configure pipeline settings."""
+    lsh_settings = LSHSettings(
+        similarity_percent=similarity_percent / 100.0,
+        min_lines=min_lines,
+        ignore_node_types=_parse_patterns(ignore_node_types),
+    )
+
+    settings = PipelineSettings(
+        rules=_create_rules_settings(ruleset),
+        shingle=ShingleSettings(),  # Uses default k=3
+        minhash=MinHashSettings(),  # Uses default num_perm=128
+        lsh=lsh_settings,
+        ignore_patterns=_parse_patterns(ignore),
+        ignore_file_patterns=_parse_patterns(ignore_files),
+    )
+    set_settings(settings)
+
+
+def _write_output(text: str, output_path: Path | None) -> None:
+    """Write output text to file or stdout."""
+    if output_path:
+        output_path.write_text(text)
+    else:
+        print(text)
+
+
+def _run_pipeline_with_ui(path: Path, output_format: str) -> SimilarityResult:
+    """Run the pipeline with appropriate UI feedback based on output format."""
+    if output_format.lower() == "console":
+        from treepeat.config import get_settings
+        settings = get_settings()
+        console.print(f"\nRuleset: [cyan]{settings.rules.ruleset}[/cyan]")
+        console.print(f"Analyzing: [cyan]{path}[/cyan]\n")
+        with console.status("[bold green]Running pipeline..."):
+            return run_pipeline(path)
+    else:
+        return run_pipeline(path)
+
+
+def _group_signatures_by_file(
+    signatures: list[RegionSignature],
+) -> dict[Path, list[RegionSignature]]:
+    """Group region signatures by file path."""
+    regions_by_file: dict[Path, list[RegionSignature]] = {}
+    for sig in signatures:
+        path = sig.region.path
+        if path not in regions_by_file:
+            regions_by_file[path] = []
+        regions_by_file[path].append(sig)
+    return regions_by_file
+
+
+def _get_group_sort_key(group: SimilarRegionGroup) -> tuple[float, float]:
+    """Get sort key for a similarity group by similarity and average line count."""
+    avg_lines = sum(r.end_line - r.start_line + 1 for r in group.regions) / len(group.regions)
+    return (group.similarity, avg_lines)
+
+
+def _format_region_name(region: Region) -> str:
+    """Format region name with type if not lines."""
+    if region.region_type == "lines":
+        return region.region_name
+    return f"{region.region_name}({region.region_type})"
+
+
+def _display_group(group: SimilarRegionGroup, show_diff: bool = False) -> None:
+    """Display a single similarity group with optional diff."""
+    from treepeat.diff import display_diff
+
+    # Display similarity group header
+    console.print(f"Similar group found ([bold]{group.similarity:.1%}[/bold] similar, {group.size} regions):")
+
+    # Display all regions in the group
+    for i, region in enumerate(group.regions):
+        lines = region.end_line - region.start_line + 1
+        prefix = "  - " if i == 0 else "    "
+        region_display = _format_region_name(region)
+        console.print(
+            f"{prefix}{region.path} [{region.start_line}:{region.end_line}] "
+            f"({lines} lines) {region_display}"
+        )
+
+    # Show diff if requested and we have at least 2 regions
+    if show_diff and len(group.regions) >= 2:
+        console.print()
+        display_diff(group.regions[0], group.regions[1])
+    else:
+        console.print()  # Blank line between groups
+
+
+def display_similar_groups(result: SimilarityResult, show_diff: bool = False) -> None:
+    """Display similar region groups with optional diff."""
+    if not result.similar_groups:
+        console.print("\n[yellow]No similar regions found above threshold.[/yellow]")
+        return
+
+    console.print("\n[bold cyan]Similar Regions:[/bold cyan]")
+    sorted_groups = sorted(result.similar_groups, key=_get_group_sort_key)
+
+    for group in sorted_groups:
+        _display_group(group, show_diff=show_diff)
+
+
+def display_failed_files(result: SimilarityResult, show_details: bool) -> None:
+    """Display failed files with optional error details."""
+    if not result.failed_files:
+        return
+
+    console.print("\n[bold red]Failed Files:[/bold red]")
+    for file_path, error in result.failed_files.items():
+        console.print(f"  [red]✗[/red] {file_path}")
+        if show_details:
+            console.print(f"    [dim]{error}[/dim]")
+
+
+def _init_language_stats(
+    stats_by_format: dict[str, dict[str, int | set[Path]]], language: str
+) -> None:
+    """Initialize stats entry for a language if not present."""
+    if language not in stats_by_format:
+        stats_by_format[language] = {"files": set(), "groups": 0, "lines": 0}
+
+
+def _collect_files_from_signatures(
+    signatures: list[RegionSignature],
+) -> dict[str, dict[str, int | set[Path]]]:
+    """Collect all processed files from signatures."""
+    stats_by_format: dict[str, dict[str, int | set[Path]]] = {}
+
+    for signature in signatures:
+        region = signature.region
+        language = region.language
+        _init_language_stats(stats_by_format, language)
+        stats = stats_by_format[language]
+        stats["files"].add(region.path)  # type: ignore[union-attr]
+
+    return stats_by_format
+
+
+def _add_duplicate_stats(
+    stats_by_format: dict[str, dict[str, int | set[Path]]],
+    similar_groups: list[SimilarRegionGroup],
+) -> None:
+    """Add group counts and duplicate lines from similar groups."""
+    for group in similar_groups:
+        for region in group.regions:
+            language = region.language
+            _init_language_stats(stats_by_format, language)
+            stats = stats_by_format[language]
+            stats["lines"] += region.end_line - region.start_line + 1  # type: ignore[operator]
+
+        # Count group once per language (use first region's language)
+        if group.regions:
+            first_language = group.regions[0].language
+            stats_by_format[first_language]["groups"] += 1  # type: ignore[operator]
+
+
+def _collect_format_statistics(result: SimilarityResult) -> dict[str, dict[str, int | set[Path]]]:
+    """Collect statistics by language/format from all processed files."""
+    stats_by_format = _collect_files_from_signatures(result.signatures)
+    _add_duplicate_stats(stats_by_format, result.similar_groups)
+    return stats_by_format
+
+
+def _populate_summary_table(
+    table: Table,
+    stats_by_format: dict[str, dict[str, int | set[Path]]],
+) -> tuple[set[Path], int, int]:
+    """Populate summary table with format statistics and return totals."""
+    total_files: set[Path] = set()
+    total_groups = 0
+    total_lines = 0
+
+    for language in sorted(stats_by_format.keys()):
+        stats = stats_by_format[language]
+        files = stats["files"]
+        groups = stats["groups"]
+        lines = stats["lines"]
+
+        # Type narrowing assertions
+        assert isinstance(files, set)
+        assert isinstance(groups, int)
+        assert isinstance(lines, int)
+
+        table.add_row(
+            language,
+            str(len(files)),
+            str(groups),
+            str(lines),
+        )
+
+        # Accumulate totals
+        total_files.update(files)
+        total_groups += groups
+        total_lines += lines
+
+    return total_files, total_groups, total_lines
+
+
+def display_summary_table(result: SimilarityResult) -> None:
+    """Display summary table with statistics by format."""
+    # Show stats even if no similar groups found (to show all processed files)
+    if not result.signatures:
+        return
+
+    stats_by_format = _collect_format_statistics(result)
+
+    # Create summary table
+    table = Table(show_header=True, header_style="bold")
+    table.add_column("Format", style="cyan")
+    table.add_column("# Files", justify="right")
+    table.add_column("Groups Found", justify="right")
+    table.add_column("Lines", justify="right")
+
+    # Populate table and calculate totals
+    total_files, total_groups, total_lines = _populate_summary_table(table, stats_by_format)
+
+    # Add totals row
+    table.add_row(
+        "[bold]Totals[/bold]",
+        f"[bold]{len(total_files)}[/bold]",
+        f"[bold]{total_groups}[/bold]",
+        f"[bold]{total_lines}[/bold]",
+        end_section=True,
+    )
+
+    console.print("\n")
+    console.print(table)
+
+
+def _handle_output(
+    result: SimilarityResult,
+    output_format: str,
+    output_path: Path | None,
+    log_level: str,
+    show_diff: bool = False,
+) -> None:
+    """Handle formatting and outputting results."""
+    if output_format.lower() == "sarif":
+        output_text = format_as_sarif(result, pretty=True)
+        _write_output(output_text, output_path)
+    else:  # console
+        display_similar_groups(result, show_diff=show_diff)
+        display_failed_files(result, show_details=(log_level.upper() == "DEBUG"))
+        display_summary_table(result)
+        console.print()
+
+
+def _check_result_errors(result: SimilarityResult, output_format: str) -> None:
+    """Check for errors in the result and exit if necessary."""
+    if result.success_count == 0 and result.failure_count > 0:
+        if output_format.lower() == "console":
+            console.print("[bold red]Error:[/bold red] Failed to parse any files")
+            display_failed_files(result, show_details=True)
+        sys.exit(1)
+
+
+@click.command()
+@click.argument("path", type=click.Path(exists=True, path_type=Path))
+@click.pass_context
+@click.option(
+    "--similarity",
+    "-s",
+    type=click.IntRange(5, 100),
+    default=100,
+    help="Percent similarity threshold (default: 100)",
+)
+@click.option(
+    "--min-lines",
+    "-ml",
+    type=click.IntRange(1),
+    default=5,
+    help="Minimum number of lines to be considered similar (default: 5)",
+)
+@click.option(
+    "--format",
+    "-f",
+    "output_format",
+    type=click.Choice(["console", "sarif"], case_sensitive=False),
+    default="console",
+    help="Output format (default: console)",
+)
+@click.option(
+    "--output",
+    "-o",
+    type=click.Path(path_type=Path),
+    default=None,
+    help="Output file path (default: stdout)",
+)
+@click.option(
+    "--ignore",
+    "-i",
+    type=str,
+    default="",
+    help="Comma-separated list of glob patterns to ignore files (e.g., '*.test.py,**/node_modules/**')",
+)
+@click.option(
+    "--ignore-files",
+    "-if",
+    type=str,
+    default="**/.*ignore",
+    help="Comma-separated list of glob patterns to find ignore files (default: '**/.*ignore')",
+)
+@click.option(
+    "--diff",
+    "-d",
+    is_flag=True,
+    default=False,
+    help="Show side-by-side diff between the first two files in each similar group (console format only)",
+)
+@click.option(
+    "--fail",
+    is_flag=True,
+    default=False,
+    help="Exit with error code 1 if any similar blocks are detected",
+)
+@click.option(
+    "--ignore-node-types",
+    "-int",
+    type=str,
+    default="",
+    help="Comma-separated list of AST node types to ignore during region extraction (e.g., 'parameters,argument_list')",
+)
+def detect(
+    ctx: click.Context,
+    path: Path,
+    similarity: float,
+    min_lines: int,
+    output_format: str,
+    output: Path | None,
+    ignore: str,
+    ignore_files: str,
+    diff: bool,
+    fail: bool,
+    ignore_node_types: str,
+) -> None:
+    """Detect similar code regions of files in a path."""
+    log_level = ctx.obj["log_level"]
+    ruleset = ctx.obj["ruleset"]
+
+    _configure_settings(
+        ruleset,
+        similarity,
+        min_lines,
+        ignore,
+        ignore_files,
+        ignore_node_types,
+    )
+    result = _run_pipeline_with_ui(path, output_format)
+    _check_result_errors(result, output_format)
+    _handle_output(result, output_format, output, log_level, diff)
+
+    # Exit with error code 1 in strict mode if any similar blocks are detected
+    if fail and result.similar_groups:
+        sys.exit(1)