onepaste 1.3.0__py3-none-any.whl

codecollector/collector.py

@@ -0,0 +1,235 @@
+"""File collector module."""
+
+import fnmatch
+import os
+import re
+from pathlib import Path
+from typing import Iterable, List, Optional, Set, Tuple
+
+from codecollector.config import CollectorConfig
+from codecollector.gitignore import GitignoreMatcher, list_git_visible_files
+
+# Own outputs should never be re-collected (prevents runaway growth).
+_OUTPUT_ARTIFACT_RE = re.compile(
+    r"^code_collection(?:_\d+)?(?:\.part\d+)?\.(md|txt)$"
+    r"|^code_collection(?:_\d+)?\.manifest\.json$"
+)
+
+
+class FileCollector:
+    """Collects code files from directories."""
+
+    def __init__(self, config: CollectorConfig):
+        self.config = config
+        self.collected_files: List[Path] = []
+        self.skipped_files: List[Tuple[Path, str]] = []
+        self._gitignore: Optional[GitignoreMatcher] = None
+        self._git_files: Optional[Set[Path]] = None
+
+        if config.respect_gitignore:
+            self._git_files = list_git_visible_files(config.root_path)
+            if self._git_files is None:
+                # Fallback when git is unavailable
+                self._gitignore = GitignoreMatcher(config.root_path)
+                self._gitignore.load_all()
+
+    def _is_output_artifact(self, file_path: Path) -> bool:
+        name = file_path.name
+        if _OUTPUT_ARTIFACT_RE.match(name):
+            return True
+
+        output_name = Path(self.config.output_file).name
+        if name == output_name:
+            return True
+
+        stem = Path(self.config.output_file).stem
+        suffix = Path(self.config.output_file).suffix
+        # matches output_1.md / output.part2.md / output_1.part2.md
+        custom_re = re.compile(
+            rf"^{re.escape(stem)}(?:_\d+)?(?:\.part\d+)?{re.escape(suffix)}$"
+            rf"|^{re.escape(stem)}(?:_\d+)?\.manifest\.json$"
+        )
+        return bool(custom_re.match(name))
+
+    def should_collect_file(self, file_path: Path, *, check_git: bool = True) -> bool:
+        """Determine if a file should be collected."""
+        if not file_path.is_file():
+            return False
+
+        if file_path.is_symlink():
+            self.skipped_files.append((file_path, "symlink"))
+            return False
+
+        if self._is_output_artifact(file_path):
+            self.skipped_files.append((file_path, "collector output"))
+            return False
+
+        if check_git:
+            if self._git_files is not None:
+                if file_path.resolve() not in self._git_files:
+                    self.skipped_files.append((file_path, "gitignored"))
+                    return False
+            elif self._gitignore and self._gitignore.is_ignored(file_path):
+                self.skipped_files.append((file_path, "gitignored"))
+                return False
+
+        try:
+            size_mb = file_path.stat().st_size / (1024 * 1024)
+            if size_mb > self.config.max_file_size_mb:
+                self.skipped_files.append(
+                    (file_path, f"too large ({size_mb:.1f}MB > {self.config.max_file_size_mb}MB)")
+                )
+                return False
+        except OSError as e:
+            self.skipped_files.append((file_path, f"access error ({e})"))
+            return False
+
+        file_name = file_path.name
+        rel_posix = self._relative_posix(file_path)
+
+        if self.config.exclude_patterns and self._matches_any(
+            rel_posix, self.config.exclude_patterns
+        ):
+            self.skipped_files.append((file_path, "excluded by pattern"))
+            return False
+
+        if self.config.include_patterns:
+            # --include overrides the extension whitelist (repomix-style).
+            if self._matches_any(rel_posix, self.config.include_patterns):
+                return self._is_text_file(file_path)
+            return False
+
+        if file_name in self.config.special_files:
+            return self._is_text_file(file_path)
+
+        suffix = file_path.suffix.lower()
+        if suffix in self.config.include_extensions:
+            return self._is_text_file(file_path)
+
+        if file_name.startswith("."):
+            for ext in self.config.include_extensions:
+                if ext.startswith(".") and file_name.endswith(ext.lstrip(".")):
+                    return self._is_text_file(file_path)
+
+        return False
+
+    def _relative_posix(self, file_path: Path) -> str:
+        try:
+            return file_path.relative_to(self.config.root_path).as_posix()
+        except ValueError:
+            return file_path.name
+
+    @staticmethod
+    def _matches_any(rel_posix: str, patterns: Iterable[str]) -> bool:
+        name = Path(rel_posix).name
+        for pattern in patterns:
+            if pattern.endswith("/"):
+                base = pattern.rstrip("/")
+                if rel_posix.startswith(base + "/") or f"/{base}/" in f"/{rel_posix}":
+                    return True
+                continue
+            if fnmatch.fnmatch(rel_posix, pattern) or fnmatch.fnmatch(name, pattern):
+                return True
+        return False
+
+    def _is_text_file(self, file_path: Path) -> bool:
+        """Check if a file is a readable text file."""
+        try:
+            with open(file_path, encoding="utf-8") as f:
+                chunk = f.read(1024)
+                if "\0" in chunk:
+                    self.skipped_files.append((file_path, "binary file"))
+                    return False
+            return True
+        except UnicodeDecodeError:
+            self.skipped_files.append((file_path, "encoding error"))
+            return False
+        except OSError as e:
+            self.skipped_files.append((file_path, f"read error ({e})"))
+            return False
+
+    def _path_in_excluded_dir(self, file_path: Path) -> bool:
+        try:
+            parts = file_path.resolve().relative_to(self.config.root_path.resolve()).parts
+        except ValueError:
+            parts = file_path.parts
+        return any(part in self.config.all_exclude_dirs for part in parts[:-1])
+
+    def _should_exclude_dir(self, dir_path: Path) -> bool:
+        """Check if a directory should be excluded."""
+        for part in dir_path.parts:
+            if part in self.config.all_exclude_dirs:
+                return True
+
+        if self._git_files is None and self._gitignore:
+            if self._gitignore.is_ignored(dir_path, is_dir=True):
+                return True
+
+        dir_name = dir_path.name
+        if dir_name.startswith(".") and dir_name != ".git":
+            if dir_name not in {".github", ".vscode", ".idea"}:
+                return True
+
+        return False
+
+    def collect_files(self) -> List[Path]:
+        """Collect all matching code files."""
+        self.collected_files.clear()
+        self.skipped_files.clear()
+
+        root = self.config.root_path
+
+        if not root.exists():
+            raise FileNotFoundError(f"Directory not found: {root}")
+
+        if not root.is_dir():
+            raise NotADirectoryError(f"Not a directory: {root}")
+
+        # Filter mode with git: start from git-visible files (stricter, correct).
+        if self.config.respect_gitignore and self._git_files is not None:
+            self._collect_from_git_files()
+        elif self.config.recursive:
+            self._collect_by_walk()
+        else:
+            for item in root.iterdir():
+                if item.is_file() and self.should_collect_file(item):
+                    self.collected_files.append(item)
+
+        self.collected_files.sort()
+        return self.collected_files
+
+    def _collect_from_git_files(self) -> None:
+        assert self._git_files is not None
+        root = self.config.root_path.resolve()
+
+        for file_path in sorted(self._git_files):
+            try:
+                file_path.relative_to(root)
+            except ValueError:
+                continue
+
+            if not self.config.recursive:
+                if file_path.parent.resolve() != root:
+                    continue
+
+            if self._path_in_excluded_dir(file_path):
+                self.skipped_files.append((file_path, "excluded directory"))
+                continue
+
+            # Already known to be git-visible; only apply local filters.
+            if self.should_collect_file(file_path, check_git=False):
+                self.collected_files.append(file_path)
+
+    def _collect_by_walk(self) -> None:
+        root = self.config.root_path
+        for dirpath, dirnames, filenames in os.walk(root):
+            current_dir = Path(dirpath)
+            dirnames[:] = [
+                d for d in dirnames
+                if not self._should_exclude_dir(current_dir / d)
+            ]
+
+            for filename in filenames:
+                file_path = current_dir / filename
+                if self.should_collect_file(file_path):
+                    self.collected_files.append(file_path)

codecollector/config.py

@@ -0,0 +1,151 @@
+"""Configuration management for CodeCollector."""
+
+import json
+from dataclasses import dataclass, field, fields
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Set
+
+from codecollector.gitignore import is_inside_git_work_tree
+
+CONFIG_DIR = Path.home() / ".config" / "codecollector"
+CONFIG_FILE = CONFIG_DIR / "config.json"
+
+
+def ensure_config_dir() -> None:
+    """Ensure configuration directory exists."""
+    CONFIG_DIR.mkdir(parents=True, exist_ok=True)
+
+
+_SET_FIELDS = {"exclude_dirs", "include_extensions", "special_files", "extra_exclude_dirs"}
+
+
+@dataclass
+class CollectorConfig:
+    """Configuration for the code collector."""
+
+    root_path: Path
+    recursive: bool = True
+    output_file: str = "code_collection.md"
+    output_dir: Optional[Path] = None
+
+    exclude_dirs: Set[str] = field(default_factory=lambda: {
+        "__pycache__", ".git", ".svn", ".hg",
+        "node_modules", "venv", ".venv", "env", ".env",
+        ".idea", ".vscode", "build", "dist", "target",
+        ".eggs", "*.egg-info", ".tox", ".mypy_cache",
+        ".pytest_cache", "__pypackages__", ".next",
+        ".nuxt", ".output", "coverage", ".coverage",
+        "tmp", "temp", "logs",
+    })
+
+    extra_exclude_dirs: Set[str] = field(default_factory=set)
+
+    include_extensions: Set[str] = field(default_factory=lambda: {
+        ".py", ".js", ".ts", ".jsx", ".tsx", ".vue", ".svelte",
+        ".html", ".css", ".scss", ".less",
+        ".java", ".kt", ".go", ".rs", ".rb", ".php", ".swift",
+        ".c", ".cpp", ".h", ".hpp",
+        ".sh", ".bash", ".zsh",
+        ".yml", ".yaml", ".json", ".xml", ".toml",
+        ".cfg", ".ini", ".conf", ".env",
+        ".md", ".txt", ".rst",
+        ".sql", ".graphql",
+        "Dockerfile", ".dockerignore",
+        "Makefile", ".gitignore",
+    })
+
+    special_files: Set[str] = field(default_factory=lambda: {
+        "Dockerfile", "Makefile", "Vagrantfile",
+        ".gitignore", ".dockerignore", ".env",
+        ".editorconfig", ".prettierrc",
+    })
+
+    max_file_size_mb: float = 5.0
+    max_output_size_mb: float = 2.0
+    respect_gitignore: bool = False
+    auto_increment_output: bool = True
+    write_manifest: bool = True
+    show_progress: bool = True
+    show_skipped: bool = True
+
+    include_patterns: List[str] = field(default_factory=list)
+    exclude_patterns: List[str] = field(default_factory=list)
+
+    @property
+    def all_exclude_dirs(self) -> Set[str]:
+        return self.exclude_dirs | self.extra_exclude_dirs
+
+    @classmethod
+    def _normalize_dict(cls, data: Dict[str, Any]) -> Dict[str, Any]:
+        normalized = {}
+        valid_names = {f.name for f in fields(cls)}
+
+        for key, value in data.items():
+            if key not in valid_names:
+                continue
+            if key in _SET_FIELDS and isinstance(value, list):
+                normalized[key] = set(value)
+            elif key in ("root_path", "output_dir") and value is not None:
+                normalized[key] = Path(value)
+            else:
+                normalized[key] = value
+
+        return normalized
+
+    @classmethod
+    def load_from_file(cls, filepath: str) -> Dict[str, Any]:
+        """Load configuration dict from a JSON file."""
+        with open(filepath, encoding="utf-8") as f:
+            data = json.load(f)
+        return cls._normalize_dict(data)
+
+    @classmethod
+    def from_sources(
+        cls,
+        root_path: Path,
+        config_file: Optional[str] = None,
+        overrides: Optional[Dict[str, Any]] = None,
+    ) -> "CollectorConfig":
+        """Build config by merging global config, file config, and CLI overrides.
+
+        When no source specifies `respect_gitignore`, it defaults to on inside
+        git work trees and off elsewhere.
+        """
+        merged: Dict[str, Any] = {}
+
+        if CONFIG_FILE.exists():
+            merged.update(cls.load_from_file(str(CONFIG_FILE)))
+
+        if config_file:
+            merged.update(cls.load_from_file(config_file))
+
+        merged["root_path"] = root_path
+
+        if overrides:
+            for key, value in overrides.items():
+                if value is not None:
+                    if key in _SET_FIELDS and isinstance(value, list):
+                        merged[key] = set(value)
+                    else:
+                        merged[key] = value
+
+        if "respect_gitignore" not in merged:
+            merged["respect_gitignore"] = is_inside_git_work_tree(root_path)
+
+        return cls(**cls._normalize_dict(merged))
+
+    def save_to_file(self, filepath: str) -> None:
+        """Save configuration to a JSON file."""
+        config_dict = {
+            "exclude_dirs": sorted(self.exclude_dirs),
+            "extra_exclude_dirs": sorted(self.extra_exclude_dirs),
+            "include_extensions": sorted(self.include_extensions),
+            "max_file_size_mb": self.max_file_size_mb,
+            "max_output_size_mb": self.max_output_size_mb,
+            "auto_increment_output": self.auto_increment_output,
+            "write_manifest": self.write_manifest,
+            "include_patterns": sorted(self.include_patterns),
+            "exclude_patterns": sorted(self.exclude_patterns),
+        }
+        with open(filepath, "w", encoding="utf-8") as f:
+            json.dump(config_dict, f, indent=2)

codecollector/formatter.py

@@ -0,0 +1,197 @@
+"""Output formatter module."""
+
+import re
+from datetime import datetime
+from pathlib import Path
+from typing import List, Tuple
+
+from codecollector.config import CollectorConfig
+from codecollector.tokens import count_tokens, method_label
+
+_FENCE_RUN_RE = re.compile(r"`{3,}")
+_TOP_TOKEN_FILES = 10
+
+
+class OutputFormatter:
+    """Formats collected code files into LLM-ready Markdown."""
+
+    @staticmethod
+    def fence_marker(content: str) -> str:
+        """Return a backtick fence long enough to safely wrap the content."""
+        longest = max(
+            (len(m.group(0)) for m in _FENCE_RUN_RE.finditer(content)),
+            default=0,
+        )
+        return "`" * max(3, longest + 1)
+
+    @staticmethod
+    def format_file_content(file_path: Path, relative_to: Path) -> str:
+        """Format a single file as detailed Markdown."""
+        relative_path = file_path.relative_to(relative_to)
+        lang = file_path.suffix.lstrip(".") or "text"
+
+        try:
+            with open(file_path, encoding="utf-8") as f:
+                content = f.read()
+
+            line_count = content.count("\n") + 1
+            file_size = file_path.stat().st_size
+            tokens = count_tokens(content)
+            fence = OutputFormatter.fence_marker(content)
+
+            return (
+                f"### `{relative_path}`\n\n"
+                f"**Lines:** {line_count} | **Size:** {file_size:,} bytes "
+                f"| **Tokens:** {tokens:,}\n\n"
+                f"{fence}{lang}\n"
+                f"{content}\n"
+                f"{fence}\n\n"
+            )
+        except Exception as e:
+            return f"### `{relative_path}`\n\n**Read Error:** {e}\n\n"
+
+    @staticmethod
+    def format_summary(
+        collected_files: List[Path],
+        skipped_files: List[Tuple[Path, str]],
+        config: CollectorConfig,
+    ) -> str:
+        """Format the collection summary as Markdown."""
+        now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+        total_lines = 0
+        total_size = 0
+        total_tokens = 0
+        file_tokens: List[Tuple[Path, int]] = []
+        for fp in collected_files:
+            try:
+                with open(fp, encoding="utf-8") as f:
+                    content = f.read()
+                total_lines += content.count("\n") + 1
+                total_size += fp.stat().st_size
+                tokens = count_tokens(content)
+                total_tokens += tokens
+                file_tokens.append((fp, tokens))
+            except Exception:
+                file_tokens.append((fp, 0))
+
+        summary = [
+            "# CodeCollector - Collection Summary",
+            "",
+            f"**Generated:** {now}  ",
+            f"**Root:** `{config.root_path.absolute()}`  ",
+            f"**Mode:** {'Recursive' if config.recursive else 'Non-recursive'}  ",
+        ]
+
+        if config.respect_gitignore:
+            summary.append("**Filter:** `.gitignore` enabled  ")
+
+        summary.extend([
+            "",
+            f"**Files collected:** {len(collected_files)}  ",
+            f"**Total lines:** {total_lines:,}  ",
+            f"**Total size:** {total_size / 1024:.1f} KB  ",
+            f"**Total tokens:** {total_tokens:,} *({method_label()})*  ",
+        ])
+
+        if collected_files:
+            summary.extend([
+                "",
+                "## Top Files by Tokens",
+                "",
+                "| File | Tokens |",
+                "| --- | ---: |",
+            ])
+            top_files = sorted(file_tokens, key=lambda x: x[1], reverse=True)
+            for fp, tokens in top_files[:_TOP_TOKEN_FILES]:
+                try:
+                    rel_path = fp.relative_to(config.root_path).as_posix()
+                except ValueError:
+                    rel_path = str(fp)
+                summary.append(f"| `{rel_path}` | {tokens:,} |")
+
+        if skipped_files:
+            summary.append("")
+            summary.append(f"**Files skipped:** {len(skipped_files)}")
+            summary.append("")
+            for fp, reason in skipped_files[:5]:
+                try:
+                    rel_path = fp.relative_to(config.root_path)
+                    summary.append(f"- `{rel_path}`: {reason}")
+                except ValueError:
+                    summary.append(f"- `{fp}`: {reason}")
+            if len(skipped_files) > 5:
+                summary.append(f"- ... and {len(skipped_files) - 5} more")
+
+        summary.extend([
+            "",
+            "## Directory Tree",
+            "",
+            "```",
+            *OutputFormatter._build_directory_tree(collected_files, config.root_path),
+            "```",
+            "",
+            "## File List",
+            "",
+        ])
+
+        files_by_ext: dict = {}
+        for fp in collected_files:
+            ext = fp.suffix or "no_ext"
+            files_by_ext.setdefault(ext, [])
+            try:
+                files_by_ext[ext].append(fp.relative_to(config.root_path))
+            except ValueError:
+                files_by_ext[ext].append(fp)
+
+        for ext, files in sorted(files_by_ext.items()):
+            summary.append(f"**{ext}** ({len(files)} files)")
+            for f in files[:5]:
+                summary.append(f"- `{f}`")
+            if len(files) > 5:
+                summary.append(f"- ... and {len(files) - 5} more")
+            summary.append("")
+
+        summary.extend([
+            "---",
+            "",
+            "# Collected Code Content",
+            "",
+        ])
+
+        return "\n".join(summary)
+
+    @staticmethod
+    def _build_directory_tree(collected_files: List[Path], root: Path) -> List[str]:
+        """Build a simple directory tree from collected files."""
+        tree: dict = {}
+        for fp in collected_files:
+            try:
+                rel = fp.relative_to(root)
+            except ValueError:
+                rel = fp
+            parts = rel.parts
+            node = tree
+            for part in parts[:-1]:
+                node = node.setdefault(part, {})
+            if parts:
+                node.setdefault(parts[-1], None)
+
+        lines: List[str] = []
+
+        def walk(node: dict, prefix: str = "") -> None:
+            items = sorted(node.items(), key=lambda x: (x[1] is None, x[0]))
+            for i, (name, children) in enumerate(items):
+                is_last = i == len(items) - 1
+                connector = "└── " if is_last else "├── "
+                lines.append(f"{prefix}{connector}{name}")
+                if isinstance(children, dict) and children:
+                    extension = "    " if is_last else "│   "
+                    walk(children, prefix + extension)
+
+        if tree:
+            walk(tree)
+        else:
+            lines.append("(empty)")
+
+        return lines