code-to-txt 0.1.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

code_to_txt/code_to_txt.py

@@ -1,180 +1,329 @@
 import os
+from fnmatch import fnmatch
 from pathlib import Path
-from typing import List, Optional, Set
+from typing import Any
+
 import pathspec
+from pathspec import PathSpec
 
+from .utils import load_patterns_from_file
 
-class CodeToText:
-    DEFAULT_IGNORE = {
-        "__pycache__",
-        "*.pyc",
-        "*.pyo",
-        "*.pyd"
-    }
 
-    DEFAULT_EXTENSIONS = {
-        ".py", ".js", ".ts", ".jsx", ".tsx", ".java", ".c", ".cpp", ".h"
-    }
+class CodeToText:
+    """Convert code files to a single text file for LLM consumption."""
 
     def __init__(
             self,
             root_path: str,
-            output_file: str = "output.txt",
-            include_extensions: Optional[Set[str]] = None,
-            exclude_patterns: Optional[List[str]] = None,
+            output_file: str | None = "output.txt",
+            include_extensions: set[str] | None = None,
+            exclude_patterns: list[str] | None = None,
+            glob_patterns: list[str] | None = None,
             gitignore: bool = True,
+            max_file_size_kb: int | None = None,
     ):
         """
-        Initialize the instance of CodeToText.
+        Initialize CodeToText instance.
 
         Args:
             root_path: Root directory to scan
-            output_file: Output file path
+            output_file: Output file path (None for clipboard-only mode)
             include_extensions: Set of file extensions to include (with dots)
             exclude_patterns: List of patterns to exclude (gitignore style)
+            glob_patterns: List of glob patterns to include (e.g., '*.py', 'src/**/*.js')
             gitignore: Whether to respect .gitignore files
+            max_file_size_kb: Skip files larger than this size in KB
         """
         self.root_path = Path(root_path).resolve()
         self.output_file = output_file
-        self.include_extensions = include_extensions or self.DEFAULT_EXTENSIONS
-        self.exclude_patterns = exclude_patterns or []
+        self.glob_patterns = glob_patterns or []
         self.gitignore = gitignore
-        self.spec = None
+        self.max_file_size_kb = max_file_size_kb
+        self.spec: PathSpec | None = None
+        self.file_count = 0
+        self.skipped_files: list[tuple[Path, str]] = []
+
+        config_dir = Path(__file__).parent
+        default_extensions = load_patterns_from_file(config_dir / ".extensions")
+        default_ignore = load_patterns_from_file(config_dir / ".ignore")
+
+        self.include_extensions = include_extensions or default_extensions
+        self.exclude_patterns = exclude_patterns or []
+        self.default_ignore = default_ignore
 
         if self.gitignore:
-            self._load_gitignore()
+            self._init_pathspec()
 
-    def _load_gitignore(self):
-        """Load .gitignore patterns if present."""
-        gitignore_path = self.root_path / ".gitignore"
-        patterns = list(self.DEFAULT_IGNORE)
+    def _init_pathspec(self) -> None:
+        """Initialize pathspec from .gitignore files and default patterns."""
+        patterns = list(self.default_ignore)
+        current_path = self.root_path
 
-        if gitignore_path.exists():
-            with open(gitignore_path, "r") as f:
-                for line in f:
-                    line = line.strip()
-                    if line and not line.startswith("#"):
-                        patterns.append(line)
+        for _ in range(5):
+            gitignore_path = current_path / ".gitignore"
+            if gitignore_path.exists():
+                try:
+                    with open(gitignore_path, encoding="utf-8") as f:
+                        for line in f:
+                            clean_line = line.strip()
+                            if clean_line and not clean_line.startswith("#"):
+                                patterns.append(clean_line)
+                except Exception:
+                    pass
+
+            parent = current_path.parent
+            if parent == current_path:
+                break
+            current_path = parent
 
         patterns.extend(self.exclude_patterns)
         self.spec = pathspec.PathSpec.from_lines("gitignore", patterns)
 
-    def _should_include_file(self, file_path: Path) -> bool:
-        """Check if a file should be included."""
-        # Check extension
-        if file_path.suffix not in self.include_extensions:
+    def _check_glob_match(self, file_path: Path) -> bool:
+        """Check if file matches any glob pattern."""
+        if not self.glob_patterns:
             return False
 
+        relative_path = file_path.relative_to(self.root_path)
+        path_str = str(relative_path)
+
+        for pattern in self.glob_patterns:
+            if fnmatch(path_str, pattern):
+                return True
+            if fnmatch(file_path.name, pattern):
+                return True
+            if fnmatch(path_str.replace(os.sep, "/"), pattern):
+                return True
+
+        return False
+
+    def _check_file_inclusion(self, file_path: Path) -> bool:
+        """Determine if a file should be included in the output."""
+        if self.max_file_size_kb is not None:
+            try:
+                file_size_kb = file_path.stat().st_size / 1024
+                if file_size_kb > self.max_file_size_kb:
+                    self.skipped_files.append(
+                        (file_path, f"exceeds size limit ({file_size_kb:.1f}KB)")
+                    )
+                    return False
+            except Exception:
+                pass
+
+        if self.glob_patterns:
+            if not self._check_glob_match(file_path):
+                return False
+
         if self.spec:
-            relative_path = file_path.relative_to(self.root_path)
-            if self.spec.match_file(str(relative_path)):
+            try:
+                relative_path = file_path.relative_to(self.root_path)
+                relative_str = str(relative_path).replace(os.sep, "/")
+
+                if self.spec.match_file(relative_str):
+                    self.skipped_files.append((file_path, "matches ignore pattern"))
+                    return False
+            except ValueError:
                 return False
 
+        if file_path.suffix not in self.include_extensions:
+            return False
+
         return True
 
-    def _get_files(self) -> List[Path]:
-        """Get all files to process."""
+    def _collect_files(self) -> list[Path]:
+        """Collect all files to process based on filters."""
         files = []
+        self.skipped_files = []
+
         for root, dirs, filenames in os.walk(self.root_path):
             root_path = Path(root)
 
             if self.spec:
-                relative_root = root_path.relative_to(self.root_path)
-                dirs[:] = [
-                    d for d in dirs
-                    if not self.spec.match_file(str(relative_root / d))
-                ]
+                try:
+                    relative_root = root_path.relative_to(self.root_path)
+                    root_str = str(relative_root).replace(os.sep, "/") if str(relative_root) != "." else ""
+
+                    filtered_dirs = []
+                    for d in dirs:
+                        dir_path = f"{root_str}/{d}" if root_str else d
+
+                        if not self.spec.match_file(dir_path) and not self.spec.match_file(f"{dir_path}/"):
+                            filtered_dirs.append(d)
+
+                    dirs[:] = filtered_dirs
+                except ValueError:
+                    pass
 
             for filename in filenames:
                 file_path = root_path / filename
-                if self._should_include_file(file_path):
+                if self._check_file_inclusion(file_path):
                     files.append(file_path)
 
         return sorted(files)
 
-    def convert(self, add_tree: bool = True, separator: str = "=" * 80) -> int:
+    def calculate_statistics(self) -> dict[str, Any]:
         """
-        Convert files to single text file.
+        Calculate statistics about the codebase.
+
+        Returns:
+            Dictionary containing total files, size, lines, breakdown by extension, etc.
+        """
+        files = self._collect_files()
+
+        stats: dict = {
+            "total_files": len(files),
+            "total_size_bytes": 0,
+            "total_lines": 0,
+            "by_extension": {},
+            "skipped_files": len(self.skipped_files),
+            "largest_files": [],
+        }
+
+        file_sizes = []
+
+        for file_path in files:
+            try:
+                size = file_path.stat().st_size
+                stats["total_size_bytes"] += size
+                file_sizes.append((file_path, size))
+
+                try:
+                    with open(file_path, encoding="utf-8") as f:
+                        lines = sum(1 for _ in f)
+                        stats["total_lines"] += lines
+                except Exception:
+                    pass
+
+                ext = file_path.suffix or "(no extension)"
+                if ext not in stats["by_extension"]:
+                    stats["by_extension"][ext] = {"count": 0, "size": 0}
+                stats["by_extension"][ext]["count"] += 1
+                stats["by_extension"][ext]["size"] += size
+
+            except Exception:
+                pass
+
+        file_sizes.sort(key=lambda x: x[1], reverse=True)
+        stats["largest_files"] = [
+            {"path": str(f.relative_to(self.root_path)), "size_kb": s / 1024}
+            for f, s in file_sizes[:10]
+        ]
+
+        return stats
+
+    def generate_content(self, add_tree: bool = True, separator: str = "=" * 80) -> str:
+        """
+        Generate content as string without writing to file.
 
         Args:
             add_tree: Whether to add directory tree at the beginning
             separator: Separator between files
 
         Returns:
-            Number of files processed
+            Generated content as string
         """
-        files = self._get_files()
+        files = self._collect_files()
+        self.file_count = len(files)
+
+        output_lines = []
+        output_lines.append(f"Code Export from: {self.root_path}")
+        output_lines.append(f"Total files: {len(files)}")
+        output_lines.append(separator)
+        output_lines.append("")
+
+        if add_tree:
+            output_lines.append("DIRECTORY TREE:")
+            output_lines.append(separator)
+            output_lines.append(self._build_tree_structure())
+            output_lines.append("")
+            output_lines.append(separator)
+            output_lines.append("")
+
+        for idx, file_path in enumerate(files, 1):
+            relative_path = file_path.relative_to(self.root_path)
 
-        with open(self.output_file, "w", encoding="utf-8") as out:
-            out.write(f"Code Export from: {self.root_path}\n")
-            out.write(f"Total files: {len(files)}\n")
-            out.write(f"{separator}\n\n")
+            output_lines.append(f"FILE {idx}/{len(files)}: {relative_path}")
+            output_lines.append(separator)
 
-            if add_tree:
-                out.write("DIRECTORY TREE:\n")
-                out.write(separator + "\n")
-                out.write(self._generate_tree())
-                out.write(f"\n{separator}\n\n")
+            try:
+                with open(file_path, encoding="utf-8") as f:
+                    content = f.read()
+                output_lines.append(content)
+            except UnicodeDecodeError:
+                output_lines.append("[Binary file - skipped]")
+            except Exception as e:
+                output_lines.append(f"[Error reading file: {e}]")
 
-            for i, file_path in enumerate(files, 1):
-                relative_path = file_path.relative_to(self.root_path)
+            output_lines.append("")
+            output_lines.append(separator)
+            output_lines.append("")
 
-                out.write(f"FILE {i}/{len(files)}: {relative_path}\n")
-                out.write(separator + "\n")
+        return "\n".join(output_lines)
 
-                try:
-                    with open(file_path, "r", encoding="utf-8") as f:
-                        content = f.read()
-                    out.write(content)
-                except UnicodeDecodeError:
-                    out.write(f"[Binary file - skipped]\n")
-                except Exception as e:
-                    out.write(f"[Error reading file: {e}]\n")
+    def convert(self, add_tree: bool = True, separator: str = "=" * 80) -> int:
+        """
+        Convert files to single text file.
+
+        Args:
+            add_tree: Whether to add directory tree at the beginning
+            separator: Separator between files
+
+        Returns:
+            Number of files processed
+        """
+        if not self.output_file:
+            raise ValueError("output_file must be specified for convert()")
+
+        content = self.generate_content(add_tree=add_tree, separator=separator)
+
+        with open(self.output_file, "w", encoding="utf-8") as out:
+            out.write(content)
 
-                out.write(f"\n{separator}\n\n")
+        return self.file_count
 
-        return len(files)
+    def _build_tree_structure(self) -> str:
+        """Build a directory tree representation of included files."""
+        tree_output = []
+        files = self._collect_files()
 
-    def _generate_tree(self) -> str:
-        """Generate a directory tree representation."""
-        tree_lines = []
-        files = self._get_files()
+        if not files:
+            return "(no files to display)"
 
-        dir_structure = {}
+        structure: dict[str, Any] = {}
         for file_path in files:
             relative_path = file_path.relative_to(self.root_path)
             parts = relative_path.parts
 
-            current = dir_structure
+            current_level = structure
             for part in parts[:-1]:
-                if part not in current:
-                    current[part] = {}
-                current = current[part]
+                if part not in current_level:
+                    current_level[part] = {}
+                current_level = current_level[part]
 
-            if "__files__" not in current:
-                current["__files__"] = []
-            current["__files__"].append(parts[-1])
+            if "__files__" not in current_level:
+                current_level["__files__"] = []
+            current_level["__files__"].append(parts[-1])
 
-        def print_tree(structure, prefix="", is_last=True):
-            items = []
-            for key in sorted(structure.keys()):
+        def render_tree(node: dict[str, Any], prefix: str = "", is_final: bool = True) -> None:
+            entries = []
+            for key in sorted(node.keys()):
                 if key != "__files__":
-                    items.append((key, True))  # directory
+                    entries.append((key, True))
 
-            if "__files__" in structure:
-                for file in sorted(structure["__files__"]):
-                    items.append((file, False))  # file
+            if "__files__" in node:
+                for file in sorted(node["__files__"]):
+                    entries.append((file, False))
 
-            for i, (name, is_dir) in enumerate(items):
-                is_last_item = i == len(items) - 1
-                connector = "└── " if is_last_item else "├── "
-                tree_lines.append(f"{prefix}{connector}{name}{'/' if is_dir else ''}")
+            for i, (name, is_directory) in enumerate(entries):
+                is_last_entry = i == len(entries) - 1
+                connector = "└── " if is_last_entry else "├── "
+                tree_output.append(f"{prefix}{connector}{name}{'/' if is_directory else ''}")
 
-                if is_dir:
-                    extension = "    " if is_last_item else "│   "
-                    print_tree(structure[name], prefix + extension, is_last_item)
+                if is_directory:
+                    extension = "    " if is_last_entry else "│   "
+                    render_tree(node[name], prefix + extension, is_last_entry)
 
-        tree_lines.append(f"{self.root_path.name}/")
-        print_tree(dir_structure)
+        tree_output.append(f"{self.root_path.name}/")
+        render_tree(structure)
 
-        return "\n".join(tree_lines)
+        return "\n".join(tree_output)

code_to_txt/config.py

@@ -0,0 +1,167 @@
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+DEFAULT_CONFIG = {
+    "output": "code-to-txt.txt",
+    "extensions": None,
+    "exclude": [
+        "tests/*",
+        "*.test.js",
+        "*.test.ts",
+        "*.spec.js",
+        "*.spec.ts",
+    ],
+    "glob": [],
+    "no_gitignore": False,
+    "no_tree": False,
+    "separator": "=" * 80,
+    "clipboard": False,
+    "clipboard_only": False,
+    "timestamp": True,
+    "max_file_size": None,
+}
+
+
+def load_config(config_path: str) -> dict[str, Any]:
+    """
+    Load configuration from YAML file.
+
+    Args:
+        config_path: Path to the configuration file
+
+    Returns:
+        Dictionary with validated configuration values
+    """
+    path = Path(config_path)
+
+    if not path.exists():
+        raise FileNotFoundError(f"Config file not found: {config_path}")
+
+    with open(path, encoding="utf-8") as f:
+        config = yaml.safe_load(f)
+
+    if config is None:
+        config = {}
+
+    validated_config: dict[str, Any] = {}
+
+    if "output" in config:
+        validated_config["output"] = str(config["output"])
+
+    if "separator" in config:
+        validated_config["separator"] = str(config["separator"])
+
+    if "extensions" in config:
+        ext = config["extensions"]
+        if isinstance(ext, str):
+            validated_config["extensions"] = ext
+        elif isinstance(ext, list):
+            validated_config["extensions"] = " ".join(str(e) for e in ext)
+        elif ext is not None:
+            validated_config["extensions"] = str(ext)
+
+    for field in ["exclude", "glob"]:
+        if field in config:
+            value = config[field]
+            if isinstance(value, list):
+                validated_config[field] = value
+            elif isinstance(value, str):
+                validated_config[field] = [value]
+            elif value is not None:
+                validated_config[field] = [str(value)]
+
+    for field in ["no_gitignore", "no_tree", "clipboard", "clipboard_only", "timestamp"]:
+        if field in config:
+            validated_config[field] = bool(config[field])
+
+    if "max_file_size" in config and config["max_file_size"] is not None:
+        validated_config["max_file_size"] = int(config["max_file_size"])
+
+    return validated_config
+
+
+def create_default_config(config_path: Path) -> None:
+    """
+    Create a default configuration file.
+
+    Args:
+        config_path: Path where to create the config file
+    """
+    config_content = """# Code-to-Txt Configuration File
+# This file defines default settings for code-to-txt
+# CLI arguments will override these settings
+
+# Output file name (supports strftime formatting)
+# Use timestamp: true to automatically add timestamp
+output: code-to-txt.txt
+
+# File extensions to include
+# Can be a list or space/comma-separated string
+# Leave as null to use default extensions
+# extensions: [.py, .js, .ts]
+# extensions: ".py .js .ts"
+extensions: null
+
+# Patterns to exclude (gitignore-style)
+# These are in addition to .gitignore patterns
+exclude:
+  - "tests/*"
+  - "*.test.js"
+  - "*.test.ts"
+  - "*.spec.js"
+  - "*.spec.ts"
+  - "node_modules/*"
+  - "__pycache__/*"
+  - "*.pyc"
+
+# Glob patterns to include (alternative to extensions)
+# If specified, only files matching these patterns will be included
+# glob:
+#   - "*.py"
+#   - "src/**/*.js"
+#   - "**/*.tsx"
+glob: []
+
+# Ignore .gitignore files
+no_gitignore: false
+
+# Don't include directory tree in output
+no_tree: false
+
+# Separator between files
+separator: "================================================================================"
+
+# Copy output to clipboard
+clipboard: false
+
+# Copy to clipboard only (don't save file)
+clipboard_only: false
+
+# Add timestamp to output filename
+timestamp: true
+
+max_file_size: null
+
+# Example configurations:
+#
+# For Python projects:
+# extensions: [.py]
+# exclude: ["tests/*", "*.pyc", "__pycache__/*", "venv/*"]
+#
+# For JavaScript/TypeScript projects:
+# extensions: [.js, .ts, .jsx, .tsx]
+# exclude: ["node_modules/*", "dist/*", "build/*", "*.test.js"]
+#
+# For C/C++ projects:
+# extensions: [.c, .cpp, .h, .hpp]
+# exclude: ["build/*", "*.o", "*.a"]
+#
+# Using glob patterns:
+# glob: ["src/**/*.py", "lib/**/*.py", "*.md"]
+# extensions: null
+"""
+
+    with open(config_path, "w", encoding="utf-8") as f:
+        f.write(config_content)

code_to_txt/utils.py

@@ -0,0 +1,13 @@
+from pathlib import Path
+
+
+def load_patterns_from_file(file_path: Path) -> set[str]:
+    """Load patterns from a text file, one per line."""
+    patterns = set()
+    if file_path.exists():
+        with open(file_path, encoding="utf-8") as f:
+            for line in f:
+                stripped = line.strip()
+                if stripped and not stripped.startswith("#"):
+                    patterns.add(stripped)
+    return patterns