gwc-pybundle 2.1.2__py3-none-any.whl

pybundle/steps/logging_analysis.py

@@ -0,0 +1,233 @@
+"""
+Step: Logging Analysis
+Analyze logging calls and log level distribution.
+"""
+
+import ast
+import re
+from pathlib import Path
+from typing import Dict, List
+
+from .base import Step, StepResult
+
+
+class LoggingAnalysisStep(Step):
+    """Analyze logging calls and log level distribution in Python code."""
+
+    name = "logging analysis"
+
+    def run(self, ctx: "BundleContext") -> StepResult:  # type: ignore[name-defined]
+        """Extract logging calls and analyze log level distribution."""
+        import time
+
+        start = time.time()
+
+        root = ctx.root
+        python_files = sorted(root.rglob("*.py"))
+        if not python_files:
+            return StepResult(self.name, "SKIP", int(time.time() - start), "No Python files found")
+
+        # Track logging patterns
+        log_levels: Dict[str, List[str]] = {
+            "debug": [],
+            "info": [],
+            "warning": [],
+            "error": [],
+            "critical": [],
+            "exception": [],
+        }
+        logging_configs = []
+        logger_names = set()
+        analyzed_files = 0
+
+        # Common logging patterns to detect
+        logging_import_pattern = re.compile(r"import\s+logging")
+        getlogger_pattern = re.compile(r"logging\.getLogger\(['\"]?([^'\")\s]+)?['\"]?\)")
+        basicconfig_pattern = re.compile(r"logging\.basicConfig\(")
+
+        for py_file in python_files:
+            # Skip non-user code
+            if any(
+                part in py_file.parts
+                for part in [
+                    "venv",
+                    ".venv",
+                    "env",
+                    "site-packages",
+                    "__pycache__",
+                    ".git",
+                    "node_modules",
+                ]
+            ):
+                continue
+
+            analyzed_files += 1
+
+            try:
+                source = py_file.read_text(encoding="utf-8", errors="ignore")
+                
+                # Check for logging usage
+                if "logging" not in source and "logger" not in source:
+                    continue
+
+                rel_path = py_file.relative_to(root)
+
+                # Find logging configurations
+                if basicconfig_pattern.search(source):
+                    logging_configs.append(str(rel_path))
+
+                # Find logger names
+                for match in getlogger_pattern.finditer(source):
+                    logger_name = match.group(1) or "__main__"
+                    logger_names.add(logger_name)
+
+                # Parse AST to find logging calls
+                try:
+                    tree = ast.parse(source, str(py_file))
+
+                    for node in ast.walk(tree):
+                        if isinstance(node, ast.Call):
+                            # Check if it's a logging call
+                            level = self._extract_log_level(node)
+                            if level:
+                                location = f"{rel_path}:{node.lineno}"
+                                log_levels[level].append(location)
+
+                except SyntaxError:
+                    continue
+
+            except (UnicodeDecodeError, OSError):
+                continue
+
+        # Calculate statistics
+        total_logs = sum(len(locations) for locations in log_levels.values())
+
+        # Generate report
+        lines = [
+            "=" * 80,
+            "LOGGING ANALYSIS",
+            "=" * 80,
+            "",
+            f"Total Python files analyzed: {analyzed_files}",
+            f"Total logging calls found: {total_logs}",
+            f"Logging configurations found: {len(logging_configs)}",
+            f"Unique logger names: {len(logger_names)}",
+            "",
+        ]
+
+        # Log level distribution
+        if total_logs > 0:
+            lines.extend(
+                [
+                    "=" * 80,
+                    "LOG LEVEL DISTRIBUTION",
+                    "=" * 80,
+                    "",
+                ]
+            )
+
+            for level in ["debug", "info", "warning", "error", "critical", "exception"]:
+                count = len(log_levels[level])
+                percentage = (count / total_logs * 100) if total_logs > 0 else 0
+                lines.append(f"{level.upper():12} {count:6} ({percentage:5.1f}%)")
+
+            lines.append("")
+
+            # Show sample locations for each level
+            lines.extend(
+                [
+                    "=" * 80,
+                    "LOG LEVEL LOCATIONS (sample)",
+                    "=" * 80,
+                    "",
+                ]
+            )
+
+            for level in ["debug", "info", "warning", "error", "critical", "exception"]:
+                locations = log_levels[level]
+                if locations:
+                    lines.append(f"{level.upper()}: {len(locations)} occurrence(s)")
+                    for loc in locations[:5]:  # Show first 5
+                        lines.append(f"  - {loc}")
+                    if len(locations) > 5:
+                        lines.append(f"  ... and {len(locations) - 5} more")
+                    lines.append("")
+
+        # Logger names
+        if logger_names:
+            lines.extend(
+                [
+                    "=" * 80,
+                    "LOGGER NAMES",
+                    "=" * 80,
+                    "",
+                ]
+            )
+            for name in sorted(logger_names):
+                lines.append(f"  - {name}")
+            lines.append("")
+
+        # Logging configurations
+        if logging_configs:
+            lines.extend(
+                [
+                    "=" * 80,
+                    "LOGGING CONFIGURATIONS (basicConfig)",
+                    "=" * 80,
+                    "",
+                ]
+            )
+            for config in logging_configs:
+                lines.append(f"  - {config}")
+            lines.append("")
+
+        # Recommendations
+        lines.extend(
+            [
+                "=" * 80,
+                "RECOMMENDATIONS",
+                "=" * 80,
+                "",
+            ]
+        )
+
+        if total_logs == 0:
+            lines.append("  - Consider adding logging to improve debuggability")
+        else:
+            debug_pct = len(log_levels["debug"]) / total_logs * 100 if total_logs > 0 else 0
+            info_pct = len(log_levels["info"]) / total_logs * 100 if total_logs > 0 else 0
+            
+            if debug_pct > 50:
+                lines.append("  - High percentage of DEBUG logs; consider reducing in production")
+            if info_pct < 10 and total_logs > 10:
+                lines.append("  - Low INFO logging; consider adding more informational logs")
+            if len(log_levels["exception"]) == 0 and len(log_levels["error"]) > 0:
+                lines.append("  - Consider using logger.exception() in exception handlers for stack traces")
+
+        lines.append("")
+
+        # Write report
+        output = "\n".join(lines)
+        dest = ctx.workdir / "meta" / "102_logging_analysis.txt"
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        dest.write_text(output, encoding="utf-8")
+
+        elapsed = int(time.time() - start)
+        return StepResult(self.name, "OK", elapsed, "")
+
+    def _extract_log_level(self, node: ast.Call) -> str:
+        """Extract log level from logging call."""
+        # Check for logger.level() or logging.level()
+        if isinstance(node.func, ast.Attribute):
+            method = node.func.attr.lower()
+            if method in ["debug", "info", "warning", "error", "critical", "exception"]:
+                # Verify it's a logging call
+                if isinstance(node.func.value, ast.Name):
+                    var_name = node.func.value.id.lower()
+                    if "log" in var_name:
+                        return method
+                elif isinstance(node.func.value, ast.Attribute):
+                    # logging.getLogger().debug()
+                    return method
+
+        return ""

pybundle/steps/memory_profile.py

@@ -0,0 +1,176 @@
+"""
+Memory profiling with tracemalloc - Milestone 3 (v1.4.0)
+"""
+
+from __future__ import annotations
+
+import subprocess
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from .base import StepResult
+from ..context import BundleContext
+
+
+@dataclass
+class MemoryProfileStep:
+    """
+    Memory profiling using tracemalloc to identify memory-consuming operations.
+
+    Outputs:
+    - logs/62_memory_profile.txt: Top memory-consuming functions and allocations
+    """
+
+    name: str = "memory_profile"
+
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+
+        if ctx.options.no_profile or not ctx.options.profile_memory:
+            return StepResult(self.name, "SKIP", 0, "memory profiling not enabled")
+
+        # Require pytest for memory profiling
+        if not ctx.tools.pytest:
+            return StepResult(self.name, "SKIP", 0, "pytest not found")
+
+        tests_dir = ctx.root / "tests"
+        if not tests_dir.is_dir():
+            return StepResult(self.name, "SKIP", 0, "no tests/ directory")
+
+        ctx.emit("  Running memory profiling on test suite")
+
+        # Create a temporary script to run pytest with tracemalloc
+        profile_script = self._create_profile_script(ctx.root)
+
+        try:
+            # Run the profiling script
+            result = subprocess.run(
+                [str(ctx.tools.python), str(profile_script)],
+                cwd=ctx.root,
+                capture_output=True,
+                text=True,
+                timeout=300,  # 5 minute timeout
+            )
+
+            # Write output
+            output_file = ctx.workdir / "logs" / "62_memory_profile.txt"
+            output_file.parent.mkdir(parents=True, exist_ok=True)
+
+            with output_file.open("w") as f:
+                f.write("=" * 70 + "\n")
+                f.write("MEMORY PROFILING (tracemalloc)\n")
+                f.write("=" * 70 + "\n\n")
+
+                if result.returncode == 0:
+                    f.write(result.stdout)
+                    if result.stderr:
+                        f.write("\n\nTest output:\n")
+                        f.write(result.stderr)
+                else:
+                    f.write("Memory profiling failed\n\n")
+                    f.write("STDOUT:\n")
+                    f.write(result.stdout)
+                    f.write("\n\nSTDERR:\n")
+                    f.write(result.stderr)
+
+            elapsed = int((time.time() - start) * 1000)
+            if result.returncode == 0:
+                return StepResult(self.name, "OK", elapsed)
+            else:
+                return StepResult(
+                    self.name, "FAIL", elapsed, f"exit {result.returncode}"
+                )
+
+        except subprocess.TimeoutExpired:
+            elapsed = int((time.time() - start) * 1000)
+            return StepResult(self.name, "FAIL", elapsed, "timeout")
+        except Exception as e:
+            elapsed = int((time.time() - start) * 1000)
+            return StepResult(self.name, "FAIL", elapsed, str(e))
+        finally:
+            # Clean up temporary script
+            if profile_script.exists():
+                profile_script.unlink()
+
+    def _create_profile_script(self, root: Path) -> Path:
+        """Create a temporary Python script that runs pytest with tracemalloc"""
+        script_path = root / ".pybundle_memory_profile.py"
+
+        script_content = '''"""Temporary memory profiling script for pybundle"""
+import tracemalloc
+import sys
+import pytest
+
+def format_size(size_bytes):
+    """Format bytes as human-readable string"""
+    for unit in ['B', 'KB', 'MB', 'GB']:
+        if size_bytes < 1024.0:
+            return f"{size_bytes:.2f} {unit}"
+        size_bytes /= 1024.0
+    return f"{size_bytes:.2f} TB"
+
+# Start tracing
+tracemalloc.start()
+
+# Record initial snapshot
+snapshot1 = tracemalloc.take_snapshot()
+
+# Run pytest
+exit_code = pytest.main(["-q"])
+
+# Take final snapshot
+snapshot2 = tracemalloc.take_snapshot()
+
+# Get traced memory
+current, peak = tracemalloc.get_traced_memory()
+
+# Stop tracing
+tracemalloc.stop()
+
+# Analyze differences
+print("=" * 70)
+print("MEMORY ALLOCATION SUMMARY")
+print("=" * 70)
+print()
+print(f"Peak memory usage: {format_size(peak)}")
+print()
+
+# Top allocations
+top_stats = snapshot2.compare_to(snapshot1, 'lineno')
+
+print("TOP 30 MEMORY ALLOCATIONS (by increase):")
+print("-" * 70)
+print(f"{'Size':<12} {'Count':<8} {'Location'}")
+print("-" * 70)
+
+for stat in top_stats[:30]:
+    print(f"{format_size(stat.size):<12} {stat.count:<8} {stat.traceback}")
+
+# Top by current size
+print()
+print("=" * 70)
+print("TOP 30 MEMORY CONSUMERS (by total size):")
+print("-" * 70)
+print(f"{'Size':<12} {'Count':<8} {'Location'}")
+print("-" * 70)
+
+current_stats = snapshot2.statistics('lineno')
+for stat in current_stats[:30]:
+    print(f"{format_size(stat.size):<12} {stat.count:<8} {stat.traceback}")
+
+print()
+print("=" * 70)
+print("RECOMMENDATIONS:")
+print("- Review functions with large memory allocations")
+print("- Check for memory leaks in repeatedly allocated objects")
+print("- Consider using generators for large data processing")
+print("=" * 70)
+
+sys.exit(exit_code)
+'''
+
+        with script_path.open("w") as f:
+            f.write(script_content)
+
+        return script_path