gwc-pybundle 2.1.2__py3-none-any.whl

pybundle/steps/env_var_usage.py

@@ -0,0 +1,253 @@
+"""
+Step: Environment Variable Usage
+Track usage of environment variables via os.getenv, os.environ patterns.
+"""
+
+import ast
+import re
+from pathlib import Path
+from typing import Dict, List, Set
+
+from .base import Step, StepResult
+
+
+class EnvVarUsageStep(Step):
+    """Analyze environment variable usage patterns in Python code."""
+
+    name = "env var usage"
+
+    def run(self, ctx: "BundleContext") -> StepResult:  # type: ignore[name-defined]
+        """Find all environment variable accesses."""
+        import time
+
+        start = time.time()
+
+        root = ctx.root
+        python_files = sorted(root.rglob("*.py"))
+        if not python_files:
+            return StepResult(self.name, "SKIP", int(time.time() - start), "No Python files found")
+
+        # Track environment variable patterns
+        env_vars: Dict[str, List[str]] = {}  # var_name -> [file:line, ...]
+        env_patterns = {
+            "os.getenv": [],
+            "os.environ.get": [],
+            "os.environ[]": [],
+            "dotenv": [],
+        }
+        analyzed_files = 0
+
+        # Regex patterns for detection
+        getenv_pattern = re.compile(r'os\.getenv\(["\']([^"\']+)["\']')
+        environ_get_pattern = re.compile(r'os\.environ\.get\(["\']([^"\']+)["\']')
+        environ_bracket_pattern = re.compile(r'os\.environ\[["\']([^"\']+)["\']\]')
+        dotenv_pattern = re.compile(r'load_dotenv|from\s+dotenv\s+import')
+
+        for py_file in python_files:
+            # Skip non-user code
+            if any(
+                part in py_file.parts
+                for part in [
+                    "venv",
+                    ".venv",
+                    "env",
+                    "site-packages",
+                    "__pycache__",
+                    ".git",
+                    "node_modules",
+                ]
+            ):
+                continue
+
+            analyzed_files += 1
+
+            try:
+                source = py_file.read_text(encoding="utf-8", errors="ignore")
+                rel_path = py_file.relative_to(root)
+
+                # Check for environment variable usage patterns
+                if "os.getenv" not in source and "os.environ" not in source and "dotenv" not in source:
+                    continue
+
+                # Check for dotenv usage
+                if dotenv_pattern.search(source):
+                    env_patterns["dotenv"].append(str(rel_path))
+
+                # Parse with regex (faster for simple patterns)
+                for line_num, line in enumerate(source.split("\n"), start=1):
+                    location = f"{rel_path}:{line_num}"
+
+                    # os.getenv()
+                    for match in getenv_pattern.finditer(line):
+                        var_name = match.group(1)
+                        if var_name not in env_vars:
+                            env_vars[var_name] = []
+                        env_vars[var_name].append(location)
+                        env_patterns["os.getenv"].append(location)
+
+                    # os.environ.get()
+                    for match in environ_get_pattern.finditer(line):
+                        var_name = match.group(1)
+                        if var_name not in env_vars:
+                            env_vars[var_name] = []
+                        env_vars[var_name].append(location)
+                        env_patterns["os.environ.get"].append(location)
+
+                    # os.environ[]
+                    for match in environ_bracket_pattern.finditer(line):
+                        var_name = match.group(1)
+                        if var_name not in env_vars:
+                            env_vars[var_name] = []
+                        env_vars[var_name].append(location)
+                        env_patterns["os.environ[]"].append(location)
+
+            except (UnicodeDecodeError, OSError):
+                continue
+
+        # Calculate statistics
+        total_vars = len(env_vars)
+        total_accesses = sum(len(locations) for locations in env_vars.values())
+
+        # Generate report
+        lines = [
+            "=" * 80,
+            "ENVIRONMENT VARIABLE USAGE ANALYSIS",
+            "=" * 80,
+            "",
+            f"Total Python files analyzed: {analyzed_files}",
+            f"Unique environment variables: {total_vars}",
+            f"Total environment variable accesses: {total_accesses}",
+            "",
+        ]
+
+        # Usage patterns
+        lines.extend(
+            [
+                "=" * 80,
+                "USAGE PATTERNS",
+                "=" * 80,
+                "",
+            ]
+        )
+
+        for pattern, locations in env_patterns.items():
+            lines.append(f"{pattern:20} {len(locations):5} occurrence(s)")
+        lines.append("")
+
+        # Dotenv detection
+        if env_patterns["dotenv"]:
+            lines.extend(
+                [
+                    "=" * 80,
+                    "DOTENV USAGE DETECTED",
+                    "=" * 80,
+                    "",
+                ]
+            )
+            for file in env_patterns["dotenv"]:
+                lines.append(f"  - {file}")
+            lines.append("")
+
+        # Environment variables
+        if env_vars:
+            lines.extend(
+                [
+                    "=" * 80,
+                    "ENVIRONMENT VARIABLES (sorted by frequency)",
+                    "=" * 80,
+                    "",
+                ]
+            )
+
+            sorted_vars = sorted(env_vars.items(), key=lambda x: len(x[1]), reverse=True)
+            for var_name, locations in sorted_vars:
+                lines.append(f"{var_name}: {len(locations)} access(es)")
+                for loc in locations[:3]:  # Show first 3 locations
+                    lines.append(f"  - {loc}")
+                if len(locations) > 3:
+                    lines.append(f"  ... and {len(locations) - 3} more")
+                lines.append("")
+
+        # Check for .env file
+        env_file = root / ".env"
+        env_example = root / ".env.example"
+        
+        lines.extend(
+            [
+                "=" * 80,
+                "ENVIRONMENT FILES",
+                "=" * 80,
+                "",
+            ]
+        )
+
+        if env_file.exists():
+            lines.append("  ✓ .env file found")
+        else:
+            lines.append("  ✗ .env file not found")
+
+        if env_example.exists():
+            lines.append("  ✓ .env.example file found")
+            # Parse .env.example for documented variables
+            try:
+                example_content = env_example.read_text(encoding="utf-8")
+                example_vars = set()
+                for line_content in example_content.split("\n"):
+                    line_content = line_content.strip()
+                    if line_content and not line_content.startswith("#"):
+                        if "=" in line_content:
+                            var = line_content.split("=")[0].strip()
+                            example_vars.add(var)
+                
+                if example_vars:
+                    lines.append(f"  - Documents {len(example_vars)} variable(s)")
+                    
+                    # Check for undocumented variables
+                    undocumented = set(env_vars.keys()) - example_vars
+                    if undocumented:
+                        lines.append(f"  - {len(undocumented)} variable(s) used but not documented:")
+                        for var in sorted(undocumented)[:10]:
+                            lines.append(f"      {var}")
+                        if len(undocumented) > 10:
+                            lines.append(f"      ... and {len(undocumented) - 10} more")
+            except (OSError, UnicodeDecodeError):
+                pass
+        else:
+            lines.append("  ✗ .env.example file not found")
+
+        lines.append("")
+
+        # Recommendations
+        lines.extend(
+            [
+                "=" * 80,
+                "RECOMMENDATIONS",
+                "=" * 80,
+                "",
+            ]
+        )
+
+        if total_vars == 0:
+            lines.append("  - No environment variables detected")
+        else:
+            if not env_example.exists():
+                lines.append("  - Create .env.example to document required environment variables")
+            
+            # Check for hardcoded defaults
+            if len(env_patterns["os.environ[]"]) > 0:
+                lines.append("  - Consider using os.getenv() with defaults instead of os.environ[]")
+                lines.append("    to avoid KeyError exceptions")
+            
+            if not env_patterns["dotenv"]:
+                lines.append("  - Consider using python-dotenv to load .env files")
+
+        lines.append("")
+
+        # Write report
+        output = "\n".join(lines)
+        dest = ctx.workdir / "meta" / "104_env_var_usage.txt"
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        dest.write_text(output, encoding="utf-8")
+
+        elapsed = int(time.time() - start)
+        return StepResult(self.name, "OK", elapsed, "")

pybundle/steps/error_refs.py

@@ -0,0 +1,204 @@
+from __future__ import annotations
+
+import re
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from .base import StepResult
+from ..context import BundleContext
+
+
+DEFAULT_EXCLUDE_PREFIXES = (
+    ".git/",
+    ".venv/",
+    ".mypy_cache/",
+    ".ruff_cache/",
+    ".pytest_cache/",
+    "__pycache__/",
+    "node_modules/",
+    "dist/",
+    "build/",
+    "artifacts/",
+)
+
+# Patterns based on your bash sed rules:
+# 1) tool-style: path:line(:col)...
+_RE_COLON_LINE = re.compile(r"^([A-Za-z0-9_.\/-]+\.[A-Za-z0-9]+):\d+(?::\d+)?\b.*$")
+
+# 2) pytest traceback: File "path", line N
+_RE_PYTEST_FILE = re.compile(r'^\s*File "([^"]+)", line \d+\b.*$')
+
+# 3) mypy: (optional "mypy:") ./path:line: (error|note|warning):
+_RE_MYPY_LINE = re.compile(
+    r"^(?:mypy:\s*)?(?:\./)?([A-Za-z0-9_.\/-]+\.[A-Za-z0-9]+):\d+:\s*(?:error|note|warning):.*$"
+)
+
+# 4) mypy rare: path: (error|note|warning): ...
+_RE_MYPY_NOLINE = re.compile(
+    r"^(?:mypy:\s*)?(?:\./)?([A-Za-z0-9_.\/-]+\.[A-Za-z0-9]+):\s*(?:error|note|warning):.*$"
+)
+
+
+def _normalize_to_repo_rel(root: Path, p: str) -> str | None:
+    p = p.strip()
+    if not p:
+        return None
+
+    # remove leading ./ for consistency
+    if p.startswith("./"):
+        p = p[2:]
+
+    # absolute path -> must be under repo root
+    if p.startswith("/"):
+        try:
+            rp = Path(p).resolve()
+            rr = rp.relative_to(root.resolve())
+            return str(rr).replace("\\", "/")
+        except Exception:
+            return None
+
+    # relative path
+    return p.replace("\\", "/")
+
+
+def _is_allowed_repo_file(root: Path, rel: str) -> bool:
+    rel = rel.lstrip("./")
+    if not rel or rel.endswith("/"):
+        return False
+
+    # exclude common junk
+    for pref in DEFAULT_EXCLUDE_PREFIXES:
+        if rel.startswith(pref):
+            return False
+    if "/__pycache__/" in f"/{rel}/":
+        return False
+
+    # must exist and be a file inside repo
+    fp = (root / rel).resolve()
+    try:
+        fp.relative_to(root.resolve())
+    except Exception:
+        return False
+
+    return fp.is_file()
+
+
+def _extract_paths_from_text(text: str) -> list[str]:
+    out: list[str] = []
+    for line in text.splitlines():
+        m = _RE_COLON_LINE.match(line)
+        if m:
+            out.append(m.group(1))
+            continue
+
+        m = _RE_PYTEST_FILE.match(line)
+        if m:
+            out.append(m.group(1))
+            continue
+
+        m = _RE_MYPY_LINE.match(line)
+        if m:
+            out.append(m.group(1))
+            continue
+
+        m = _RE_MYPY_NOLINE.match(line)
+        if m:
+            out.append(m.group(1))
+            continue
+
+    return out
+
+
+@dataclass
+class ErrorReferencedFilesStep:
+    name: str = "collect error-referenced files"
+    max_files: int = 250
+    # Paths are relative to the bundle workdir
+    log_files: list[str] | None = None
+
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+
+        # Default set aligned to our step numbers
+        log_files = self.log_files or [
+            "logs/31_ruff_check.txt",
+            "logs/32_ruff_format_check.txt",
+            "logs/33_mypy.txt",
+            "logs/34_pytest_q.txt",
+        ]
+
+        out_list = ctx.workdir / "error_files_from_logs.txt"
+        out_count = ctx.workdir / "error_refs_count.txt"
+        report = ctx.metadir / "60_error_refs_report.txt"
+
+        dest_root = ctx.srcdir / "_error_refs"
+        dest_root.mkdir(parents=True, exist_ok=True)
+
+        # Collect candidate paths
+        candidates: set[str] = set()
+        scanned = 0
+        missing_logs = 0
+
+        for lf in log_files:
+            lp = ctx.workdir / lf
+            if not lp.is_file():
+                missing_logs += 1
+                continue
+            scanned += 1
+            try:
+                txt = lp.read_text(encoding="utf-8", errors="replace")
+            except Exception:
+                continue
+
+            for raw in _extract_paths_from_text(txt):
+                norm = _normalize_to_repo_rel(ctx.root, raw)
+                if norm:
+                    candidates.add(norm)
+
+        # Normalize / filter to real repo files
+        allowed = sorted([p for p in candidates if _is_allowed_repo_file(ctx.root, p)])
+
+        # Write list file (even if empty)
+        out_list.write_text(
+            "\n".join(allowed) + ("\n" if allowed else ""), encoding="utf-8"
+        )
+
+        # Copy up to max_files
+        copied = 0
+        for rel in allowed:
+            if copied >= self.max_files:
+                break
+            src = ctx.root / rel
+            dst = dest_root / rel
+            dst.parent.mkdir(parents=True, exist_ok=True)
+            try:
+                # preserve mode/timestamps
+                dst.write_bytes(src.read_bytes())
+                copied += 1
+            except Exception:
+                continue
+
+        out_count.write_text(f"{copied}\n", encoding="utf-8")
+
+        report.write_text(
+            "\n".join(
+                [
+                    f"scanned_logs={scanned}",
+                    f"missing_logs={missing_logs}",
+                    f"candidates_total={len(candidates)}",
+                    f"allowed_repo_files={len(allowed)}",
+                    f"copied={copied}",
+                    f"max_files={self.max_files}",
+                    "dest=src/_error_refs",
+                ]
+            )
+            + "\n",
+            encoding="utf-8",
+        )
+
+        dur = int(time.time() - start)
+        note = f"allowed={len(allowed)} copied={copied}"
+        if copied >= self.max_files:
+            note += " (HIT MAX)"
+        return StepResult(self.name, "PASS", dur, note)