gwc-pybundle 2.1.2__py3-none-any.whl

pybundle/steps/config_docs.py

@@ -0,0 +1,319 @@
+"""Configuration documentation step.
+
+Extracts and documents environment variables and configuration options
+from Python source code.
+"""
+
+import ast
+import re
+import time
+from pathlib import Path
+from typing import Dict, List, Tuple
+from dataclasses import dataclass
+
+from .base import StepResult
+from ..context import BundleContext
+
+
+class ConfigVarExtractor(ast.NodeVisitor):
+    """AST visitor for extracting environment variable accesses."""
+
+    def __init__(self, filepath: str) -> None:
+        self.filepath = filepath
+        self.env_vars: List[Tuple[int, str, str]] = []  # (line, var_name, default)
+
+    def visit_Call(self, node: ast.Call) -> None:
+        """Visit function calls looking for os.getenv, os.environ.get, etc."""
+        # Check for os.getenv(var, default) or os.environ.get(var, default)
+        if isinstance(node.func, ast.Attribute):
+            if node.func.attr in ("getenv", "get"):
+                # Check if it's os.getenv or os.environ.get
+                if isinstance(node.func.value, ast.Name):
+                    if node.func.value.id == "os":
+                        self._extract_env_var(node)
+                elif isinstance(node.func.value, ast.Attribute):
+                    if (
+                        node.func.value.attr == "environ"
+                        and isinstance(node.func.value.value, ast.Name)
+                        and node.func.value.value.id == "os"
+                    ):
+                        self._extract_env_var(node)
+
+        self.generic_visit(node)
+
+    def visit_Subscript(self, node: ast.Subscript) -> None:
+        """Visit subscript operations like os.environ['VAR']."""
+        if isinstance(node.value, ast.Attribute):
+            if (
+                node.value.attr == "environ"
+                and isinstance(node.value.value, ast.Name)
+                and node.value.value.id == "os"
+            ):
+                if isinstance(node.slice, ast.Constant):
+                    var_name = node.slice.value
+                    if isinstance(var_name, str):
+                        self.env_vars.append((node.lineno, var_name, ""))
+
+        self.generic_visit(node)
+
+    def _extract_env_var(self, node: ast.Call):
+        """Extract environment variable from getenv/get call."""
+        if len(node.args) >= 1:
+            # First argument is the variable name
+            if isinstance(node.args[0], ast.Constant):
+                var_name = node.args[0].value
+                if isinstance(var_name, str):
+                    # Check for default value
+                    default = ""
+                    if len(node.args) >= 2:
+                        if isinstance(node.args[1], ast.Constant):
+                            default = repr(node.args[1].value)
+                        else:
+                            default = "<expression>"
+
+                    self.env_vars.append((node.lineno, var_name, default))
+
+
+@dataclass
+class ConfigDocumentationStep:
+    """Step that documents configuration variables."""
+
+    name: str = "config-docs"
+    outfile: str = "meta/83_config_vars.txt"
+
+    def run(self, context: BundleContext) -> StepResult:
+        """Extract and document configuration variables."""
+        start = time.time()
+
+        # Find all Python files
+        python_files = self._find_python_files(context.root)
+
+        if not python_files:
+            elapsed = time.time() - start
+            note = "No Python files found"
+            return StepResult(self.name, "SKIP", int(elapsed), note)
+
+        # Extract environment variables from code
+        env_vars: List[Tuple[str, int, str, str]] = self._extract_env_vars(
+            python_files, context.root
+        )
+
+        # Look for config files
+        config_files = self._find_config_files(context.root)
+
+        # Parse config file patterns
+        config_patterns = self._extract_config_patterns(config_files, context.root)
+
+        elapsed = int(time.time() - start)
+
+        # Write report
+        log_path = context.workdir / self.outfile
+        log_path.parent.mkdir(parents=True, exist_ok=True)
+        with open(log_path, "w") as f:
+            f.write("=" * 80 + "\n")
+            f.write("CONFIGURATION DOCUMENTATION\n")
+            f.write("=" * 80 + "\n\n")
+
+            # Environment variables section
+            if env_vars:
+                f.write("Environment Variables Used:\n")
+                f.write("-" * 80 + "\n\n")
+
+                # Group by variable name
+                vars_grouped: Dict[str, List[Tuple[str, int, str]]] = {}
+                for filepath, lineno, var_name, default in env_vars:
+                    if var_name not in vars_grouped:
+                        vars_grouped[var_name] = []
+                    vars_grouped[var_name].append((filepath, lineno, default))
+
+                # Sort by variable name
+                for var_name in sorted(vars_grouped.keys()):
+                    locations = vars_grouped[var_name]
+                    f.write(f"{var_name}\n")
+
+                    # Show default if consistent
+                    defaults = [d for _, _, d in locations if d is not None]
+                    if defaults:
+                        if len(set(defaults)) == 1:
+                            f.write(f"  Default: {defaults[0]}\n")
+                        else:
+                            f.write(f"  Defaults: {', '.join(set(defaults))}\n")
+
+                    f.write("  Used in:\n")
+                    for filepath, lineno, _ in sorted(
+                        locations, key=lambda x: (x[0], x[1])
+                    ):
+                        f.write(f"    {filepath}:{lineno}\n")
+
+                    f.write("\n")
+
+                f.write(f"Total: {len(vars_grouped)} unique environment variables\n\n")
+            else:
+                f.write("No environment variables found in source code.\n\n")
+
+            # Configuration files section
+            if config_files:
+                f.write("Configuration Files:\n")
+                f.write("-" * 80 + "\n")
+                for config_file in sorted(config_files):
+                    f.write(f"  {config_file}\n")
+                f.write("\n")
+
+            # Configuration patterns section
+            if config_patterns:
+                f.write("Configuration Patterns Detected:\n")
+                f.write("-" * 80 + "\n")
+                for pattern, pattern_locations in sorted(config_patterns.items()):
+                    f.write(f"\n{pattern}\n")
+                    for filepath, lineno in pattern_locations[:5]:  # Limit to 5
+                        f.write(f"  {filepath}:{lineno}\n")
+                    if len(pattern_locations) > 5:
+                        f.write(f"  ... and {len(pattern_locations) - 5} more\n")
+                f.write("\n")
+
+            f.write("=" * 80 + "\n")
+            f.write("Configuration documentation complete\n")
+            f.write("=" * 80 + "\n")
+
+        # Determine status
+        total_items = len(env_vars) + len(config_patterns)
+        if total_items > 0:
+            status = "OK"
+            note = f"Found {len(set(v[2] for v in env_vars))} env vars, {len(config_files)} config files"
+        else:
+            status = "SKIP"
+            note = "No configuration found"
+
+        return StepResult(self.name, status, elapsed, note)
+
+    def _find_python_files(self, root: Path) -> List[Path]:
+        """Find all Python source files."""
+        python_files = []
+        exclude_dirs = {
+            "__pycache__",
+            ".git",
+            ".tox",
+            "venv",
+            "env",
+            ".venv",
+            ".env",
+            "node_modules",
+            "artifacts",
+            "build",
+            "dist",
+            ".pytest_cache",
+            ".mypy_cache",
+            ".ruff_cache",
+            ".pybundle-venv",  # pybundle's venv
+        }
+
+        for path in root.rglob("*.py"):
+            if any(part in exclude_dirs for part in path.parts):
+                continue
+            python_files.append(path)
+
+        return python_files
+
+    def _extract_env_vars(
+        self, python_files: List[Path], root: Path
+    ) -> List[Tuple[str, int, str, str]]:
+        """Extract environment variables from Python files.
+
+        Returns list of (filepath, lineno, var_name, default).
+        """
+        env_vars = []
+
+        for filepath in python_files:
+            try:
+                with open(filepath, "r", encoding="utf-8") as f:
+                    source = f.read()
+
+                tree = ast.parse(source, filename=str(filepath))
+                extractor = ConfigVarExtractor(str(filepath))
+                extractor.visit(tree)
+
+                rel_path = str(filepath.relative_to(root))
+                for lineno, var_name, default in extractor.env_vars:
+                    env_vars.append((rel_path, lineno, var_name, default))
+
+            except (SyntaxError, Exception):
+                # Skip files that can't be parsed
+                continue
+
+        return env_vars
+
+    def _find_config_files(self, root: Path) -> List[str]:
+        """Find configuration files in the project."""
+        config_patterns = [
+            "*.ini",
+            "*.cfg",
+            "*.conf",
+            "*.yaml",
+            "*.yml",
+            "*.toml",
+            ".env*",
+            "config.*",
+        ]
+
+        config_files = []
+        exclude_dirs = {
+            "__pycache__",
+            ".git",
+            ".tox",
+            "venv",
+            "env",
+            ".venv",
+            "node_modules",
+            "artifacts",
+            "build",
+            "dist",
+            ".pybundle-venv",  # pybundle's venv
+        }
+
+        for pattern in config_patterns:
+            for path in root.rglob(pattern):
+                if path.is_file():
+                    if any(part in exclude_dirs for part in path.parts):
+                        continue
+                    try:
+                        rel_path = str(path.relative_to(root))
+                        config_files.append(rel_path)
+                    except ValueError:
+                        continue
+
+        return config_files
+
+    def _extract_config_patterns(
+        self, config_files: List[str], root: Path
+    ) -> Dict[str, List[Tuple[str, int]]]:
+        """Extract configuration patterns from config files.
+
+        Returns dict of {pattern: [(filepath, lineno)]}.
+        """
+        patterns: Dict[str, List[Tuple[str, int]]] = {}
+
+        # Simple pattern matching for common config formats
+        var_patterns = [
+            (re.compile(r"^([A-Z_][A-Z0-9_]*)\s*="), "Environment variable"),
+            (re.compile(r"^\s*([a-z_][a-z0-9_]*)\s*:"), "YAML/INI key"),
+            (re.compile(r"^\s*\[([^\]]+)\]"), "Section header"),
+        ]
+
+        for config_file in config_files:
+            filepath = root / config_file
+            try:
+                with open(filepath, "r", encoding="utf-8") as f:
+                    for lineno, line in enumerate(f, 1):
+                        for pattern, description in var_patterns:
+                            match = pattern.match(line)
+                            if match:
+                                key = f"{description}: {match.group(1)}"
+                                if key not in patterns:
+                                    patterns[key] = []
+                                patterns[key].append((config_file, lineno))
+                                break
+            except Exception:
+                # Skip files that can't be read
+                continue
+
+        return patterns

pybundle/steps/config_validation.py

@@ -0,0 +1,302 @@
+"""
+Step: Configuration Schema Validation
+Validate configuration files and environment variable schemas.
+"""
+
+import re
+from pathlib import Path
+from typing import Dict, List, Set, Optional
+
+from .base import Step, StepResult
+
+
+class ConfigValidationStep(Step):
+    """Validate configuration schemas and .env completeness."""
+
+    name = "config validation"
+
+    def run(self, ctx: "BundleContext") -> StepResult:  # type: ignore[name-defined]
+        """Validate configuration schemas and environment files."""
+        import time
+
+        start = time.time()
+
+        root = ctx.root
+
+        # Find configuration files
+        config_files = self._find_config_files(root)
+        env_file = root / ".env"
+        env_example = root / ".env.example"
+
+        # Analyze Pydantic settings
+        pydantic_settings = self._analyze_pydantic_settings(root)
+
+        # Compare .env and .env.example
+        env_comparison = self._compare_env_files(env_file, env_example)
+
+        # Generate report
+        lines = [
+            "=" * 80,
+            "CONFIGURATION VALIDATION REPORT",
+            "=" * 80,
+            "",
+            f"Project root: {root}",
+            "",
+        ]
+
+        # Configuration files found
+        lines.extend(
+            [
+                "=" * 80,
+                "CONFIGURATION FILES",
+                "=" * 80,
+                "",
+            ]
+        )
+
+        if config_files:
+            for file_type, files in config_files.items():
+                lines.append(f"{file_type}:")
+                for f in files:
+                    rel_path = f.relative_to(root)
+                    lines.append(f"  - {rel_path}")
+            lines.append("")
+        else:
+            lines.append("No standard configuration files found")
+            lines.append("")
+
+        # Pydantic settings analysis
+        if pydantic_settings:
+            lines.extend(
+                [
+                    "=" * 80,
+                    "PYDANTIC SETTINGS ANALYSIS",
+                    "=" * 80,
+                    "",
+                ]
+            )
+
+            for settings_class, details in pydantic_settings.items():
+                lines.append(f"Class: {settings_class}")
+                lines.append(f"  Location: {details['file']}")
+                lines.append(f"  Fields: {details['field_count']}")
+
+                if details["fields"]:
+                    lines.append("  Field annotations:")
+                    for field, field_type in details["fields"][:10]:
+                        lines.append(f"    - {field}: {field_type}")
+                    if len(details["fields"]) > 10:
+                        lines.append(f"    ... and {len(details['fields']) - 10} more")
+
+                lines.append("")
+
+        # .env file analysis
+        if env_file.exists() or env_example.exists():
+            lines.extend(
+                [
+                    "=" * 80,
+                    "ENVIRONMENT FILES ANALYSIS",
+                    "=" * 80,
+                    "",
+                ]
+            )
+
+            if env_example.exists():
+                lines.append("✓ .env.example found")
+                example_count = len(env_comparison["example_vars"])
+                lines.append(f"  Documented variables: {example_count}")
+
+                if env_comparison["example_vars"]:
+                    lines.append("  Variables:")
+                    for var in sorted(list(env_comparison["example_vars"]))[:15]:
+                        lines.append(f"    - {var}")
+                    if len(env_comparison["example_vars"]) > 15:
+                        lines.append(
+                            f"    ... and {len(env_comparison['example_vars']) - 15} more"
+                        )
+
+                lines.append("")
+
+            if env_file.exists():
+                lines.append("✓ .env found")
+                env_count = len(env_comparison["env_vars"])
+                lines.append(f"  Variables set: {env_count}")
+                lines.append("")
+
+            # Comparison
+            if env_file.exists() and env_example.exists():
+                missing = env_comparison["missing_in_env"]
+                extra = env_comparison["extra_in_env"]
+
+                lines.append("ENV FILE COMPARISON:")
+                if missing:
+                    lines.append(
+                        f"  ⚠ Missing in .env (documented but not set): {len(missing)}"
+                    )
+                    for var in sorted(missing)[:10]:
+                        lines.append(f"    - {var}")
+                    if len(missing) > 10:
+                        lines.append(f"    ... and {len(missing) - 10} more")
+                else:
+                    lines.append("  ✓ All documented variables are set in .env")
+
+                if extra:
+                    lines.append(
+                        f"  ⚠ Extra in .env (not in .env.example): {len(extra)}"
+                    )
+                    for var in sorted(extra)[:10]:
+                        lines.append(f"    - {var}")
+                    if len(extra) > 10:
+                        lines.append(f"    ... and {len(extra) - 10} more")
+                else:
+                    lines.append("  ✓ All .env variables are documented in .env.example")
+
+                lines.append("")
+
+        # Recommendations
+        lines.extend(
+            [
+                "=" * 80,
+                "RECOMMENDATIONS",
+                "=" * 80,
+                "",
+            ]
+        )
+
+        if not env_example.exists():
+            lines.append("  - Create .env.example to document required variables")
+        if env_file.exists() and not env_example.exists():
+            lines.append("  - Use .env.example to version-control configuration schema")
+        if pydantic_settings:
+            lines.append("  - Pydantic settings detected: leverage validation at runtime")
+        if not pydantic_settings and (env_file.exists() or env_example.exists()):
+            lines.append("  - Consider using Pydantic BaseSettings for type-safe configuration")
+
+        lines.append("")
+
+        # Write report
+        output = "\n".join(lines)
+        dest = ctx.workdir / "logs" / "120_config_validation.txt"
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        dest.write_text(output, encoding="utf-8")
+
+        elapsed = int(time.time() - start)
+        return StepResult(self.name, "OK", elapsed, "")
+
+    def _find_config_files(self, root: Path) -> Dict[str, List[Path]]:
+        """Find common configuration files."""
+        config_patterns = {
+            "YAML/YML": ["*.yaml", "*.yml"],
+            "TOML": ["*.toml"],
+            "JSON": ["*.json"],
+            "INI": ["*.ini", "*.cfg"],
+            "YAML/JSON": ["config.yaml", "config.json", "settings.yaml"],
+        }
+
+        found = {}
+        for file_type, patterns in config_patterns.items():
+            files = []
+            for pattern in patterns:
+                files.extend(root.glob(pattern))
+            if files:
+                found[file_type] = files
+
+        return found
+
+    def _analyze_pydantic_settings(self, root: Path) -> Dict[str, dict]:
+        """Find and analyze Pydantic BaseSettings classes."""
+        settings = {}
+
+        python_files = root.rglob("*.py")
+        for py_file in python_files:
+            # Skip venv and cache
+            if any(
+                part in py_file.parts
+                for part in ["venv", ".venv", "env", "__pycache__", "site-packages"]
+            ):
+                continue
+
+            try:
+                source = py_file.read_text(encoding="utf-8", errors="ignore")
+
+                # Check if file imports BaseSettings
+                if "BaseSettings" not in source:
+                    continue
+
+                # Find class definitions that inherit from BaseSettings
+                class_pattern = (
+                    r"class\s+(\w+)\s*\(\s*.*BaseSettings.*\s*\):"
+                )
+                matches = re.finditer(class_pattern, source)
+
+                for match in matches:
+                    class_name = match.group(1)
+                    rel_path = py_file.relative_to(root)
+
+                    # Extract fields (simple annotation parsing)
+                    fields = []
+                    class_body_start = source.find(match.group(0)) + len(match.group(0))
+                    class_body = source[class_body_start :]
+
+                    # Find annotations before any method definitions
+                    annotation_pattern = r"(\w+)\s*:\s*([^\n=]+)"
+                    for field_match in re.finditer(annotation_pattern, class_body):
+                        field_name = field_match.group(1)
+                        field_type = field_match.group(2).strip()
+
+                        # Stop at method definitions
+                        if field_name == "def":
+                            break
+
+                        fields.append((field_name, field_type))
+
+                    settings[class_name] = {
+                        "file": str(rel_path),
+                        "field_count": len(fields),
+                        "fields": fields,
+                    }
+
+            except (OSError, UnicodeDecodeError):
+                continue
+
+        return settings
+
+    def _compare_env_files(
+        self, env_file: Path, env_example: Path
+    ) -> Dict[str, Set[str]]:
+        """Compare .env and .env.example files."""
+        env_vars = set()
+        example_vars = set()
+
+        # Parse .env
+        if env_file.exists():
+            try:
+                content = env_file.read_text(encoding="utf-8", errors="ignore")
+                for line in content.split("\n"):
+                    line = line.strip()
+                    if line and not line.startswith("#") and "=" in line:
+                        var_name = line.split("=")[0].strip()
+                        if var_name:
+                            env_vars.add(var_name)
+            except (OSError, UnicodeDecodeError):
+                pass
+
+        # Parse .env.example
+        if env_example.exists():
+            try:
+                content = env_example.read_text(encoding="utf-8", errors="ignore")
+                for line in content.split("\n"):
+                    line = line.strip()
+                    if line and not line.startswith("#") and "=" in line:
+                        var_name = line.split("=")[0].strip()
+                        if var_name:
+                            example_vars.add(var_name)
+            except (OSError, UnicodeDecodeError):
+                pass
+
+        return {
+            "env_vars": env_vars,
+            "example_vars": example_vars,
+            "missing_in_env": example_vars - env_vars,
+            "extra_in_env": env_vars - example_vars,
+        }