gwc-pybundle 2.1.2__py3-none-any.whl

pybundle/steps/sqlalchemy_validation.py

@@ -0,0 +1,269 @@
+"""
+Step: SQLAlchemy Validation
+Validate SQLAlchemy models and relationships.
+"""
+
+import re
+from pathlib import Path
+from typing import Dict, List, Any, Optional, Tuple
+
+from .base import Step, StepResult
+
+
+class SQLAlchemyValidationStep(Step):
+    """Validate SQLAlchemy model definitions and relationships."""
+
+    name = "sqlalchemy validation"
+
+    def run(self, ctx: "BundleContext") -> StepResult:  # type: ignore[name-defined]
+        """Validate SQLAlchemy models."""
+        import time
+
+        start = time.time()
+
+        root = ctx.root
+
+        # Analyze models
+        models = self._find_sqlalchemy_models(root)
+        if not models:
+            elapsed = int(time.time() - start)
+            return StepResult(
+                self.name, "SKIP", elapsed, "No SQLAlchemy models found"
+            )
+
+        # Validate relationships
+        issues = self._validate_relationships(root, models)
+
+        # Generate report
+        lines = [
+            "=" * 80,
+            "SQLALCHEMY VALIDATION REPORT",
+            "=" * 80,
+            "",
+        ]
+
+        lines.extend(
+            [
+                "SUMMARY",
+                "=" * 80,
+                f"Models found: {len(models)}",
+                "",
+            ]
+        )
+
+        # Models list
+        lines.extend(
+            [
+                "MODEL DEFINITIONS",
+                "-" * 80,
+                "",
+            ]
+        )
+
+        for model in sorted(models, key=lambda m: m["name"]):
+            lines.append(f"  {model['name']}")
+            lines.append(f"    File: {model['file']}")
+            lines.append(f"    Table: {model.get('table', '(auto-generated)')}")
+            if model.get("columns"):
+                lines.append(f"    Columns: {', '.join(model['columns'][:5])}")
+                if len(model["columns"]) > 5:
+                    lines.append(f"             ... and {len(model['columns']) - 5} more")
+            if model.get("relationships"):
+                lines.append(f"    Relationships: {', '.join(model['relationships'])}")
+            lines.append("")
+
+        # Validation issues
+        lines.extend(
+            [
+                "VALIDATION ISSUES",
+                "=" * 80,
+                "",
+            ]
+        )
+
+        if issues:
+            error_count = sum(1 for _, level, _, _ in issues if level == "ERROR")
+            warning_count = sum(1 for _, level, _, _ in issues if level == "WARNING")
+
+            lines.append(
+                f"Found {len(issues)} issue(s): {error_count} error(s), {warning_count} warning(s)"
+            )
+            lines.append("")
+
+            for model_name, level, issue, detail in issues:
+                icon = "✗" if level == "ERROR" else "⚠"
+                lines.append(f"  {icon} [{model_name}] {issue}")
+                if detail:
+                    lines.append(f"      {detail}")
+            lines.append("")
+        else:
+            lines.append("✓ No validation issues detected")
+            lines.append("")
+
+        # Relationship analysis
+        lines.extend(
+            [
+                "RELATIONSHIP ANALYSIS",
+                "-" * 80,
+                "",
+            ]
+        )
+
+        relationships = self._analyze_relationships(models)
+        if relationships:
+            for rel in relationships:
+                lines.append(f"  {rel}")
+        else:
+            lines.append("  ℹ No relationships defined")
+
+        lines.append("")
+
+        # Recommendations
+        lines.extend(
+            [
+                "=" * 80,
+                "BEST PRACTICES & RECOMMENDATIONS",
+                "=" * 80,
+                "",
+                "1. MODEL DESIGN",
+                "   ✓ Use descriptive model and column names",
+                "   ✓ Define primary keys explicitly",
+                "   ✓ Use UUID or serial primary keys",
+                "   ✓ Add created_at and updated_at timestamps",
+                "",
+                "2. RELATIONSHIPS",
+                "   ✓ Use relationship() for ORM-level access",
+                "   ✓ Define foreign keys explicitly",
+                "   ✓ Set cascade rules (delete-orphan for children)",
+                "   ✓ Use back_populates for bidirectional relationships",
+                "",
+                "3. CONSTRAINTS",
+                "   ✓ Add CHECK constraints for valid data",
+                "   ✓ Use nullable=False for required fields",
+                "   ✓ Add unique=True for unique fields",
+                "   ✓ Add indexes to frequently queried fields",
+                "",
+                "4. INHERITANCE",
+                "   ✓ Consider single table inheritance for polymorphism",
+                "   ✓ Use joined table inheritance for distinct tables",
+                "   ✓ Document inheritance strategy clearly",
+                "",
+                "5. SERIALIZATION",
+                "   ✓ Use Pydantic models for API responses",
+                "   ✓ Define __repr__ for debugging",
+                "   ✓ Define to_dict() for serialization",
+                "   ✓ Exclude sensitive fields from serialization",
+                "",
+                "6. QUERIES",
+                "   ✓ Use lazy='select' or 'selectin' for relationships",
+                "   ✓ Use .only() or .defer() to limit columns",
+                "   ✓ Use exists() for existence checks",
+                "   ✓ Use bulk_insert_mappings() for bulk operations",
+                "",
+                "7. TESTING",
+                "   ✓ Test model creation and validation",
+                "   ✓ Test relationship cascades",
+                "   ✓ Test constraint violations",
+                "   ✓ Use pytest fixtures for model factories",
+                "",
+            ]
+        )
+
+        # Write report
+        output = "\n".join(lines)
+        dest = ctx.workdir / "logs" / "152_sqlalchemy_validation.txt"
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        dest.write_text(output, encoding="utf-8")
+
+        elapsed = int(time.time() - start)
+        return StepResult(self.name, "OK", elapsed, "")
+
+    def _find_sqlalchemy_models(self, root: Path) -> List[Dict[str, Any]]:
+        """Find SQLAlchemy model definitions."""
+        models = []
+        python_files = list(root.rglob("*.py"))
+
+        for py_file in python_files:
+            if any(
+                part in py_file.parts
+                for part in ["venv", ".venv", "env", "__pycache__", "site-packages"]
+            ):
+                continue
+
+            try:
+                source = py_file.read_text(encoding="utf-8", errors="ignore")
+
+                if "from sqlalchemy" not in source:
+                    continue
+
+                # Look for model class definitions
+                # Pattern: class ModelName(Base): or class ModelName(declarative_base()):
+                class_pattern = r"class\s+(\w+)\s*\((.*?(Base|DeclarativeMeta|declarative_base).*?)\):"
+                for match in re.finditer(class_pattern, source):
+                    model_name = match.group(1)
+
+                    # Extract table name if specified
+                    table_match = re.search(
+                        rf"class\s+{model_name}.*?\n\s+__tablename__\s*=\s*['\"](\w+)['\"]",
+                        source,
+                    )
+                    table_name = table_match.group(1) if table_match else None
+
+                    # Extract columns
+                    columns = re.findall(r"(\w+)\s*=\s*Column\(", source)
+
+                    # Extract relationships
+                    relationships = re.findall(r"(\w+)\s*=\s*relationship\(", source)
+
+                    models.append(
+                        {
+                            "name": model_name,
+                            "file": str(py_file.relative_to(root)),
+                            "table": table_name,
+                            "columns": columns,
+                            "relationships": relationships,
+                        }
+                    )
+
+            except (OSError, UnicodeDecodeError):
+                continue
+
+        return models
+
+    def _validate_relationships(
+        self, root: Path, models: List[Dict[str, Any]]
+    ) -> List[Tuple[str, str, str, str]]:
+        """Validate model relationships."""
+        issues = []
+        model_names = {m["name"] for m in models}
+
+        for model in models:
+            # Check if relationships reference existing models
+            for rel in model.get("relationships", []):
+                # Very basic check - relationship should reference a model
+                if not any(model_name in rel for model_name in model_names):
+                    # This is a heuristic - might be false positive
+                    pass
+
+            # Check for missing primary key
+            if not any(col.lower() == "id" for col in model.get("columns", [])):
+                issues.append(
+                    (
+                        model["name"],
+                        "WARNING",
+                        "No obvious primary key found",
+                        "Ensure model has a primary key defined",
+                    )
+                )
+
+        return issues
+
+    def _analyze_relationships(self, models: List[Dict[str, Any]]) -> List[str]:
+        """Analyze relationships between models."""
+        relationships = []
+
+        for model in models:
+            for rel in model.get("relationships", []):
+                relationships.append(f"  {model['name']}.{rel}")
+
+        return relationships

pybundle/steps/test_flakiness.py

@@ -0,0 +1,184 @@
+"""
+Test flakiness detection - Milestone 4 (v1.4.1)
+"""
+
+from __future__ import annotations
+
+import subprocess
+import time
+from dataclasses import dataclass
+
+from .base import StepResult
+from ..context import BundleContext
+
+
+@dataclass
+class TestFlakinessStep:
+    """
+    Run tests multiple times to detect non-deterministic failures (flaky tests).
+
+    Outputs:
+    - logs/70_test_flakiness.txt: Report of flaky tests with pass/fail patterns
+    """
+
+    name: str = "test_flakiness"
+
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+
+        if not ctx.tools.pytest:
+            return StepResult(self.name, "SKIP", 0, "pytest not found")
+
+        tests_dir = ctx.root / "tests"
+        if not tests_dir.is_dir():
+            return StepResult(self.name, "SKIP", 0, "no tests/ directory")
+
+        runs = ctx.options.test_flakiness_runs
+        ctx.emit(f"  Running tests {runs}x to detect flakiness...")
+
+        output_file = ctx.workdir / "logs" / "70_test_flakiness.txt"
+        output_file.parent.mkdir(parents=True, exist_ok=True)
+
+        try:
+            # Run tests multiple times and collect results
+            results = []
+            test_outcomes: dict[str, list[str]] = {}  # test_name -> [pass/fail/error]
+
+            for i in range(runs):
+                ctx.emit(f"    Run {i + 1}/{runs}...")
+                result = subprocess.run(
+                    [str(ctx.tools.pytest), "-v", "--tb=no"],
+                    cwd=ctx.root,
+                    capture_output=True,
+                    text=True,
+                    timeout=180,  # 3 minute timeout per run
+                )
+                results.append(result)
+
+                # Parse test results
+                self._parse_test_outcomes(result.stdout, test_outcomes, i)
+
+            # Analyze for flakiness
+            flaky_tests = self._identify_flaky_tests(test_outcomes)
+
+            # Generate report
+            with output_file.open("w") as f:
+                f.write("=" * 70 + "\n")
+                f.write(f"TEST FLAKINESS DETECTION ({runs} runs)\n")
+                f.write("=" * 70 + "\n\n")
+
+                if not test_outcomes:
+                    f.write("No test results collected.\n\n")
+                    for i, result in enumerate(results):
+                        f.write(f"Run {i + 1} output:\n")
+                        f.write(result.stdout[:500])
+                        f.write("\n\n")
+                else:
+                    total_tests = len(test_outcomes)
+                    f.write(f"Total tests analyzed: {total_tests}\n")
+                    f.write(f"Flaky tests detected: {len(flaky_tests)}\n\n")
+
+                    if flaky_tests:
+                        f.write("=" * 70 + "\n")
+                        f.write("FLAKY TESTS (non-deterministic results):\n")
+                        f.write("=" * 70 + "\n\n")
+
+                        for test_name, outcomes in flaky_tests.items():
+                            pattern = " -> ".join(outcomes)
+                            f.write(f"⚠️  {test_name}\n")
+                            f.write(f"    Pattern: {pattern}\n\n")
+                    else:
+                        f.write(
+                            "✅ No flaky tests detected - all tests deterministic!\n\n"
+                        )
+
+                    # Summary of all tests
+                    f.write("=" * 70 + "\n")
+                    f.write("ALL TESTS SUMMARY:\n")
+                    f.write("=" * 70 + "\n\n")
+
+                    stable_pass = []
+                    stable_fail = []
+                    flaky = []
+
+                    for test_name, outcomes in test_outcomes.items():
+                        unique_outcomes = set(outcomes)
+                        if len(unique_outcomes) == 1:
+                            if "PASSED" in unique_outcomes:
+                                stable_pass.append(test_name)
+                            else:
+                                stable_fail.append(test_name)
+                        else:
+                            flaky.append(test_name)
+
+                    f.write(f"Stable passing: {len(stable_pass)}\n")
+                    f.write(f"Stable failing: {len(stable_fail)}\n")
+                    f.write(f"Flaky: {len(flaky)}\n\n")
+
+                    if stable_fail:
+                        f.write("Consistently failing tests:\n")
+                        for test in stable_fail[:20]:  # Limit to 20
+                            f.write(f"  - {test}\n")
+                        if len(stable_fail) > 20:
+                            f.write(f"  ... and {len(stable_fail) - 20} more\n")
+                        f.write("\n")
+
+                    f.write("=" * 70 + "\n")
+                    f.write("RECOMMENDATIONS:\n")
+                    f.write(
+                        "- Fix flaky tests by removing non-deterministic behavior\n"
+                    )
+                    f.write(
+                        "- Common causes: timing issues, random data, external dependencies\n"
+                    )
+                    f.write("- Use pytest-randomly to test with different orderings\n")
+
+            elapsed = int((time.time() - start) * 1000)
+
+            if flaky_tests:
+                return StepResult(
+                    self.name, "OK", elapsed, f"{len(flaky_tests)} flaky tests"
+                )
+            else:
+                return StepResult(self.name, "OK", elapsed)
+
+        except subprocess.TimeoutExpired:
+            elapsed = int((time.time() - start) * 1000)
+            return StepResult(self.name, "FAIL", elapsed, "timeout")
+        except Exception as e:
+            elapsed = int((time.time() - start) * 1000)
+            return StepResult(self.name, "FAIL", elapsed, str(e))
+
+    def _parse_test_outcomes(
+        self, output: str, test_outcomes: dict, run_num: int
+    ) -> None:
+        """Parse pytest -v output to extract test results"""
+        for line in output.splitlines():
+            # Look for pytest verbose output: "test_file.py::test_name PASSED"
+            if "::" in line and any(
+                status in line for status in ["PASSED", "FAILED", "ERROR", "SKIPPED"]
+            ):
+                parts = line.split()
+                if len(parts) >= 2:
+                    test_name = parts[0]
+                    # Find status
+                    status = None
+                    for s in ["PASSED", "FAILED", "ERROR", "SKIPPED"]:
+                        if s in line:
+                            status = s
+                            break
+
+                    if status:
+                        if test_name not in test_outcomes:
+                            test_outcomes[test_name] = []
+                        test_outcomes[test_name].append(status)
+
+    def _identify_flaky_tests(self, test_outcomes: dict) -> dict:
+        """Identify tests with inconsistent results across runs"""
+        flaky = {}
+        for test_name, outcomes in test_outcomes.items():
+            unique_outcomes = set(outcomes)
+            # Flaky if not all the same outcome
+            if len(unique_outcomes) > 1:
+                flaky[test_name] = outcomes
+        return flaky

pybundle/steps/tree.py

@@ -0,0 +1,116 @@
+from __future__ import annotations
+
+import os
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from .base import StepResult
+from pybundle.context import BundleContext
+from pybundle.policy import AIContextPolicy, PathFilter
+
+
+@dataclass
+class TreeStep:
+    name: str = "tree (filtered)"
+    max_depth: int = 4
+    excludes: list[str] | None = None
+    policy: AIContextPolicy | None = None
+
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+        policy = self.policy or AIContextPolicy()
+
+        # allow overrides
+        exclude_dirs = set(self.excludes) if self.excludes else set(policy.exclude_dirs)
+        filt = PathFilter(
+            exclude_dirs=exclude_dirs,
+            exclude_patterns=set(policy.exclude_patterns),
+            exclude_file_exts=set(policy.exclude_file_exts),
+        )
+
+        out = ctx.metadir / "10_tree.txt"
+        out.parent.mkdir(parents=True, exist_ok=True)
+
+        root = ctx.root
+        lines: list[str] = []
+
+        for dirpath, dirnames, filenames in os.walk(root):
+            dp = Path(dirpath)
+            rel_dp = dp.relative_to(root)
+            depth = 0 if rel_dp == Path(".") else len(rel_dp.parts)
+
+            if depth > self.max_depth:
+                dirnames[:] = []
+                continue
+
+            # prune dirs (name + venv-structure)
+            kept = []
+            for d in dirnames:
+                if filt.should_prune_dir(dp, d):
+                    continue
+                kept.append(d)
+            dirnames[:] = kept
+
+            for fn in filenames:
+                p = dp / fn
+                if not filt.should_include_file(root, p):
+                    continue
+                lines.append(str(p.relative_to(root)))
+
+        lines.sort()
+        out.write_text("\n".join(lines) + ("\n" if lines else ""), encoding="utf-8")
+        dur = int(time.time() - start)
+        return StepResult(self.name, "PASS", dur, "python-walk")
+
+
+@dataclass
+class LargestFilesStep:
+    name: str = "largest files"
+    limit: int = 80
+    excludes: list[str] | None = None
+    policy: AIContextPolicy | None = None
+
+    def run(self, ctx: BundleContext) -> StepResult:
+        start = time.time()
+        policy = self.policy or AIContextPolicy()
+
+        exclude_dirs = set(self.excludes) if self.excludes else set(policy.exclude_dirs)
+        filt = PathFilter(
+            exclude_dirs=exclude_dirs,
+            exclude_patterns=set(policy.exclude_patterns),
+            exclude_file_exts=set(policy.exclude_file_exts),
+        )
+
+        out = ctx.metadir / "11_largest_files.txt"
+        out.parent.mkdir(parents=True, exist_ok=True)
+
+        files: list[tuple[int, str]] = []
+        root = ctx.root
+
+        for dirpath, dirnames, filenames in os.walk(root):
+            dp = Path(dirpath)
+
+            kept = []
+            for d in dirnames:
+                if filt.should_prune_dir(dp, d):
+                    continue
+                kept.append(d)
+            dirnames[:] = kept
+
+            for fn in filenames:
+                p = dp / fn
+                if not filt.should_include_file(root, p):
+                    continue
+                try:
+                    size = p.stat().st_size
+                except OSError:
+                    continue
+                files.append((size, str(p.relative_to(root))))
+
+        files.sort(key=lambda x: x[0], reverse=True)
+        lines = [f"{size}\t{path}" for size, path in files[: self.limit]]
+        out.write_text("\n".join(lines) + ("\n" if lines else ""), encoding="utf-8")
+
+        dur = int(time.time() - start)
+        return StepResult(self.name, "PASS", dur, f"count={len(files)}")