buildlog 0.5.0__py3-none-any.whl → 0.6.1__py3-none-any.whl

buildlog/data/seeds/test_terrorist.yaml

@@ -0,0 +1,280 @@
+persona: test_terrorist
+version: 1
+rules:
+- rule: Tests must not depend on execution order
+  category: isolation
+  context: Test suites with multiple tests, shared fixtures, database state
+  antipattern: Test A creates data that Test B asserts on; tests fail when run individually
+  rationale: Order-dependent tests are flaky and hide real failures. Each test must be hermetic.
+  tags:
+  - isolation
+  - order-independent
+  - hermetic
+  - test_terrorist
+  references:
+  - url: https://testing.googleblog.com/2010/12/test-sizes.html
+    title: Google Testing Blog - Test Sizes
+- rule: Tests must clean up after themselves
+  category: isolation
+  context: Tests using databases, files, external services, global state
+  antipattern: Tests leaving data in shared resources; no teardown; assuming clean state
+  rationale: Test pollution causes cascading failures and makes debugging impossible.
+  tags:
+  - isolation
+  - test_terrorist
+  - cleanup
+  - teardown
+  references:
+  - url: https://testing.googleblog.com/2010/12/test-sizes.html
+    title: Google Testing Blog - Test Sizes
+- rule: Tests should run in under 10 seconds for fast feedback
+  category: anti-patterns
+  context: Unit test suites, developer workflows, pre-commit hooks
+  antipattern: Minute-long test suites; 'just run CI'; tests that require coffee breaks
+  rationale: Slow tests don't get run. Fast feedback enables TDD and catches bugs early.
+  tags:
+  - feedback
+  - anti-patterns
+  - slow-tests
+  - test_terrorist
+  references:
+  - url: https://testing.googleblog.com/2010/12/test-sizes.html
+    title: Google Testing Blog - Test Sizes
+- rule: Every public API must have at least one happy path test
+  category: coverage
+  context: New endpoints, public functions, exported modules
+  antipattern: Shipping code with no tests; 'I'll add tests later'; PRs without test changes
+  rationale: Untested code is legacy code the moment it's merged. Tests are executable documentation.
+  tags:
+  - happy-path
+  - public-api
+  - test_terrorist
+  - coverage
+  references:
+  - url: https://testing.googleblog.com/2015/04/just-say-no-to-more-end-to-end-tests.html
+    title: Google Testing Blog - Just Say No to More E2E Tests
+- rule: New bug fixes must include a regression test
+  category: coverage
+  context: Any bug fix PR or commit
+  antipattern: Fixing bugs without adding tests that would have caught them
+  rationale: A bug that escapes once will escape again. Regression tests prevent recurrence.
+  tags:
+  - regression
+  - bug-fix
+  - test_terrorist
+  - coverage
+  references:
+  - url: https://testing.googleblog.com/2015/04/just-say-no-to-more-end-to-end-tests.html
+    title: Google Testing Blog - Just Say No to More E2E Tests
+- rule: Critical paths require edge case and error path coverage
+  category: coverage
+  context: Payment flows, authentication, data mutations, external integrations
+  antipattern: Only happy path tests for critical code; no error handling tests
+  rationale: Edge cases in critical paths cause production incidents. Murphy's law applies.
+  tags:
+  - test_terrorist
+  - critical-path
+  - edge-cases
+  - coverage
+  references:
+  - url: https://martinfowler.com/articles/practical-test-pyramid.html
+    title: Martin Fowler - Testing Pyramid
+- rule: Assert on behavior, not implementation details
+  category: assertions
+  context: Unit tests, refactoring scenarios
+  antipattern: Asserting on private method calls; testing internal state; mock call counts
+  rationale: Implementation-coupled tests break on refactoring. Test the contract, not the code.
+  tags:
+  - assertions
+  - behavior
+  - test_terrorist
+  - contract
+  references:
+  - url: https://martinfowler.com/articles/practical-test-pyramid.html
+    title: Martin Fowler - Testing Pyramid
+- rule: Mock external dependencies at test boundaries
+  category: isolation
+  context: Tests calling APIs, databases, file systems, network services
+  antipattern: Real network calls in unit tests; tests that require running services
+  rationale: External dependencies make tests slow, flaky, and expensive. Mock at boundaries.
+  tags:
+  - isolation
+  - test_terrorist
+  - mocking
+  - boundaries
+  references:
+  - url: https://martinfowler.com/bliki/TestDouble.html
+    title: Martin Fowler - Test Double
+- rule: Use property-based testing for functions with clear invariants
+  category: property-testing
+  context: Serializers, parsers, encoders/decoders, sorting, mathematical operations
+  antipattern: Only example-based tests for encode/decode pairs; hand-picked edge cases
+  rationale: Property tests generate thousands of examples, finding edge cases humans miss.
+  tags:
+  - invariants
+  - hypothesis
+  - test_terrorist
+  - property
+  references:
+  - url: https://hypothesis.readthedocs.io/en/latest/
+    title: Hypothesis Documentation
+- rule: Define roundtrip properties for serialization code
+  category: property-testing
+  context: JSON, protobuf, custom serializers, data transformation pipelines
+  antipattern: Testing serialize and deserialize separately with fixed examples
+  rationale: decode(encode(x)) == x is a universal property. Hypothesis finds corner cases.
+  tags:
+  - roundtrip
+  - test_terrorist
+  - property
+  - serialization
+  references:
+  - url: https://hypothesis.readthedocs.io/en/latest/
+    title: Hypothesis Documentation
+- rule: Apply metamorphic relations when test oracles are unavailable
+  category: metamorphic-testing
+  context: ML models, search engines, optimization algorithms, complex computations
+  antipattern: No testing because 'we don't know the right answer'; only manual inspection
+  rationale: Metamorphic testing validates input-output relationships without ground truth.
+  tags:
+  - metamorphic
+  - test_terrorist
+  - ml
+  - oracle-free
+  references:
+  - url: https://www.sciencedirect.com/science/article/pii/S0950584918300016
+    title: Metamorphic Testing - Chen et al. Survey
+- rule: Define permutation invariance for order-independent operations
+  category: metamorphic-testing
+  context: Aggregations, set operations, commutative functions
+  antipattern: Testing with single fixed input order; assuming order doesn't matter
+  rationale: sum([1,2,3]) == sum([3,1,2]) is a metamorphic relation that catches bugs.
+  tags:
+  - metamorphic
+  - test_terrorist
+  - permutation
+  - invariance
+  references:
+  - url: https://www.sciencedirect.com/science/article/pii/S0950584918300016
+    title: Metamorphic Testing - Chen et al. Survey
+- rule: Validate data distributions at pipeline boundaries
+  category: statistical-testing
+  context: ETL pipelines, ML feature stores, data ingestion, API responses
+  antipattern: Assuming input data matches expected distribution; no schema validation
+  rationale: Distribution drift breaks models silently. Validate expectations at boundaries.
+  tags:
+  - statistical
+  - drift
+  - test_terrorist
+  - distribution
+  references:
+  - url: https://docs.greatexpectations.io/docs/
+    title: Great Expectations Documentation
+- rule: Define and enforce data contracts with schemas
+  category: statistical-testing
+  context: Data pipelines, API integrations, database migrations
+  antipattern: Implicit schemas; duck typing for data; hoping fields exist
+  rationale: Schema validation catches contract violations before they cause failures.
+  tags:
+  - statistical
+  - contracts
+  - test_terrorist
+  - schema
+  references:
+  - url: https://pandera.readthedocs.io/en/stable/
+    title: Pandera Documentation
+- rule: LLM outputs require structured validation beyond string matching
+  category: llm-testing
+  context: Any code using LLM-generated content in production paths
+  antipattern: No validation; trusting raw LLM output; regex-only validation
+  rationale: '[GAP] Standard test frameworks don''t cover LLM eval. See Guardrails/DeepEval for emerging
+    patterns. This is a known gap requiring specialized tooling.'
+  tags:
+  - test_terrorist
+  - emerging
+  - llm-testing
+  - validation
+  - gap
+  references:
+  - url: https://www.guardrailsai.com/docs/concepts/guard
+    title: Guardrails AI Documentation
+- rule: LLM-based features need evaluation datasets and metrics
+  category: llm-testing
+  context: RAG systems, chatbots, content generation, code assistants
+  antipattern: Vibes-based testing; manual spot checks; no regression tracking
+  rationale: '[GAP] LLM behavior is non-deterministic. Evaluation datasets with metrics enable regression
+    detection. Tooling is immature.'
+  tags:
+  - emerging
+  - llm-testing
+  - evaluation
+  - gap
+  - test_terrorist
+  references:
+  - url: https://docs.confident-ai.com/docs/getting-started
+    title: DeepEval Documentation
+- rule: Every test must have at least one meaningful assertion
+  category: assertions
+  context: All test functions
+  antipattern: Tests that only call code without asserting; assert True; empty test bodies
+  rationale: A test without assertions is not a test. It's a false sense of security.
+  tags:
+  - assertions
+  - no-pass-through
+  - meaningful
+  - test_terrorist
+  references:
+  - url: http://xunitpatterns.com/Test%20Smells.html
+    title: xUnit Test Patterns - Test Smells
+- rule: Follow Arrange-Act-Assert (AAA) pattern
+  category: structure
+  context: All test functions
+  antipattern: Interleaved setup and assertions; multiple acts per test; unclear test phases
+  rationale: AAA makes tests readable and debuggable. One logical assertion per test.
+  tags:
+  - structure
+  - aaa
+  - test_terrorist
+  - readability
+  references:
+  - url: http://xunitpatterns.com/Test%20Smells.html
+    title: xUnit Test Patterns - Test Smells
+- rule: Avoid testing implementation details that change frequently
+  category: anti-patterns
+  context: Refactoring scenarios, internal APIs, private methods
+  antipattern: Tests break on every refactor; testing private method behavior
+  rationale: Fragile tests slow development. Test stable interfaces, not implementation.
+  tags:
+  - fragile
+  - implementation-details
+  - anti-patterns
+  - test_terrorist
+  references:
+  - url: http://xunitpatterns.com/Test%20Smells.html
+    title: xUnit Test Patterns - Test Smells
+- rule: Flaky tests must be fixed or quarantined immediately
+  category: anti-patterns
+  context: CI pipelines, test suites with intermittent failures
+  antipattern: Rerunning CI until green; ignoring flaky tests; 'it works on my machine'
+  rationale: Flaky tests erode trust in the test suite. A flaky test is worse than no test.
+  tags:
+  - anti-patterns
+  - flaky
+  - ci
+  - test_terrorist
+  references:
+  - url: https://testing.googleblog.com/2016/05/flaky-tests-at-google-and-how-we.html
+    title: Google Testing Blog - Test Flakiness
+- rule: Test names should describe the scenario and expected outcome
+  category: structure
+  context: Test function naming
+  antipattern: test_1, test_function, test_it_works; names that don't explain the test
+  rationale: Test names are documentation. A failing test name should tell you what broke.
+  tags:
+  - structure
+  - documentation
+  - test_terrorist
+  - naming
+  references:
+  - url: https://docs.pytest.org/en/stable/explanation/goodpractices.html
+    title: pytest - Good Integration Practices

buildlog/seed_engine/__init__.py

@@ -0,0 +1,74 @@
+"""Seed Engine - Formalized pipeline for creating reviewer personas.
+
+The seed engine abstracts the 4-step process for bootstrapping
+defensible reviewer personas from authoritative domain sources:
+
+    1. SOURCE IDENTIFICATION - Define authoritative sources
+    2. RULE EXTRACTION - Extract candidate rules with defensibility fields
+    3. CATEGORIZATION - Map rules to persona concern categories
+    4. SEED GENERATION - Output validated YAML seed file
+
+Usage:
+    from buildlog.seed_engine import Pipeline, Source, SourceType
+
+    # Define sources
+    sources = [
+        Source(
+            name="OWASP Top 10",
+            url="https://owasp.org/Top10/",
+            source_type=SourceType.REFERENCE_DOC,
+            domain="security",
+        )
+    ]
+
+    # Run pipeline
+    pipeline = Pipeline(persona="security_karen")
+    seed_file = pipeline.run(sources)
+"""
+
+from buildlog.seed_engine.categorizers import (
+    Categorizer,
+    CategoryMapping,
+    TagBasedCategorizer,
+)
+from buildlog.seed_engine.extractors import ManualExtractor, RuleExtractor
+from buildlog.seed_engine.generators import SeedGenerator
+from buildlog.seed_engine.models import (
+    CandidateRule,
+    CategorizedRule,
+    Source,
+    SourceType,
+)
+from buildlog.seed_engine.pipeline import Pipeline
+from buildlog.seed_engine.sources import (
+    FetchStatus,
+    SourceEntry,
+    SourceFetcher,
+    SourceManifest,
+    url_to_cache_filename,
+)
+
+__all__ = [
+    # Models
+    "Source",
+    "SourceType",
+    "CandidateRule",
+    "CategorizedRule",
+    # Pipeline
+    "Pipeline",
+    # Extractors
+    "RuleExtractor",
+    "ManualExtractor",
+    # Categorizers
+    "Categorizer",
+    "TagBasedCategorizer",
+    "CategoryMapping",
+    # Generators
+    "SeedGenerator",
+    # Sources
+    "FetchStatus",
+    "SourceEntry",
+    "SourceManifest",
+    "SourceFetcher",
+    "url_to_cache_filename",
+]

buildlog/seed_engine/categorizers.py

@@ -0,0 +1,145 @@
+"""Rule categorizers for Step 3 of the seed engine pipeline.
+
+Categorizers take candidate rules and assign final categories and tags.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import Callable
+from dataclasses import dataclass
+
+from buildlog.seed_engine.models import CandidateRule, CategorizedRule
+
+
+class Categorizer(ABC):
+    """Protocol for categorizing rules.
+
+    Implementations:
+    - TagBasedCategorizer: Category from tags/keywords
+    - MappingCategorizer: Explicit source→category mapping
+    """
+
+    @abstractmethod
+    def categorize(self, rule: CandidateRule) -> CategorizedRule:
+        """Assign category and final tags to a rule.
+
+        Args:
+            rule: The candidate rule to categorize.
+
+        Returns:
+            Categorized rule ready for seed generation.
+        """
+        ...
+
+
+@dataclass
+class CategoryMapping:
+    """Mapping from keywords/tags to category."""
+
+    category: str
+    keywords: list[str]  # If any of these appear in tags/rule, assign this category
+    priority: int = 0  # Higher priority wins on conflicts
+
+
+class TagBasedCategorizer(Categorizer):
+    """Categorize rules based on their tags and keywords.
+
+    Usage:
+        categorizer = TagBasedCategorizer(
+            default_category="testing",
+            mappings=[
+                CategoryMapping("coverage", ["coverage", "untested"]),
+                CategoryMapping("isolation", ["flaky", "order", "hermetic"]),
+                CategoryMapping("assertions", ["assert", "expect", "verify"]),
+            ],
+            tag_normalizer=lambda t: t.lower().replace("-", "_"),
+        )
+
+        categorized = categorizer.categorize(candidate_rule)
+    """
+
+    def __init__(
+        self,
+        default_category: str,
+        mappings: list[CategoryMapping] | None = None,
+        tag_normalizer: Callable[[str], str] | None = None,
+        additional_tags: list[str] | None = None,
+    ) -> None:
+        self.default_category = default_category
+        self.mappings = sorted(mappings or [], key=lambda m: m.priority, reverse=True)
+        self.tag_normalizer = tag_normalizer or (lambda t: t.lower())
+        self.additional_tags = additional_tags or []
+
+    def categorize(self, rule: CandidateRule) -> CategorizedRule:
+        """Assign category based on tag matching."""
+        # Normalize tags
+        normalized_tags = [self.tag_normalizer(t) for t in rule.raw_tags]
+
+        # Also check rule text for keywords
+        rule_text_lower = rule.rule.lower()
+
+        # Find matching category
+        category = self.default_category
+        for mapping in self.mappings:
+            for keyword in mapping.keywords:
+                keyword_lower = keyword.lower()
+                if keyword_lower in normalized_tags or keyword_lower in rule_text_lower:
+                    category = mapping.category
+                    break
+            else:
+                continue
+            break
+
+        # Build final tags
+        final_tags = list(set(normalized_tags + self.additional_tags))
+
+        return CategorizedRule.from_candidate(
+            candidate=rule,
+            category=category,
+            tags=final_tags,
+        )
+
+
+class MappingCategorizer(Categorizer):
+    """Categorize rules via explicit source→category mapping.
+
+    Useful when sources map directly to categories
+    (e.g., OWASP A03 → "injection").
+
+    Usage:
+        categorizer = MappingCategorizer(
+            source_category_map={
+                "https://owasp.org/Top10/A03": "injection",
+                "https://owasp.org/Top10/A01": "access-control",
+            },
+            default_category="security",
+        )
+    """
+
+    def __init__(
+        self,
+        source_category_map: dict[str, str],
+        default_category: str,
+        tag_transform: Callable[[list[str]], list[str]] | None = None,
+    ) -> None:
+        self.source_category_map = source_category_map
+        self.default_category = default_category
+        self.tag_transform = tag_transform or (lambda tags: tags)
+
+    def categorize(self, rule: CandidateRule) -> CategorizedRule:
+        """Assign category based on source URL."""
+        # Find category by matching source URL prefix
+        category = self.default_category
+        for url_prefix, cat in self.source_category_map.items():
+            if rule.source.url.startswith(url_prefix):
+                category = cat
+                break
+
+        final_tags = self.tag_transform(rule.raw_tags)
+
+        return CategorizedRule.from_candidate(
+            candidate=rule,
+            category=category,
+            tags=final_tags,
+        )

buildlog/seed_engine/extractors.py

@@ -0,0 +1,148 @@
+"""Rule extractors for Step 2 of the seed engine pipeline.
+
+Extractors take sources and produce candidate rules.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Callable
+
+from buildlog.seed_engine.models import CandidateRule, Source
+
+
+class RuleExtractor(ABC):
+    """Protocol for extracting rules from sources.
+
+    Implementations:
+    - ManualExtractor: Human-curated rules (highest quality)
+    - LLMExtractor: LLM-assisted extraction (future)
+    - StructuredExtractor: Parse structured docs like OWASP (future)
+    """
+
+    @abstractmethod
+    def extract(self, source: Source) -> list[CandidateRule]:
+        """Extract candidate rules from a source.
+
+        Args:
+            source: The source to extract rules from.
+
+        Returns:
+            List of candidate rules with defensibility fields.
+        """
+        ...
+
+    @abstractmethod
+    def validate(self, rule: CandidateRule) -> list[str]:
+        """Validate a candidate rule, returning any issues.
+
+        Args:
+            rule: The rule to validate.
+
+        Returns:
+            List of validation issues (empty if valid).
+        """
+        ...
+
+
+class ManualExtractor(RuleExtractor):
+    """Manual rule extraction via human curation.
+
+    This is the gold standard—humans read the source and
+    extract rules with full defensibility metadata.
+
+    Usage:
+        extractor = ManualExtractor()
+
+        # Register rules for a source
+        extractor.register(
+            source=google_testing_blog,
+            rules=[
+                CandidateRule(
+                    rule="Tests must not depend on execution order",
+                    context="Test suites with multiple tests",
+                    antipattern="Test A sets state that Test B relies on",
+                    rationale="Order-dependent tests are flaky",
+                    source=google_testing_blog,
+                    raw_tags=["isolation", "flaky"],
+                )
+            ]
+        )
+
+        # Extract returns registered rules
+        rules = extractor.extract(google_testing_blog)
+    """
+
+    def __init__(self) -> None:
+        self._rules_by_source: dict[str, list[CandidateRule]] = {}
+
+    def register(self, source: Source, rules: list[CandidateRule]) -> None:
+        """Register manually curated rules for a source.
+
+        Args:
+            source: The source these rules come from.
+            rules: The curated rules.
+        """
+        # Validate all rules are complete
+        for rule in rules:
+            issues = self.validate(rule)
+            if issues:
+                raise ValueError(
+                    f"Invalid rule '{rule.rule[:50]}...': {'; '.join(issues)}"
+                )
+        self._rules_by_source[source.url] = rules
+
+    def extract(self, source: Source) -> list[CandidateRule]:
+        """Return registered rules for this source."""
+        return self._rules_by_source.get(source.url, [])
+
+    def validate(self, rule: CandidateRule) -> list[str]:
+        """Validate defensibility fields are populated."""
+        issues = []
+        if not rule.rule.strip():
+            issues.append("Rule text is empty")
+        if not rule.context.strip():
+            issues.append("Context is required for defensibility")
+        if not rule.antipattern.strip():
+            issues.append("Antipattern is required for defensibility")
+        if not rule.rationale.strip():
+            issues.append("Rationale is required for defensibility")
+        return issues
+
+
+class FunctionExtractor(RuleExtractor):
+    """Extraction via custom function (for structured sources).
+
+    Allows plugging in custom extraction logic for sources
+    with known structure (e.g., OWASP pages, API docs).
+
+    Usage:
+        def extract_from_owasp(source: Source) -> list[CandidateRule]:
+            # Custom parsing logic for OWASP format
+            ...
+
+        extractor = FunctionExtractor(extract_from_owasp)
+        rules = extractor.extract(owasp_source)
+    """
+
+    def __init__(
+        self,
+        extract_fn: Callable[[Source], list[CandidateRule]],
+        validate_fn: Callable[[CandidateRule], list[str]] | None = None,
+    ) -> None:
+        self._extract_fn = extract_fn
+        self._validate_fn = validate_fn or self._default_validate
+
+    def extract(self, source: Source) -> list[CandidateRule]:
+        """Run the custom extraction function."""
+        return self._extract_fn(source)
+
+    def validate(self, rule: CandidateRule) -> list[str]:
+        """Run the validation function."""
+        return self._validate_fn(rule)
+
+    def _default_validate(self, rule: CandidateRule) -> list[str]:
+        """Default validation: check completeness."""
+        if not rule.is_complete():
+            return ["Rule is missing required defensibility fields"]
+        return []