source-kb 0.2.2__py3-none-any.whl

core/validators/consistency.py

@@ -0,0 +1,203 @@
+"""Consistency validator — reference path and code snippet checks.
+
+Validates that document references (file paths, code snippets) are consistent
+with actual source code. Migrated from engine/skeleton/validate_output.py.
+
+Usage:
+    from core.validators.consistency import ConsistencyValidator
+    result = ConsistencyValidator().validate(module_dir, source_cache=path)
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Any
+
+from core.interfaces import Validator, ValidationResult
+from core.validators import register_validator
+
+# Pattern to match file path references in docs (e.g., `src/main/java/...`)
+_PATH_REF_PATTERN = re.compile(
+    r"`((?:src|main|java|com|org|net)/[a-zA-Z0-9_/.-]+\.(?:java|kt|py|go|ts|js))`"
+)
+
+# Pattern to match markdown links to local files
+_LINK_PATTERN = re.compile(r"\[([^\]]*)\]\(([^)]+)\)")
+
+# Pattern to match code block language hints
+_CODE_BLOCK_PATTERN = re.compile(r"^```(\w+)?")
+
+
+@register_validator
+class ConsistencyValidator(Validator):
+    """Validate reference paths and code snippet consistency."""
+
+    name = "consistency"
+
+    def validate(self, module_dir: Path, **kwargs: Any) -> ValidationResult:
+        result = ValidationResult()
+        source_cache = kwargs.get("source_cache")
+        skeleton_entries = kwargs.get("skeleton_entries")
+
+        for md_file in sorted(module_dir.glob("*.md")):
+            if md_file.name.startswith("."):
+                continue
+            try:
+                content = md_file.read_text(encoding="utf-8")
+            except OSError:
+                continue
+
+            # 1. Check cross-document link validity
+            _check_cross_doc_links(md_file, module_dir, content, result)
+
+            # 2. Check source path references (if source_cache provided)
+            if source_cache:
+                _check_path_references(
+                    md_file.name, content, Path(source_cache), result
+                )
+
+            # 3. Check code snippet consistency (if skeleton available)
+            if skeleton_entries:
+                _check_code_snippets(
+                    md_file.name, content, skeleton_entries, result
+                )
+
+        return result
+
+
+# ---------------------------------------------------------------------------
+# Cross-document link checks
+# ---------------------------------------------------------------------------
+
+
+def _check_cross_doc_links(
+    md_file: Path, module_dir: Path, content: str, result: ValidationResult
+) -> None:
+    """Verify that local markdown links point to existing files."""
+    for match in _LINK_PATTERN.finditer(content):
+        target = match.group(2)
+        # Skip external links and anchors-only
+        if target.startswith("http") or target.startswith("#"):
+            continue
+        # Strip anchor part
+        file_part = target.split("#")[0]
+        if not file_part:
+            continue
+        # Only check .md links
+        if not file_part.endswith(".md"):
+            continue
+        link_path = module_dir / file_part
+        if not link_path.exists():
+            result.warnings.append(
+                f"consistency: {md_file.name}: broken link to '{file_part}'"
+            )
+
+
+# ---------------------------------------------------------------------------
+# Source path reference checks
+# ---------------------------------------------------------------------------
+
+
+def _check_path_references(
+    fname: str, content: str, source_cache: Path, result: ValidationResult
+) -> None:
+    """Verify that referenced source file paths exist in source cache."""
+    refs = _PATH_REF_PATTERN.findall(content)
+    if not refs:
+        return
+
+    # Sample up to 10 references to avoid excessive I/O
+    sample = refs[:10] if len(refs) > 10 else refs
+    missing_count = 0
+
+    for ref_path in sample:
+        # Try to find the file in source_cache (search subdirectories)
+        found = _find_in_source_cache(ref_path, source_cache)
+        if not found:
+            missing_count += 1
+
+    if missing_count > 0:
+        result.warnings.append(
+            f"consistency: {fname}: {missing_count}/{len(sample)} "
+            f"referenced source paths not found in source cache"
+        )
+
+
+def _find_in_source_cache(ref_path: str, source_cache: Path) -> bool:
+    """Check if a referenced path exists somewhere under source_cache."""
+    # Direct check
+    if (source_cache / ref_path).exists():
+        return True
+    # Check under any subdirectory (module repos)
+    for subdir in source_cache.iterdir():
+        if subdir.is_dir() and (subdir / ref_path).exists():
+            return True
+    return False
+
+
+# ---------------------------------------------------------------------------
+# Code snippet consistency checks
+# ---------------------------------------------------------------------------
+
+
+def _check_code_snippets(
+    fname: str, content: str, skeleton_entries: list[dict], result: ValidationResult
+) -> None:
+    """Verify code snippets reference real methods/classes from skeleton."""
+    # Extract method names mentioned in code blocks
+    code_methods = _extract_code_block_identifiers(content)
+    if not code_methods:
+        return
+
+    # Build set of known method/class names from skeleton
+    known_names = _build_known_names(skeleton_entries)
+    if not known_names:
+        return
+
+    # Check that at least some code block identifiers match skeleton
+    matched = sum(1 for m in code_methods if m in known_names)
+    total = len(code_methods)
+
+    if total >= 5 and matched == 0:
+        result.warnings.append(
+            f"consistency: {fname}: code snippets contain {total} identifiers "
+            f"but none match skeleton data"
+        )
+
+
+def _extract_code_block_identifiers(content: str) -> list[str]:
+    """Extract method/class-like identifiers from code blocks."""
+    identifiers: list[str] = []
+    in_code = False
+    for line in content.splitlines():
+        stripped = line.strip()
+        if _CODE_BLOCK_PATTERN.match(stripped):
+            in_code = not in_code
+            continue
+        if in_code:
+            # Look for method declarations or class names
+            m = re.search(
+                r"\b(?:public|private|protected)?\s*(?:static\s+)?(?:\w+\s+)?(\w+)\s*\(",
+                stripped,
+            )
+            if m:
+                name = m.group(1)
+                if name and len(name) > 2 and name[0].islower():
+                    identifiers.append(name)
+    return identifiers
+
+
+def _build_known_names(entries: list[dict]) -> set[str]:
+    """Build set of known method and class names from skeleton."""
+    names: set[str] = set()
+    for entry in entries:
+        for cls in entry.get("classes", []):
+            name = cls.get("name", "")
+            if name:
+                names.add(name)
+        for method in entry.get("methods", []):
+            name = method.get("name", "")
+            if name:
+                names.add(name)
+    return names

core/validators/coverage.py

@@ -0,0 +1,171 @@
+"""Coverage validator — file and method coverage calculation.
+
+Validates that generated documents cover the expected source files and methods
+based on skeleton data. Migrated from engine/skeleton/validate_output.py.
+
+Usage:
+    from core.validators.coverage import CoverageValidator
+    result = CoverageValidator().validate(module_dir, preset=preset, skeleton_entries=entries)
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from pathlib import Path
+from typing import Any
+
+from core.interfaces import Validator, ValidationResult
+from core.validators import register_validator
+
+logger = logging.getLogger(__name__)
+
+
+def get_size_limits(preset: dict | None = None) -> dict[str, dict[str, int]]:
+    """Get document size expectations from preset config.
+
+    Reads from each doc_type's `size_expectations` field if available,
+    then falls back to preset-level `size_expectations`.
+    Returns empty dict if no preset provided.
+    """
+    result: dict[str, dict[str, int]] = {}
+
+    if not preset:
+        return result
+
+    # Layer 1: preset-level size_expectations (backward compat)
+    if "size_expectations" in preset:
+        for filename, limits in preset["size_expectations"].items():
+            if isinstance(limits, dict):
+                result[filename] = {**result.get(filename, {}), **limits}
+
+    # Layer 2: per-doc-type size_expectations (preferred, more granular)
+    doc_types = preset.get("doc_types", {})
+    for dt_key, dt_config in doc_types.items():
+        if not isinstance(dt_config, dict):
+            continue
+        if "size_expectations" in dt_config and "filename" in dt_config:
+            filename = dt_config["filename"]
+            limits = dt_config["size_expectations"]
+            if isinstance(limits, dict):
+                result[filename] = {**result.get(filename, {}), **limits}
+
+    return result
+
+
+@register_validator
+class CoverageValidator(Validator):
+    """Validate file completeness and size coverage."""
+
+    name = "coverage"
+
+    def validate(self, module_dir: Path, **kwargs: Any) -> ValidationResult:
+        result = ValidationResult()
+        preset = kwargs.get("preset", {})
+        module_type = kwargs.get("module_type", "service")
+        size_limits = get_size_limits(preset)
+
+        skip_list = _get_module_type_skip(preset, module_type)
+        if "*" in skip_list:
+            return result
+
+        existing_files = {f.name for f in module_dir.glob("*.md")}
+
+        # 1. File completeness check
+        required = _get_required_files(preset)
+        _check_file_completeness(existing_files, skip_list, required, result)
+
+        # 2. Size coverage check per file
+        for md_file in sorted(module_dir.glob("*.md")):
+            if md_file.name.startswith("."):
+                continue
+            _check_file_size(md_file, size_limits, result)
+
+        # 3. Preset-configured validators (skeleton_count_match, sampling, etc.)
+        skeleton_entries = kwargs.get("skeleton_entries")
+        if skeleton_entries:
+            from core.validators.engine import run_doc_validators
+            doc_types = preset.get("doc_types", {})
+            for dt_key, dt_config in doc_types.items():
+                if not isinstance(dt_config, dict):
+                    continue
+                filename = dt_config.get("filename", f"{dt_key}.md")
+                doc_path = module_dir / filename
+                if doc_path.exists() and dt_config.get("validators"):
+                    engine_result = run_doc_validators(
+                        doc_path, dt_key, preset, skeleton_entries
+                    )
+                    result = result.merge(engine_result)
+
+        return result
+
+
+# ---------------------------------------------------------------------------
+# Validation helpers
+# ---------------------------------------------------------------------------
+
+
+def _check_file_completeness(
+    existing: set[str], skip_list: list[str], required: list[str], result: ValidationResult
+) -> None:
+    """Check that all required files exist."""
+    for fname in required:
+        if fname in skip_list:
+            continue
+        if fname not in existing:
+            result.errors.append(f"coverage: missing required file: {fname}")
+
+
+def _get_required_files(preset: dict) -> list[str]:
+    """Get required files from preset doc_types (conditional=false entries)."""
+    doc_types = preset.get("doc_types", {})
+    if not doc_types:
+        return []
+    required = []
+    for dt_key, dt_config in doc_types.items():
+        if isinstance(dt_config, dict) and not dt_config.get("conditional", True):
+            filename = dt_config.get("filename", f"{dt_key}.md")
+            required.append(filename)
+    return required
+
+
+def _get_module_type_skip(preset: dict, module_type: str) -> list[str]:
+    """Get skip list from preset module_types config."""
+    module_types = preset.get("module_types", {})
+    if module_types and module_type in module_types:
+        return list(module_types[module_type].get("skip", []))
+    return []
+
+
+def _check_file_size(
+    md_file: Path, size_limits: dict[str, dict[str, int]], result: ValidationResult
+) -> None:
+    """Check file line count against size limits."""
+    fname = md_file.name
+    limits = size_limits.get(fname)
+    if not limits:
+        return
+
+    try:
+        lines = len(md_file.read_text(encoding="utf-8").splitlines())
+    except OSError:
+        return
+
+    min_lines = limits.get("min", 0)
+    hard_max = limits.get("hard_max", 999999)
+    warn_max = limits.get("warn_max", hard_max)
+
+    if min_lines and lines < min_lines:
+        result.errors.append(
+            f"coverage: {fname}: too short ({lines} lines < {min_lines} minimum)"
+        )
+    elif lines > hard_max:
+        result.warnings.append(
+            f"coverage: {fname}: too large ({lines} lines > {hard_max}), consider splitting"
+        )
+    elif lines > warn_max:
+        result.warnings.append(
+            f"coverage: {fname}: approaching size limit ({lines} lines)"
+        )
+

core/validators/duplicates.py

@@ -0,0 +1,76 @@
+"""Duplicate content detector — rule-based cross-doc redundancy check.
+
+Detects content that belongs in one document but appears in another,
+using ownership keywords defined in preset doc_types config.
+
+Usage:
+    from core.validators.duplicates import DuplicatesValidator
+    result = DuplicatesValidator().validate(module_dir, preset=preset)
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Any
+
+from core.interfaces import Validator, ValidationResult
+from core.validators import register_validator
+
+# Minimum doc size to trigger duplicate check (KB)
+MIN_SIZE_KB = 50
+
+
+@register_validator
+class DuplicatesValidator(Validator):
+    """Detect cross-document content duplication using preset ownership keywords."""
+
+    name = "duplicates"
+
+    def validate(self, module_dir: Path, **kwargs: Any) -> ValidationResult:
+        result = ValidationResult()
+
+        # Get ownership keywords from preset config
+        preset = kwargs.get("preset", {})
+        ownership = _get_ownership_signals(preset)
+        if not ownership:
+            return result
+
+        # Check business-logic.md (most common source of duplication)
+        # and any other large docs against ownership rules
+        for md_file in sorted(module_dir.glob("*.md")):
+            if md_file.name.startswith("."):
+                continue
+            if md_file.stat().st_size < MIN_SIZE_KB * 1024:
+                continue
+
+            content = md_file.read_text(encoding="utf-8")
+
+            for owner_doc, keywords in ownership.items():
+                if md_file.name == owner_doc:
+                    continue  # Don't check a doc against itself
+
+                for keyword in keywords:
+                    matches = re.findall(re.escape(keyword), content, re.IGNORECASE)
+                    if len(matches) > 3:
+                        result.warnings.append(
+                            f"duplicates: {md_file.name} contains {len(matches)} instances of "
+                            f"'{keyword}' (belongs in {owner_doc})"
+                        )
+
+        return result
+
+
+def _get_ownership_signals(preset: dict) -> dict[str, list[str]]:
+    """Extract ownership keywords from preset doc_types config.
+
+    Returns {doc_filename: [keyword1, keyword2, ...]}
+    """
+    if not preset:
+        return {}
+
+    try:
+        from core.preset import get_ownership_keywords
+        return get_ownership_keywords(preset)
+    except (ImportError, Exception):
+        return {}

core/validators/engine.py

@@ -0,0 +1,224 @@
+"""Validator engine — configurable, preset-driven document validation.
+
+Dispatches validators declared in doc_types.yaml. Each doc_type can declare
+a list of validators with type + params. The engine looks up the handler
+from a registry and runs it.
+
+Built-in validator types:
+    - skeleton_count_match: Compare skeleton annotation count vs doc heading count
+    - sampling_check: Spot-check values from skeleton appear in doc
+
+Usage:
+    from core.validators.engine import run_doc_validators
+    results = run_doc_validators(doc_path, doc_type, preset, skeleton_entries)
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import random
+import re
+from pathlib import Path
+from typing import Any, Callable
+
+from core.interfaces import ValidationResult
+
+logger = logging.getLogger(__name__)
+
+ValidatorHandler = Callable[..., ValidationResult]
+
+_HANDLER_REGISTRY: dict[str, ValidatorHandler] = {}
+
+
+def register_handler(name: str):
+    """Decorator to register a validator handler function."""
+    def decorator(fn: ValidatorHandler) -> ValidatorHandler:
+        _HANDLER_REGISTRY[name] = fn
+        return fn
+    return decorator
+
+
+def run_doc_validators(
+    doc_path: Path,
+    doc_type: str,
+    preset: dict,
+    skeleton_entries: list[dict] | None = None,
+) -> ValidationResult:
+    """Run all validators configured for this doc_type in preset.
+
+    Reads validators from preset doc_types[doc_type].validators list.
+    Each entry has {type, params}. Dispatches to registered handlers.
+    """
+    combined = ValidationResult()
+
+    from core.preset import get_doc_type_config
+    dt_config = get_doc_type_config(preset, doc_type)
+    validators = dt_config.get("validators", [])
+
+    if not validators:
+        return combined
+
+    for v_spec in validators:
+        v_type = v_spec.get("type", "")
+        params = v_spec.get("params", {})
+
+        handler = _HANDLER_REGISTRY.get(v_type)
+        if not handler:
+            logger.warning("unknown validator type '%s' for doc_type=%s", v_type, doc_type)
+            continue
+
+        try:
+            result = handler(
+                doc_path=doc_path,
+                skeleton_entries=skeleton_entries or [],
+                **params,
+            )
+            combined = combined.merge(result)
+        except Exception as e:
+            combined.warnings.append(f"validator:{v_type}: {e}")
+
+    return combined
+
+
+# ---------------------------------------------------------------------------
+# Built-in validator handlers
+# ---------------------------------------------------------------------------
+
+
+@register_handler("skeleton_count_match")
+def _skeleton_count_match(
+    doc_path: Path,
+    skeleton_entries: list[dict],
+    *,
+    annotations: list[str] | None = None,
+    skeleton_field: str = "controllers",
+    doc_heading_level: int = 2,
+    min_ratio: float = 0.5,
+    **_kwargs: Any,
+) -> ValidationResult:
+    """Compare count of annotated classes in skeleton vs heading count in doc.
+
+    Params:
+        annotations: List of annotation strings to match (e.g. ["@RestController"])
+        skeleton_field: Label for the count (used in messages)
+        doc_heading_level: Heading level to count in doc (default H2)
+        min_ratio: Minimum ratio of headings/annotations before warning
+    """
+    result = ValidationResult()
+
+    if not doc_path.exists():
+        return result
+
+    # Count annotated classes in skeleton
+    anno_count = 0
+    if annotations:
+        for entry in skeleton_entries:
+            for cls in entry.get("classes", []):
+                annos_str = " ".join(
+                    a.get("name", "") if isinstance(a, dict) else str(a)
+                    for a in cls.get("annotations", [])
+                )
+                if any(anno in annos_str for anno in annotations):
+                    anno_count += 1
+
+    if anno_count == 0:
+        return result
+
+    # Count headings in doc
+    try:
+        content = doc_path.read_text(encoding="utf-8")
+    except OSError:
+        return result
+
+    heading_pattern = re.compile(r"^#{" + str(doc_heading_level) + r"}\s+", re.MULTILINE)
+    heading_count = len(heading_pattern.findall(content))
+
+    if heading_count > 0:
+        ratio = heading_count / anno_count
+        if ratio < min_ratio:
+            result.warnings.append(
+                f"skeleton_count_match: {doc_path.name}: "
+                f"H{doc_heading_level} sections ({heading_count}) much fewer than "
+                f"skeleton {skeleton_field} ({anno_count})"
+            )
+
+    return result
+
+
+@register_handler("sampling_check")
+def _sampling_check(
+    doc_path: Path,
+    skeleton_entries: list[dict],
+    *,
+    source: str = "field_names",
+    sample_size: int = 5,
+    **_kwargs: Any,
+) -> ValidationResult:
+    """Spot-check that values from skeleton appear in the document.
+
+    Params:
+        source: What to sample — "enum_values" or "field_names"
+        sample_size: Number of items to sample
+    """
+    result = ValidationResult()
+
+    if not doc_path.exists():
+        return result
+
+    try:
+        content = doc_path.read_text(encoding="utf-8")
+    except OSError:
+        return result
+
+    extractor = _EXTRACTORS.get(source)
+    if not extractor:
+        logger.warning("unknown sampling source '%s'", source)
+        return result
+
+    values = extractor(skeleton_entries)
+    if not values:
+        return result
+
+    samples = random.sample(values, min(sample_size, len(values)))
+    for val in samples:
+        if val not in content:
+            result.warnings.append(
+                f"sampling_check: '{val}' (from {source}) not found in {doc_path.name}"
+            )
+
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Extractors for sampling_check
+# ---------------------------------------------------------------------------
+
+
+def _extract_enum_values(entries: list[dict]) -> list[str]:
+    """Extract enum constant names from skeleton entries."""
+    values: list[str] = []
+    for entry in entries:
+        for cls in entry.get("classes", []):
+            if cls.get("type", "").lower() == "enum":
+                for m in entry.get("methods", []):
+                    if m.get("name", "").isupper():
+                        values.append(m["name"])
+    return values
+
+
+def _extract_field_names(entries: list[dict]) -> list[str]:
+    """Extract field names from skeleton entries."""
+    names: list[str] = []
+    for entry in entries:
+        for field in entry.get("fields", []):
+            name = field.get("name", "")
+            if name and len(name) > 2:
+                names.append(name)
+    return names
+
+
+_EXTRACTORS: dict[str, Callable[[list[dict]], list[str]]] = {
+    "enum_values": _extract_enum_values,
+    "field_names": _extract_field_names,
+}