reporails-cli 0.0.1__py3-none-any.whl

reporails_cli/core/engine.py

@@ -0,0 +1,177 @@
+"""Validation engine - orchestration only, no domain logic.
+
+Coordinates other modules to run validation. Target: <200 lines.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import time
+from pathlib import Path
+
+from reporails_cli.bundled import get_capability_patterns_path
+from reporails_cli.core.applicability import detect_features_filesystem, get_applicable_rules
+from reporails_cli.core.bootstrap import get_agent_vars, get_opengrep_bin, is_initialized
+from reporails_cli.core.cache import record_scan
+from reporails_cli.core.capability import (
+    detect_features_content,
+    determine_capability_level,
+)
+from reporails_cli.core.discover import generate_backbone_yaml, run_discovery, save_backbone
+from reporails_cli.core.init import run_init
+from reporails_cli.core.models import PendingSemantic, Rule, RuleType, ValidationResult
+from reporails_cli.core.opengrep import get_rule_yml_paths, run_opengrep
+from reporails_cli.core.registry import load_rules
+from reporails_cli.core.sarif import dedupe_violations, parse_sarif
+from reporails_cli.core.scorer import calculate_score, estimate_friction
+from reporails_cli.core.semantic import build_semantic_requests
+
+
+def run_validation(
+    target: Path,
+    rules: dict[str, Rule] | None = None,
+    opengrep_path: Path | None = None,
+    rules_dir: Path | None = None,
+    use_cache: bool = True,
+    record_analytics: bool = True,
+    agent: str = "",
+) -> ValidationResult:
+    """Run full validation on target directory.
+
+    Two-pass approach:
+    1. Capability detection (small pattern set) → determines final level
+    2. Rule validation (filtered by final level) → violations + score
+
+    Args:
+        target: Directory or file to validate
+        rules: Pre-loaded rules (optional, loads from rules_dir if not provided)
+        opengrep_path: Path to OpenGrep binary (optional, auto-detects)
+        rules_dir: Directory containing rules (optional)
+        use_cache: Whether to use cached results
+        record_analytics: Whether to record scan analytics
+        agent: Agent identifier for loading template vars (empty = no agent-specific vars)
+    """
+    start_time = time.perf_counter()
+    project_root = target.parent if target.is_file() else target
+
+    # Auto-init if needed
+    if not is_initialized():
+        run_init()
+    if opengrep_path is None:
+        opengrep_path = get_opengrep_bin()
+
+    # Auto-create backbone if missing
+    backbone_path = project_root / ".reporails" / "backbone.yml"
+    if not backbone_path.exists():
+        save_backbone(project_root, generate_backbone_yaml(run_discovery(project_root)))
+
+    # Get template vars from agent config for yml placeholder resolution
+    template_context = get_agent_vars(agent) if agent else {}
+
+    # Load rules if not provided
+    if rules is None:
+        rules = load_rules(rules_dir)
+
+    # =========================================================================
+    # PASS 1: Capability Detection (determines final level)
+    # =========================================================================
+
+    # Filesystem feature detection (fast)
+    features = detect_features_filesystem(project_root)
+
+    # Content feature detection via OpenGrep (capability patterns only)
+    capability_patterns = get_capability_patterns_path()
+    capability_sarif = {}
+    if capability_patterns.exists():
+        capability_sarif = run_opengrep(
+            [capability_patterns], target, opengrep_path, template_context
+        )
+
+    content_features = detect_features_content(capability_sarif)
+
+    # Determine FINAL capability level (filesystem + content)
+    capability = determine_capability_level(features, content_features)
+    final_level = capability.level
+
+    # =========================================================================
+    # PASS 2: Rule Validation (filtered by final level)
+    # =========================================================================
+
+    # Filter rules by FINAL level - this ensures scoring matches displayed level
+    applicable_rules = get_applicable_rules(rules, final_level)
+
+    # Run OpenGrep on applicable rules only
+    rule_yml_paths = get_rule_yml_paths(applicable_rules)
+    rule_sarif = {}
+    if rule_yml_paths:
+        rule_sarif = run_opengrep(
+            rule_yml_paths, target, opengrep_path, template_context
+        )
+
+    # Split by type
+    deterministic = {k: v for k, v in applicable_rules.items() if v.type == RuleType.DETERMINISTIC}
+    semantic = {k: v for k, v in applicable_rules.items() if v.type == RuleType.SEMANTIC}
+
+    # Parse violations from rule SARIF (only deterministic rules)
+    violations = parse_sarif(rule_sarif, deterministic)
+
+    # Build semantic requests from rule SARIF (only semantic rules)
+    judgment_requests = build_semantic_requests(rule_sarif, semantic, project_root)
+
+    # =========================================================================
+    # Scoring (uses same rules that were filtered by final level)
+    # =========================================================================
+
+    unique_violations = dedupe_violations(violations)
+    score = calculate_score(len(applicable_rules), unique_violations)
+    friction = estimate_friction(unique_violations)
+    rules_failed = len({v.rule_id for v in unique_violations})
+
+    # Record analytics
+    elapsed_ms = (time.perf_counter() - start_time) * 1000
+    if record_analytics:
+        with contextlib.suppress(OSError):
+            record_scan(target, score, final_level.value, len(violations),
+                        len(applicable_rules), elapsed_ms, features.instruction_file_count)
+
+    # Build pending semantic summary
+    pending_semantic = None
+    if judgment_requests:
+        unique_rules = sorted({jr.rule_id for jr in judgment_requests})
+        unique_files = {jr.location.rsplit(":", 1)[0] for jr in judgment_requests}
+        pending_semantic = PendingSemantic(
+            rule_count=len(unique_rules),
+            file_count=len(unique_files),
+            rules=tuple(unique_rules),
+        )
+
+    return ValidationResult(
+        score=score,
+        level=final_level,
+        violations=tuple(violations),
+        judgment_requests=tuple(judgment_requests),
+        rules_checked=len(applicable_rules),
+        rules_passed=len(applicable_rules) - rules_failed,
+        rules_failed=rules_failed,
+        feature_summary=capability.feature_summary,
+        friction=friction,
+        is_partial=bool(judgment_requests),  # Partial if semantic rules pending
+        pending_semantic=pending_semantic,
+    )
+
+
+def run_validation_sync(
+    target: Path,
+    rules: dict[str, Rule] | None = None,
+    opengrep_path: Path | None = None,
+    rules_dir: Path | None = None,
+    use_cache: bool = True,
+    record_analytics: bool = True,
+    agent: str = "",
+    checks_dir: Path | None = None,  # Legacy alias
+) -> ValidationResult:
+    """Legacy alias for run_validation (now sync)."""
+    # Support legacy checks_dir parameter
+    if checks_dir is not None and rules_dir is None:
+        rules_dir = checks_dir
+    return run_validation(target, rules, opengrep_path, rules_dir, use_cache, record_analytics, agent)

reporails_cli/core/init.py

@@ -0,0 +1,309 @@
+"""Init command - downloads opengrep and syncs rules."""
+
+from __future__ import annotations
+
+import importlib.resources
+import platform
+import shutil
+import stat
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+import httpx
+
+from reporails_cli.core.bootstrap import get_global_config, get_opengrep_bin, get_reporails_home
+
+# Hardcoded version - no env var handling
+OPENGREP_VERSION = "1.15.1"
+
+OPENGREP_URLS: dict[tuple[str, str], str] = {
+    ("linux", "x86_64"): (
+        "https://github.com/opengrep/opengrep/releases/download/"
+        f"v{OPENGREP_VERSION}/opengrep_manylinux_x86"
+    ),
+    ("linux", "aarch64"): (
+        "https://github.com/opengrep/opengrep/releases/download/"
+        f"v{OPENGREP_VERSION}/opengrep_manylinux_aarch64"
+    ),
+    ("darwin", "x86_64"): (
+        "https://github.com/opengrep/opengrep/releases/download/"
+        f"v{OPENGREP_VERSION}/opengrep_osx_x86"
+    ),
+    ("darwin", "arm64"): (
+        "https://github.com/opengrep/opengrep/releases/download/"
+        f"v{OPENGREP_VERSION}/opengrep_osx_arm64"
+    ),
+    ("windows", "x86_64"): (
+        "https://github.com/opengrep/opengrep/releases/download/"
+        f"v{OPENGREP_VERSION}/opengrep-core_windows_x86.zip"
+    ),
+}
+
+RULES_VERSION = "v0.0.1"
+RULES_TARBALL_URL = "https://github.com/reporails/rules/releases/download/{version}/reporails-rules-{version}.tar.gz"
+
+
+def get_platform() -> tuple[str, str]:
+    """Detect current platform."""
+    system = platform.system().lower()
+    machine = platform.machine().lower()
+
+    if system == "darwin":
+        os_name = "darwin"
+    elif system == "linux":
+        os_name = "linux"
+    elif system == "windows":
+        os_name = "windows"
+    else:
+        msg = f"Unsupported operating system: {system}"
+        raise RuntimeError(msg)
+
+    if machine in ("x86_64", "amd64"):
+        arch = "x86_64"
+    elif machine in ("arm64", "aarch64"):
+        arch = "arm64" if os_name == "darwin" else "aarch64"
+    else:
+        msg = f"Unsupported architecture: {machine}"
+        raise RuntimeError(msg)
+
+    return os_name, arch
+
+
+def download_opengrep() -> Path:
+    """Download opengrep binary to ~/.reporails/bin/opengrep."""
+    os_name, arch = get_platform()
+    key = (os_name, arch)
+
+    if key not in OPENGREP_URLS:
+        msg = f"Unsupported platform: {os_name}/{arch}"
+        raise RuntimeError(msg)
+
+    url = OPENGREP_URLS[key]
+    bin_path = get_opengrep_bin()
+
+    # Create bin directory
+    bin_path.parent.mkdir(parents=True, exist_ok=True)
+
+    # Download
+    with httpx.Client(follow_redirects=True, timeout=120.0) as client:
+        response = client.get(url)
+        response.raise_for_status()
+
+        # Write binary directly (raw binary, not archive for non-windows)
+        bin_path.write_bytes(response.content)
+
+        # Make executable on Unix
+        if os_name != "windows":
+            bin_path.chmod(bin_path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+
+    return bin_path
+
+
+def get_bundled_checks_path() -> Path | None:
+    """
+    Get path to bundled checks (.yml files) in installed package.
+
+    Returns:
+        Path to bundled_checks directory, or None if not found
+    """
+    try:
+        # Use importlib.resources to find bundled checks
+        files = importlib.resources.files("reporails_cli")
+        bundled = files / "bundled_checks"
+        # Convert to Path - this works for installed packages
+        with importlib.resources.as_file(bundled) as path:
+            if path.exists():
+                return path
+    except (TypeError, FileNotFoundError):
+        pass
+    return None
+
+
+def copy_bundled_yml_files(dest: Path) -> int:
+    """
+    Copy bundled .yml files from package to destination.
+
+    Args:
+        dest: Destination directory
+
+    Returns:
+        Number of .yml files copied
+    """
+    bundled_path = get_bundled_checks_path()
+    if bundled_path is None:
+        return 0
+
+    dest.mkdir(parents=True, exist_ok=True)
+    count = 0
+
+    for yml_file in bundled_path.rglob("*.yml"):
+        # Preserve directory structure
+        relative = yml_file.relative_to(bundled_path)
+        dest_file = dest / relative
+        dest_file.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(yml_file, dest_file)
+        count += 1
+
+    return count
+
+
+def copy_local_framework(source: Path) -> tuple[Path, int]:
+    """
+    Copy rules from local framework directory to ~/.reporails/rules/.
+
+    Used in dev mode when framework_path is configured in ~/.reporails/config.yml.
+
+    Local framework structure:
+        source/
+        ├── core/           # Core rules
+        ├── agents/         # Agent-specific rules
+        │   └── claude/
+        │       └── rules/  # Claude-specific rules
+        ├── schemas/
+        └── docs/
+
+    Args:
+        source: Local framework directory path
+
+    Returns:
+        Tuple of (rules_path, total_file_count)
+    """
+    rules_path = get_reporails_home() / "rules"
+
+    # Clear existing rules
+    if rules_path.exists():
+        shutil.rmtree(rules_path)
+
+    rules_path.mkdir(parents=True, exist_ok=True)
+    count = 0
+
+    # Directories to copy from framework root
+    dirs_to_copy = ["core", "agents", "schemas", "docs"]
+
+    for dir_name in dirs_to_copy:
+        source_dir = source / dir_name
+        if source_dir.exists() and source_dir.is_dir():
+            dest_dir = rules_path / dir_name
+            shutil.copytree(source_dir, dest_dir)
+            # Count files copied
+            count += sum(1 for _ in dest_dir.rglob("*") if _.is_file())
+
+    return rules_path, count
+
+
+def download_rules_tarball(dest: Path) -> int:
+    """
+    Download rules from GitHub release tarball.
+
+    Args:
+        dest: Destination directory (~/.reporails/rules/)
+
+    Returns:
+        Number of files extracted
+    """
+    import tarfile
+
+    url = RULES_TARBALL_URL.format(version=RULES_VERSION)
+
+    with httpx.Client(follow_redirects=True, timeout=120.0) as client:
+        response = client.get(url)
+        response.raise_for_status()
+
+        with TemporaryDirectory() as tmpdir:
+            tarball_path = Path(tmpdir) / "rules.tar.gz"
+            tarball_path.write_bytes(response.content)
+
+            # Extract
+            with tarfile.open(tarball_path, "r:gz") as tar:
+                tar.extractall(path=dest)
+
+            # Count files
+            count = sum(1 for _ in dest.rglob("*") if _.is_file())
+
+    return count
+
+
+def download_from_github() -> tuple[Path, int]:
+    """
+    Setup rules from GitHub at ~/.reporails/rules/.
+
+    Merges two sources:
+    1. Bundled .yml files (OpenGrep patterns) from package
+    2. Downloaded files from GitHub release tarball
+
+    Returns:
+        Tuple of (rules_path, total_file_count)
+    """
+    rules_path = get_reporails_home() / "rules"
+
+    # Clear existing rules
+    if rules_path.exists():
+        shutil.rmtree(rules_path)
+
+    rules_path.mkdir(parents=True, exist_ok=True)
+
+    # 1. Copy bundled .yml files
+    yml_count = copy_bundled_yml_files(rules_path)
+
+    # 2. Download from GitHub release tarball
+    tarball_count = download_rules_tarball(rules_path)
+
+    return rules_path, yml_count + tarball_count
+
+
+def download_rules() -> tuple[Path, int]:
+    """
+    Setup rules at ~/.reporails/rules/.
+
+    Checks for local framework_path in config first (dev mode),
+    otherwise downloads from GitHub.
+
+    Returns:
+        Tuple of (rules_path, total_file_count)
+    """
+    # Check for local framework override (dev mode)
+    config = get_global_config()
+    if config.framework_path and config.framework_path.exists():
+        return copy_local_framework(config.framework_path)
+
+    # Otherwise download from GitHub
+    return download_from_github()
+
+
+def sync_rules_to_local(local_checks_dir: Path) -> int:
+    """
+    Sync rules from GitHub release tarball to local checks directory.
+
+    For development: downloads rules from release tarball.
+
+    Args:
+        local_checks_dir: Local checks directory (e.g., ./checks/)
+
+    Returns:
+        Number of files synced
+    """
+    return download_rules_tarball(local_checks_dir)
+
+
+def run_init() -> dict[str, str | int | Path]:
+    """
+    Run global initialization.
+
+    1. Download opengrep binary to ~/.reporails/bin/
+    2. Setup rules at ~/.reporails/rules/ (from local framework or GitHub)
+
+    Returns dict with status info.
+    """
+    results: dict[str, str | int | Path] = {}
+
+    # 1. Download opengrep
+    bin_path = download_opengrep()
+    results["opengrep_path"] = bin_path
+    results["opengrep_version"] = OPENGREP_VERSION
+
+    # 2. Setup rules (check local framework_path first, then GitHub)
+    rules_path, rule_count = download_rules()
+    results["rules_path"] = rules_path
+    results["rule_count"] = rule_count
+
+    return results

reporails_cli/core/levels.py

@@ -0,0 +1,177 @@
+"""Level configuration and rule-to-level mapping.
+
+Loads from bundled levels.yml. All functions are pure after initial load.
+"""
+
+from __future__ import annotations
+
+from functools import lru_cache
+from typing import TYPE_CHECKING, Any
+
+import yaml
+
+from reporails_cli.bundled import get_levels_path
+from reporails_cli.core.models import Level
+
+if TYPE_CHECKING:
+    from reporails_cli.core.models import DetectedFeatures
+
+# Level labels - must match levels.yml
+LEVEL_LABELS: dict[Level, str] = {
+    Level.L1: "Absent",
+    Level.L2: "Basic",
+    Level.L3: "Structured",
+    Level.L4: "Abstracted",
+    Level.L5: "Governed",
+    Level.L6: "Adaptive",
+}
+
+
+@lru_cache(maxsize=1)
+def get_level_config() -> dict[str, Any]:
+    """Load bundled levels.yml configuration.
+
+    Cached for performance.
+
+    Returns:
+        Parsed levels.yml content
+    """
+    levels_path = get_levels_path()
+    if not levels_path.exists():
+        return {"levels": {}, "score_thresholds": {}, "detection": {}}
+
+    content = levels_path.read_text(encoding="utf-8")
+    config: dict[str, Any] = yaml.safe_load(content) or {}
+    return config
+
+
+def get_rules_for_level(level: Level) -> set[str]:
+    """Get all rule IDs required for a given level.
+
+    Includes rules from all levels up to and including the given level.
+
+    Args:
+        level: Target capability level
+
+    Returns:
+        Set of rule IDs applicable at this level
+    """
+    config = get_level_config()
+    levels_data = config.get("levels", {})
+
+    # Build rules set by traversing level inheritance
+    all_rules: set[str] = set()
+    level_order = [Level.L1, Level.L2, Level.L3, Level.L4, Level.L5, Level.L6]
+    target_index = level_order.index(level)
+
+    for lvl in level_order[: target_index + 1]:
+        level_key = lvl.value
+        if level_key in levels_data:
+            rules = levels_data[level_key].get("required_rules", [])
+            all_rules.update(rules)
+
+    return all_rules
+
+
+def get_level_label(level: Level) -> str:
+    """Get human-readable label for level.
+
+    Args:
+        level: Capability level
+
+    Returns:
+        Label string (e.g., "Abstracted")
+    """
+    return LEVEL_LABELS.get(level, "Unknown")
+
+
+def get_level_includes(level: Level) -> list[Level]:
+    """Get levels included by inheritance.
+
+    Args:
+        level: Target level
+
+    Returns:
+        List of included levels (lower levels)
+    """
+    config = get_level_config()
+    levels_data = config.get("levels", {})
+
+    level_key = level.value
+    if level_key not in levels_data:
+        return []
+
+    includes = levels_data[level_key].get("includes", [])
+    return [Level(inc) for inc in includes if inc in [lv.value for lv in Level]]
+
+
+def get_score_threshold(level: Level) -> int:
+    """Get capability score threshold for a level.
+
+    Args:
+        level: Target level
+
+    Returns:
+        Minimum score required for this level
+    """
+    config = get_level_config()
+    thresholds = config.get("score_thresholds", {})
+    result = thresholds.get(level.value, 0)
+    return int(result)
+
+
+def capability_score_to_level(score: int) -> Level:
+    """Map capability score to level.
+
+    Args:
+        score: Capability score (0-12)
+
+    Returns:
+        Corresponding level
+    """
+    config = get_level_config()
+    thresholds = config.get("score_thresholds", {})
+
+    # Default thresholds if not in config
+    if not thresholds:
+        thresholds = {"L1": 0, "L2": 1, "L3": 3, "L4": 5, "L5": 7, "L6": 10}
+
+    # Find highest level where score meets threshold
+    level_order = [Level.L6, Level.L5, Level.L4, Level.L3, Level.L2, Level.L1]
+    for level in level_order:
+        threshold = thresholds.get(level.value, 0)
+        if score >= threshold:
+            return level
+
+    return Level.L1
+
+
+def detect_orphan_features(features: DetectedFeatures, base_level: Level) -> bool:
+    """Check if project has features from levels above base level.
+
+    Example: L3 project with backbone.yml (L6 feature) → has_orphan = True
+    Display as "L3+" to indicate advanced features present.
+
+    Args:
+        features: Detected project features
+        base_level: Base capability level
+
+    Returns:
+        True if features above base level are present
+    """
+    level_features: dict[Level, list[bool]] = {
+        Level.L6: [features.has_backbone],
+        Level.L5: [features.component_count >= 3, features.has_shared_files],
+        Level.L4: [features.has_rules_dir],
+        Level.L3: [features.has_imports, features.has_multiple_instruction_files],
+    }
+
+    level_order = [Level.L1, Level.L2, Level.L3, Level.L4, Level.L5, Level.L6]
+    base_index = level_order.index(base_level)
+
+    # Check features from levels above base
+    for level in level_order[base_index + 1 :]:
+        if level in level_features and any(level_features[level]):
+            return True
+
+    return False