ixt-cli 0.8.0__py3-none-any.whl

ixt/config/heuristics.py

@@ -0,0 +1,465 @@
+"""Heuristic asset selection engine for binary backend.
+
+Loads heuristics.toml (embedded defaults + optional user overrides)
+and implements the 3-level asset selection algorithm:
+  Level 1 — Exact pattern (from ixt.setup.toml)
+  Level 2 — Generic patterns (from heuristics.toml)
+  Level 3 — Scoring fallback
+"""
+
+from __future__ import annotations
+
+import fnmatch
+import functools
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from ixt.config.toml import _load_toml
+from ixt.data import get_data_path
+from ixt.net.source import Asset
+from ixt.platform import OS, Arch, get_arch, get_os
+
+# ---------------------------------------------------------------------------
+# Config models
+# ---------------------------------------------------------------------------
+
+
+@dataclass(slots=True)
+class ScoringConfig:
+    prefer: list[str] = field(default_factory=list)
+    avoid: list[str] = field(default_factory=list)
+    skip_ext: list[str] = field(default_factory=list)
+    archive_ext: list[str] = field(default_factory=list)
+    archive_priority: list[str] = field(default_factory=list)
+
+
+@dataclass(slots=True)
+class ExposeConfig:
+    skip_names: list[str] = field(default_factory=list)
+    skip_patterns: list[str] = field(default_factory=list)
+    skip_keywords: list[str] = field(default_factory=list)
+
+
+@dataclass(slots=True)
+class NamesConfig:
+    strip_suffix_tokens: list[str] = field(default_factory=list)
+
+
+@dataclass(slots=True)
+class HeuristicsConfig:
+    os_aliases: dict[str, list[str]] = field(default_factory=dict)
+    arch_aliases: dict[str, list[str]] = field(default_factory=dict)
+    patterns: list[str] = field(default_factory=list)
+    scoring: ScoringConfig = field(default_factory=ScoringConfig)
+    expose: ExposeConfig = field(default_factory=ExposeConfig)
+    names: NamesConfig = field(default_factory=NamesConfig)
+
+
+# ---------------------------------------------------------------------------
+# Loading
+# ---------------------------------------------------------------------------
+
+_OS_MAP: dict[OS, str] = {
+    OS.LINUX: "linux",
+    OS.MACOS: "macos",
+    OS.WINDOWS: "windows",
+}
+
+_ARCH_MAP: dict[Arch, str] = {
+    Arch.X86_64: "x86_64",
+    Arch.ARM64: "aarch64",
+    Arch.ARMV7: "armv7",
+}
+
+
+def _parse_heuristics(data: dict[str, Any]) -> HeuristicsConfig:
+    """Build a HeuristicsConfig from parsed TOML dict."""
+    scoring_data = data.get("scoring", {})
+    expose_data = data.get("expose", {})
+    names_data = data.get("names", {})
+
+    return HeuristicsConfig(
+        os_aliases=data.get("os", {}),
+        arch_aliases=data.get("arch", {}),
+        patterns=data.get("patterns", {}).get("templates", []),
+        scoring=ScoringConfig(
+            prefer=scoring_data.get("prefer", []),
+            avoid=scoring_data.get("avoid", []),
+            skip_ext=scoring_data.get("skip_ext", []),
+            archive_ext=scoring_data.get("archive_ext", []),
+            archive_priority=scoring_data.get("archive_priority", []),
+        ),
+        expose=ExposeConfig(
+            skip_names=expose_data.get("skip_names", []),
+            skip_patterns=expose_data.get("skip_patterns", []),
+            skip_keywords=expose_data.get("skip_keywords", []),
+        ),
+        names=NamesConfig(
+            strip_suffix_tokens=names_data.get("strip_suffix_tokens", []),
+        ),
+    )
+
+
+def _merge_heuristics(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]:
+    """Shallow merge: override replaces entire top-level sections."""
+    merged = dict(base)
+    for key, val in override.items():
+        if isinstance(val, dict) and isinstance(merged.get(key), dict):
+            merged[key] = {**merged[key], **val}
+        else:
+            merged[key] = val
+    return merged
+
+
+@functools.lru_cache(maxsize=1)
+def load_heuristics(user_path: Path | None = None) -> HeuristicsConfig:
+    """Load heuristics config (embedded defaults + optional user overrides).
+
+    Args:
+        user_path: Explicit path to user override file.
+                   If None, checks $IXT_HOME/config/heuristics.toml.
+    """
+    # Embedded defaults
+    data = _load_toml(get_data_path("heuristics.toml"))
+
+    # User overrides
+    if user_path is None:
+        from ixt.config.settings import get_ixt_config_dir
+
+        user_path = get_ixt_config_dir() / "heuristics.toml"
+
+    if user_path.is_file():
+        user_data = _load_toml(user_path)
+        data = _merge_heuristics(data, user_data)
+
+    return _parse_heuristics(data)
+
+
+# ---------------------------------------------------------------------------
+# Verbose trace
+# ---------------------------------------------------------------------------
+
+
+class _Trace:
+    """Collects selection trace lines. No-op when level=0.
+
+    level 1 (-v):  key decisions — pattern matches, winner, scoring
+    level 2 (-vv): adds filtering — skipped assets, all patterns tried
+    """
+
+    def __init__(self, level: int = 0):
+        self.level = level
+        self.lines: list[str] = []
+
+    def __call__(self, msg: str, *, min_level: int = 1) -> None:
+        if self.level >= min_level:
+            self.lines.append(msg)
+
+    def flush(self) -> None:
+        if not self.level or not self.lines:
+            return
+        from ixt.libs.logger import Level, get_logger
+
+        log = get_logger("heuristics", Level.DEBUG)
+        for line in self.lines:
+            log.debug(line)
+
+
+# ---------------------------------------------------------------------------
+# Pattern helpers
+# ---------------------------------------------------------------------------
+
+
+def _resolve_pattern(pattern: str, name: str, os_str: str, arch_str: str, version: str) -> str:
+    """Replace {name}, {os}, {arch}, {version} in a pattern template."""
+    result = pattern.replace("{name}", name)
+    result = result.replace("{os}", os_str)
+    result = result.replace("{arch}", arch_str)
+    result = result.replace("{version}", version)
+    return result
+
+
+def _get_os_aliases(cfg: HeuristicsConfig, os_key: str) -> list[str]:
+    """Return all aliases for the given OS key (includes the key itself)."""
+    aliases = cfg.os_aliases.get(os_key, [])
+    if os_key not in aliases:
+        return [os_key, *aliases]
+    return aliases
+
+
+def _get_arch_aliases(cfg: HeuristicsConfig, arch_key: str) -> list[str]:
+    """Return all aliases for the given arch key (includes the key itself)."""
+    aliases = cfg.arch_aliases.get(arch_key, [])
+    if arch_key not in aliases:
+        return [arch_key, *aliases]
+    return aliases
+
+
+def get_platform_aliases(
+    cfg: HeuristicsConfig | None = None,
+    *,
+    os_override: OS | None = None,
+    arch_override: Arch | None = None,
+) -> tuple[list[str], list[str]]:
+    """Return OS and architecture aliases for the current platform."""
+    cfg = cfg or load_heuristics()
+    current_os = _OS_MAP[os_override or get_os()]
+    current_arch = _ARCH_MAP[arch_override or get_arch()]
+    return _get_os_aliases(cfg, current_os), _get_arch_aliases(cfg, current_arch)
+
+
+def get_name_suffix_tokens(cfg: HeuristicsConfig | None = None) -> list[str]:
+    """Return extra suffix tokens stripped from generated executable names."""
+    cfg = cfg or load_heuristics()
+    return list(cfg.names.strip_suffix_tokens)
+
+
+def _has_skip_ext(name_lower: str, skip_ext: list[str]) -> bool:
+    """Check if asset name ends with a skippable extension."""
+    return any(name_lower.endswith(ext) for ext in skip_ext)
+
+
+def _has_avoid_keyword(name_lower: str, avoid: list[str]) -> bool:
+    """Check if asset name contains an avoid keyword."""
+    return any(kw in name_lower for kw in avoid)
+
+
+def _get_archive_priority_score(name_lower: str, priority: list[str]) -> int:
+    """Return archive priority score (higher index = lower priority)."""
+    for i, ext in enumerate(priority):
+        if name_lower.endswith(ext):
+            return len(priority) - i  # first in list = highest score
+    return 0
+
+
+# ---------------------------------------------------------------------------
+# Asset selection — 3-level algorithm
+# ---------------------------------------------------------------------------
+
+
+def select_asset(
+    assets: list[Asset],
+    name: str,
+    version: str = "",
+    *,
+    heuristics: HeuristicsConfig | None = None,
+    asset_pattern: str | None = None,
+    os_override: OS | None = None,
+    arch_override: Arch | None = None,
+    verbose: bool | int = False,
+) -> Asset | None:
+    """Select the best matching asset from a release.
+
+    Args:
+        assets: List of release assets.
+        name: Tool name (e.g. "ripgrep", "fd").
+        version: Release version tag (e.g. "14.1.0"), stripped of "v" prefix.
+        heuristics: Config to use. Loads defaults if None.
+        asset_pattern: Exact pattern from ixt.setup.toml (Level 1).
+        os_override: Override detected OS (for testing).
+        arch_override: Override detected arch (for testing).
+        verbose: Verbosity level (0=off, 1=decisions, 2=filtering details).
+
+    Returns:
+        Best matching Asset, or None if no match found.
+    """
+    if not assets:
+        return None
+
+    cfg = heuristics or load_heuristics()
+    current_os = _OS_MAP[os_override or get_os()]
+    current_arch = _ARCH_MAP[arch_override or get_arch()]
+    t = _Trace(int(verbose))
+
+    # Strip v prefix from version for pattern matching
+    ver = version.lstrip("v") if version else ""
+
+    os_aliases = _get_os_aliases(cfg, current_os)
+    arch_aliases = _get_arch_aliases(cfg, current_arch)
+
+    t(f"select: name={name!r} version={ver!r} os={current_os} arch={current_arch}")
+    t(f"  os_aliases:   {os_aliases}")
+    t(f"  arch_aliases: {arch_aliases}")
+    t(f"  assets ({len(assets)}): {[a.name for a in assets]}")
+
+    # Level 1 — Exact pattern
+    if asset_pattern:
+        t(f"  level 1: asset_pattern={asset_pattern!r}")
+        matches = _match_pattern(assets, asset_pattern, name, current_os, current_arch, ver, t)
+        if len(matches) == 1:
+            t(f"  level 1 -> exact match: {matches[0].name}")
+            t.flush()
+            return matches[0]
+        if matches:
+            result = _score_best(matches, os_aliases, arch_aliases, cfg.scoring, t)
+            t(f"  level 1 -> scored winner: {result.name if result else 'none'}")
+            t.flush()
+            return result
+        t("  level 1 -> no match")
+
+    # Level 2 — Generic patterns
+    t("  level 2: trying generic patterns")
+    for pattern in cfg.patterns:
+        if "{version}" in pattern and not ver:
+            t(f"    pattern {pattern!r} -> skip (no version)", min_level=2)
+            continue
+        matches = _match_pattern(assets, pattern, name, current_os, current_arch, ver, t)
+        if matches:
+            t(f"    pattern {pattern!r} -> {len(matches)} match(es): {[a.name for a in matches]}")
+            if len(matches) == 1:
+                t(f"  level 2 -> single match: {matches[0].name}")
+                t.flush()
+                return matches[0]
+            result = _score_best(matches, os_aliases, arch_aliases, cfg.scoring, t)
+            t(f"  level 2 -> scored winner: {result.name if result else 'none'}")
+            t.flush()
+            return result
+        t(f"    pattern {pattern!r} -> no match", min_level=2)
+
+    # Level 3 — Scoring fallback
+    t("  level 3: scoring all assets")
+    result = _score_assets(assets, os_aliases, arch_aliases, cfg.scoring, t)
+    t(f"  level 3 -> winner: {result.name if result else 'none'}")
+    t.flush()
+    return result
+
+
+def _match_pattern(
+    assets: list[Asset],
+    pattern: str,
+    name: str,
+    os_str: str,
+    arch_str: str,
+    version: str,
+    trace: _Trace | None = None,
+) -> list[Asset]:
+    """Try to match assets against a pattern template.
+
+    For each OS alias x arch alias combination, resolve the pattern
+    and fnmatch against all assets. Assets with skip_ext are excluded.
+    Return list of matches (may be empty).
+    """
+    t = trace or _Trace(0)
+    cfg = load_heuristics()
+    os_aliases = _get_os_aliases(cfg, os_str)
+    arch_aliases = _get_arch_aliases(cfg, arch_str)
+
+    matches: list[Asset] = []
+    seen: set[str] = set()
+    skipped: list[str] = []
+
+    for os_alias in os_aliases:
+        for arch_alias in arch_aliases:
+            resolved = _resolve_pattern(pattern, name, os_alias, arch_alias, version)
+            for asset in assets:
+                if asset.name in seen:
+                    continue
+                lower = asset.name.lower()
+                if _has_skip_ext(lower, cfg.scoring.skip_ext):
+                    seen.add(asset.name)
+                    skipped.append(asset.name)
+                    continue
+                if fnmatch.fnmatch(lower, resolved.lower()):
+                    matches.append(asset)
+                    seen.add(asset.name)
+
+    if skipped:
+        t(f"    skip_ext: {skipped}", min_level=2)
+
+    return matches
+
+
+def _score_best(
+    assets: list[Asset],
+    os_aliases: list[str],
+    arch_aliases: list[str],
+    scoring: ScoringConfig,
+    trace: _Trace | None = None,
+) -> Asset | None:
+    """Score a pre-filtered list and return the best."""
+    scored = _compute_scores(assets, os_aliases, arch_aliases, scoring, trace)
+    if not scored:
+        return None
+    scored.sort(key=lambda x: (-x[1], len(x[0].name)))
+    return scored[0][0]
+
+
+def _score_assets(
+    assets: list[Asset],
+    os_aliases: list[str],
+    arch_aliases: list[str],
+    scoring: ScoringConfig,
+    trace: _Trace | None = None,
+) -> Asset | None:
+    """Level 3: score all assets and return the best match."""
+    t = trace or _Trace(0)
+
+    # Pre-filter
+    candidates = []
+    for asset in assets:
+        lower = asset.name.lower()
+        if _has_skip_ext(lower, scoring.skip_ext):
+            t(f"    skip (ext): {asset.name}", min_level=2)
+            continue
+        if _has_avoid_keyword(lower, scoring.avoid):
+            t(f"    skip (avoid): {asset.name}", min_level=2)
+            continue
+        candidates.append(asset)
+
+    scored = _compute_scores(candidates, os_aliases, arch_aliases, scoring, trace)
+    if not scored:
+        return None
+
+    # Must have at least OS or arch match
+    scored = [(a, s) for a, s in scored if s > 0]
+    if not scored:
+        return None
+
+    scored.sort(key=lambda x: (-x[1], len(x[0].name)))
+    return scored[0][0]
+
+
+def _compute_scores(
+    assets: list[Asset],
+    os_aliases: list[str],
+    arch_aliases: list[str],
+    scoring: ScoringConfig,
+    trace: _Trace | None = None,
+) -> list[tuple[Asset, int]]:
+    """Compute score for each asset."""
+    t = trace or _Trace(False)
+    results: list[tuple[Asset, int]] = []
+
+    for asset in assets:
+        lower = asset.name.lower()
+        score = 0
+        reasons: list[str] = []
+
+        # OS match (+10)
+        if any(alias.lower() in lower for alias in os_aliases):
+            score += 10
+            reasons.append("os+10")
+
+        # Arch match (+10)
+        if any(alias.lower() in lower for alias in arch_aliases):
+            score += 10
+            reasons.append("arch+10")
+
+        # Prefer keywords (earlier in list = higher bonus)
+        for i, kw in enumerate(scoring.prefer):
+            if kw.lower() in lower:
+                bonus = len(scoring.prefer) - i + 1
+                score += bonus
+                reasons.append(f"{kw}+{bonus}")
+
+        # Archive priority (+1..N)
+        ap = _get_archive_priority_score(lower, scoring.archive_priority)
+        if ap:
+            score += ap
+            reasons.append(f"archive+{ap}")
+
+        t(f"    score {score:3d} ({', '.join(reasons) or '-':30s}) {asset.name}")
+        results.append((asset, score))
+
+    return results

ixt/config/models.py

@@ -0,0 +1,176 @@
+"""Backend-agnostic metadata persisted per installed tool."""
+
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any
+
+from ixt.libs.constants import EXPOSE_MAIN
+
+_METADATA_FILENAME = "ixt.json"
+_MAX_METADATA_BYTES = 1024 * 1024
+
+
+def _metadata_path(file_path: str | Path) -> Path:
+    """Return a ToolRecord metadata path after cheap path-safety checks."""
+    path = Path(file_path).expanduser()
+    if path.name != _METADATA_FILENAME:
+        raise ValueError(f"ToolRecord metadata path must end with {_METADATA_FILENAME!r}: {path}")
+    if ".." in path.parts:
+        raise ValueError(f"Path traversal detected in ToolRecord metadata path: {path}")
+    return path
+
+
+def _absolute_path(path: str | Path) -> Path:
+    return Path(os.path.abspath(Path(path).expanduser()))
+
+
+def _encode_under_base(value: str, base: Path) -> str:
+    """Persist *value* relative to *base* when it is inside that tree."""
+    path = Path(value).expanduser()
+    if not path.is_absolute():
+        return value
+    try:
+        rel = _absolute_path(path).relative_to(_absolute_path(base))
+    except ValueError:
+        return value
+    return "." if rel == Path(".") else str(rel)
+
+
+def _metadata_env_dir(data: dict[str, Any], path: Path) -> tuple[Path, Path | None]:
+    """Return current env dir and legacy absolute env dir, if one was stored."""
+    raw = data.get("env_dir", ".")
+    raw_path = Path(raw)
+    if not raw_path.is_absolute():
+        return _absolute_path(path.parent / raw_path), None
+
+    current = _absolute_path(raw_path)
+    metadata_parent = _absolute_path(path.parent)
+    # Legacy records stored absolute env_dir values. If the metadata file is
+    # now inside a same-named env dir elsewhere, trust the file location so a
+    # copied installed layer can rebase itself on load.
+    if current != metadata_parent and current.name == metadata_parent.name:
+        return metadata_parent, current
+    return current, current
+
+
+def _decode_exposed_bin(value: str, current_env_dir: Path, legacy_env_dir: Path | None) -> str:
+    path = Path(value)
+    if path.is_absolute():
+        if legacy_env_dir is not None:
+            try:
+                return str(current_env_dir / path.relative_to(legacy_env_dir))
+            except ValueError:
+                pass
+        return value
+    return str(current_env_dir / path)
+
+
+@dataclass(slots=True)
+class ToolRecord:
+    """Persisted state for one installed tool (``ixt.json``).
+
+    This model is intentionally flat and backend-agnostic.
+    Backend-specific details (venv layout, node_modules structure)
+    stay in the backend implementations.
+    """
+
+    name: str  # installed tool id (equals env_dir basename; may include a --slot prefix)
+    backend: str  # "python" | "node" | "binary"
+    spec: str  # original install spec as given by user
+    env_dir: str  # in-memory absolute path; persisted relative when possible
+    version: str | None = None
+    # Underlying package name for resolve/expose. None on legacy records →
+    # pkg() falls back to ``name`` for records without explicit package metadata.
+    pkg_name: str | None = None
+    injected: list[str] = field(default_factory=list)
+    expose_rules: list[str] = field(default_factory=lambda: [EXPOSE_MAIN])
+    exposed_bins: dict[str, str] = field(default_factory=dict)
+    node_shim: bool | None = None
+    asset_pattern: str | None = None
+    asset_pattern_forced: bool = False
+    env_base: str = "all"
+    env_allow: list[str] = field(default_factory=list)
+    env_deny: dict[str, dict[str, list[str]]] = field(default_factory=dict)
+    fs_base: str = "all"
+    fs_ro: list[str] = field(default_factory=list)
+    fs_rw: list[str] = field(default_factory=list)
+    fs_scratch: list[str] = field(default_factory=list)
+    source: str = "registry"  # "registry" | "local" (extensible: "git", "url", ...)
+    config_version: str = "1"
+
+    def pkg(self) -> str:
+        """Underlying package name (for PyPI/npm resolve, backend bin lookup)."""
+        return self.pkg_name or self.name
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> ToolRecord:
+        return cls(
+            name=data["name"],
+            backend=data["backend"],
+            spec=data["spec"],
+            env_dir=data.get("env_dir", "."),
+            version=data.get("version"),
+            pkg_name=data.get("pkg_name"),
+            injected=list(data.get("injected", [])),
+            expose_rules=list(data.get("expose_rules", [EXPOSE_MAIN])),
+            exposed_bins=dict(data.get("exposed_bins", {})),
+            node_shim=data.get("node_shim"),
+            asset_pattern=data.get("asset_pattern"),
+            asset_pattern_forced=bool(data.get("asset_pattern_forced", False)),
+            env_base=data.get("env_base", "all"),
+            env_allow=list(data.get("env_allow", [])),
+            env_deny=dict(data.get("env_deny", {})),
+            fs_base=data.get("fs_base", "all"),
+            fs_ro=list(data.get("fs_ro", [])),
+            fs_rw=list(data.get("fs_rw", [])),
+            fs_scratch=list(data.get("fs_scratch", [])),
+            source=data.get("source", "registry"),
+            config_version=data.get("config_version", "1"),
+        )
+
+    @classmethod
+    def load_json(cls, file_path: str | Path) -> ToolRecord:
+        path = _metadata_path(file_path)
+        if not path.exists():
+            raise FileNotFoundError(path)
+        if path.is_symlink() or not path.is_file():
+            raise ValueError(f"ToolRecord metadata must be a regular file: {path}")
+        if path.stat().st_size > _MAX_METADATA_BYTES:
+            raise ValueError(f"ToolRecord metadata is too large: {path}")
+        data = json.loads(path.read_text(encoding="utf-8"))
+        current_env_dir, legacy_env_dir = _metadata_env_dir(data, path)
+        record = cls.from_dict(data)
+        record.env_dir = str(current_env_dir)
+        record.exposed_bins = {
+            name: _decode_exposed_bin(source, current_env_dir, legacy_env_dir)
+            for name, source in record.exposed_bins.items()
+        }
+        return record
+
+    def save_json(self, file_path: str | Path) -> Path:
+        path = _metadata_path(file_path)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        if path.exists() and (path.is_symlink() or not path.is_file()):
+            raise ValueError(f"ToolRecord metadata destination must be a regular file: {path}")
+        env_dir = _absolute_path(self.env_dir)
+        data = asdict(self)
+        data["env_dir"] = _encode_under_base(str(env_dir), path.parent)
+        data["exposed_bins"] = {
+            name: _encode_under_base(source, env_dir)
+            for name, source in self.exposed_bins.items()
+        }
+        fd, tmp = tempfile.mkstemp(prefix=path.name + ".", suffix=".tmp", dir=str(path.parent))
+        try:
+            with os.fdopen(fd, "w", encoding="utf-8") as f:
+                json.dump(data, f, indent=2)
+                f.write("\n")
+            os.replace(tmp, path)
+        except Exception:
+            Path(tmp).unlink(missing_ok=True)
+            raise
+        return path