trustsight 0.1.0__py3-none-any.whl

trustsight/analysis.py

@@ -0,0 +1,178 @@
+import json
+
+from .buckets import classify_urls
+from .config import ensure_default_configs, load_config
+from .db import (
+    get_last_analysis,
+    init_db,
+    insert_analysis,
+    update_package_version,
+    upsert_package,
+)
+from .differ import extract_urls_from_diff, generate_diff
+from .discovery import find_outdated_packages, get_aur_latest_versions, get_installed_aur_packages
+from .fetcher import clone_or_fetch, get_head_commit, get_maintainer_from_commit, get_pkgver_from_head
+from .llm import generate_verdict
+from .novelty import build_novelty_context
+from .rules import apply_rules, get_raw_diff_lines
+from .scoring import calculate_score
+from .llm import fallback_verdict
+from .schema import (
+    DiffSummary,
+    ExecutionChanges,
+    PackageFact,
+    fact_to_dict,
+)
+from .tokenizer import tokenize_and_resolve
+
+
+def analyze_package(pkg_name: str, old_commit: str = "", new_version: str = "") -> PackageFact:
+    ensure_default_configs()
+    init_db()
+    config = load_config()
+
+    repo = clone_or_fetch(pkg_name)
+    head_commit = get_head_commit(repo)
+    head_version = get_pkgver_from_head(repo) or new_version
+
+    if not head_version:
+        head_version = new_version
+
+    package_id = upsert_package(pkg_name, head_version)
+
+    if not old_commit:
+        last = get_last_analysis(package_id)
+        if last and last.get("new_commit"):
+            old_commit = last["new_commit"]
+        else:
+            return _make_fresh_analysis(pkg_name, head_version, head_commit, package_id, repo, config)
+
+    diff_text, diff_summary = generate_diff(repo, old_commit, head_commit, config.get("diff", {}).get("max_context_lines", 3))
+    source_changes = extract_urls_from_diff(diff_text)
+
+    old_maintainer = get_maintainer_from_commit(repo, old_commit) or ""
+    new_maintainer = get_maintainer_from_commit(repo, head_commit) or ""
+    maintainer_changed = bool(old_maintainer and new_maintainer and old_maintainer != new_maintainer)
+
+    novelty = build_novelty_context(
+        source_changes.added_urls,
+        package_id,
+        maintainer=new_maintainer,
+    )
+
+    source_buckets = classify_urls(source_changes.added_urls)
+
+    resolved_strings, unresolved_strings = tokenize_and_resolve(diff_text)
+    raw_lines = get_raw_diff_lines(diff_text)
+
+    triggered_rules = apply_rules(resolved_strings, raw_lines)
+    rule_ids = [r["rule_id"] for r in triggered_rules]
+
+    checksum_map = {
+        "changed_from_sha256_to_skip": ("R012", "Checksum changed to SKIP", "HIGH"),
+        "checksum_array_emptied": ("R013", "Checksum array emptied", "HIGH"),
+    }
+    cs_behavior = source_changes.checksum_behavior
+    if cs_behavior in checksum_map:
+        rid, rname, rsev = checksum_map[cs_behavior]
+        triggered_rules.append({
+            "rule_id": rid,
+            "name": rname,
+            "severity": rsev,
+            "category": "integrity",
+            "match": cs_behavior,
+        })
+        rule_ids.append(rid)
+
+    score, breakdown, risk = calculate_score(triggered_rules, source_buckets, novelty, config)
+
+    fact = PackageFact(
+        package_name=pkg_name,
+        old_version="",
+        new_version=head_version,
+        old_commit=old_commit,
+        new_commit=head_commit,
+        maintainer_changed=maintainer_changed,
+        previous_maintainer=old_maintainer,
+        current_maintainer=new_maintainer,
+        diff_summary=diff_summary,
+        source_changes=source_changes,
+        source_buckets=source_buckets,
+        execution_changes=ExecutionChanges(
+            resolved_commands=resolved_strings,
+            suspicious_patterns_detected=rule_ids,
+            unresolved_patterns=unresolved_strings,
+        ),
+        novelty_context=novelty,
+        score_breakdown=breakdown,
+        final_score=score,
+    )
+
+    insert_analysis(
+        package_id=package_id,
+        old_version="",
+        new_version=head_version,
+        old_commit=old_commit,
+        new_commit=head_commit,
+        final_score=score,
+        raw_diff=diff_text,
+        fact_json=json.dumps(fact_to_dict(fact)),
+        triggered_rules=triggered_rules,
+    )
+
+    update_package_version(pkg_name, head_version)
+    return fact
+
+
+def _make_fresh_analysis(
+    pkg_name: str, version: str, commit: str, package_id: int, repo, config: dict
+) -> PackageFact:
+    novelty = build_novelty_context([], package_id)
+    fact = PackageFact(
+        package_name=pkg_name,
+        new_version=version,
+        new_commit=commit,
+        diff_summary=DiffSummary(),
+        novelty_context=novelty,
+        final_score=0,
+    )
+    insert_analysis(
+        package_id=package_id,
+        old_version="",
+        new_version=version,
+        old_commit="",
+        new_commit=commit,
+        final_score=0,
+        raw_diff="",
+        fact_json=json.dumps(fact_to_dict(fact)),
+        triggered_rules=[],
+    )
+    update_package_version(pkg_name, version)
+    return fact
+
+
+def discover_updates(limit: int = 20) -> list[dict]:
+    ensure_default_configs()
+    init_db()
+
+    installed = get_installed_aur_packages()
+    names = list(installed.keys())
+    latest = get_aur_latest_versions(names)
+    outdated = find_outdated_packages(installed, latest)
+
+    results = []
+    for name, (old_ver, new_ver) in list(outdated.items())[:limit]:
+        fact = analyze_package(name)
+        verdict = generate_verdict(fact) if fact.final_score > 0 else fallback_verdict(fact)
+        results.append(
+            {
+                "package": name,
+                "old_version": old_ver,
+                "new_version": new_ver,
+                "score": fact.final_score,
+                "verdict": verdict,
+                "risk": "Low" if fact.final_score <= 20 else "Medium" if fact.final_score <= 50 else "High" if fact.final_score <= 80 else "Critical",
+            }
+        )
+
+    return results

trustsight/buckets.py

@@ -0,0 +1,59 @@
+import unicodedata
+from urllib.parse import urlparse
+
+import tldextract
+
+from .config import load_domains
+
+CONFUSABLES = {
+    "g": "ɡ", "a": "а", "e": "е", "o": "о", "c": "с",
+    "p": "р", "x": "х", "y": "у", "i": "і", "l": "ӏ",
+}
+
+
+def has_homograph(domain: str) -> bool:
+    for ch in domain:
+        if unicodedata.name(ch, "").startswith("LATIN") and ord(ch) > 127:
+            return True
+    return False
+
+
+def classify_url(url: str, domain_config: dict | None = None) -> tuple[str, str]:
+    if domain_config is None:
+        domain_config = load_domains()
+
+    parsed = urlparse(url)
+    domain = parsed.netloc.lower()
+
+    if has_homograph(domain):
+        return "homograph_attack", domain
+
+    extracted = tldextract.extract(url)
+    registered = f"{extracted.domain}.{extracted.suffix}"
+
+    raw_hosting = domain_config.get("raw_hosting", {}).get("domains", [])
+    for d in raw_hosting:
+        if domain == d:
+            return "raw_hosting", domain
+
+    trusted_forges = domain_config.get("trusted_forges", {}).get("domains", [])
+    for d in trusted_forges:
+        if registered == d or domain.endswith("." + d):
+            return "trusted_forge", registered
+
+    official = domain_config.get("official_projects", {}).get("domains", [])
+    for d in official:
+        if domain == d or domain.endswith("." + d):
+            return "official", domain
+
+    return "unknown", domain
+
+
+def classify_urls(
+    urls: list[str], domain_config: dict | None = None
+) -> dict[str, str]:
+    result = {}
+    for url in urls:
+        bucket, matched_domain = classify_url(url, domain_config)
+        result[url] = bucket
+    return result

trustsight/cli.py

@@ -0,0 +1,224 @@
+import argparse
+import sys
+
+from .analysis import analyze_package, discover_updates
+from .config import ensure_default_configs, load_config, set_config, CONFIG_DIR
+from .db import get_history, get_package_id, get_triggered_rules, init_db
+from .scoring import risk_level
+
+RISK_COLORS = {"Low": "green", "Medium": "yellow", "High": "red", "Critical": "bold red"}
+
+try:
+    from rich.console import Console
+    from rich.table import Table
+    from rich.text import Text
+
+    HAS_RICH = True
+except ImportError:
+    HAS_RICH = False
+
+
+def cmd_review(args):
+    ensure_default_configs()
+    config = load_config()
+    limit = args.limit or config.get("limits", {}).get("default_review_limit", 20)
+
+    results = discover_updates(limit=limit)
+
+    if not results:
+        print("No outdated AUR packages found.")
+        return
+
+    if HAS_RICH:
+        console = Console()
+        table = Table(title="TrustSight Review")
+        table.add_column("Package", style="cyan")
+        table.add_column("Score", justify="right")
+        table.add_column("Verdict")
+
+        for r in results:
+            score_text = Text(f"{r['score']}/100", style=RISK_COLORS.get(r["risk"], "white"))
+            table.add_row(r["package"], score_text, r["verdict"])
+
+        console.print(table)
+    else:
+        print(f"{'Package':<20} {'Score':<10} Verdict")
+        print("-" * 80)
+        for r in results:
+            print(f"{r['package']:<20} {r['score']:<10} {r['verdict']}")
+
+
+def cmd_inspect(args):
+    ensure_default_configs()
+    init_db()
+
+    fact = analyze_package(args.package)
+
+    if HAS_RICH:
+        console = Console()
+        console.print(f"\n[bold cyan]TrustSight Inspect: {fact.package_name}[/]")
+        console.print(f"  Version: {fact.old_version} → {fact.new_version}")
+        console.print(f"  Score: {fact.final_score}/100 [bold {RISK_COLORS.get(risk_level(fact.final_score), 'white')}]({risk_level(fact.final_score)})[/]")
+
+        if fact.maintainer_changed:
+            console.print(f"  [yellow]Maintainer changed: {fact.previous_maintainer} → {fact.current_maintainer}[/]")
+
+        console.print("\n  [underline]Diff Summary[/]")
+        console.print(f"  Files changed: {', '.join(fact.diff_summary.files_changed) or 'none'}")
+        console.print(f"  Lines: +{fact.diff_summary.lines_added}/-{fact.diff_summary.lines_removed}")
+
+        cs_behavior = fact.source_changes.checksum_behavior
+        if cs_behavior and cs_behavior != "unchanged":
+            console.print(f"\n  [yellow]Checksum behavior: {cs_behavior}[/]")
+
+        if fact.source_changes.added_urls:
+            console.print("\n  [underline]Source URLs Added[/]")
+            for url in fact.source_changes.added_urls:
+                bucket = fact.source_buckets.get(url, "unknown")
+                console.print(f"    {url} [dim]({bucket})[/]")
+
+        if fact.execution_changes.resolved_commands:
+            console.print("\n  [underline]Resolved Commands[/]")
+            for cmd in fact.execution_changes.resolved_commands:
+                console.print(f"    {cmd}")
+
+        if fact.score_breakdown:
+            console.print("\n  [underline]Score Breakdown[/]")
+            for entry in fact.score_breakdown:
+                sign = "+" if entry.weight >= 0 else ""
+                style = RISK_COLORS.get(entry.severity.capitalize(), "white")
+                console.print(f"  {sign}{entry.weight} [{style}]{entry.severity}[/]  {entry.rule_id}  {entry.reason[:80]}")
+
+        console.print("\n  [underline]Verdict[/]")
+        from .llm import fallback_verdict
+        console.print(f"  {fallback_verdict(fact)}")
+    else:
+        print(f"TrustSight Inspect: {fact.package_name}")
+        print(f"  Version: {fact.old_version} -> {fact.new_version}")
+        print(f"  Score: {fact.final_score}/100 ({risk_level(fact.final_score)})")
+        if fact.source_changes.checksum_behavior and fact.source_changes.checksum_behavior != "unchanged":
+            print(f"  Checksum: {fact.source_changes.checksum_behavior}")
+        if fact.source_changes.added_urls:
+            print("  Source URLs Added:")
+            for url in fact.source_changes.added_urls:
+                bucket = fact.source_buckets.get(url, "unknown")
+                print(f"    {url} ({bucket})")
+
+
+def cmd_history(args):
+    ensure_default_configs()
+    init_db()
+
+    pkg_id = get_package_id(args.package)
+    if pkg_id is None:
+        print(f"Package '{args.package}' not found in history.")
+        return
+
+    history = get_history(pkg_id, limit=args.limit or 20)
+
+    if not history:
+        print(f"No analysis history for '{args.package}'.")
+        return
+
+    if HAS_RICH:
+        console = Console()
+        table = Table(title=f"History: {args.package}")
+        table.add_column("Date", style="dim")
+        table.add_column("Old", justify="right")
+        table.add_column("→ New", justify="right")
+        table.add_column("Score", justify="right")
+        table.add_column("Risk")
+
+        for h in history:
+            ts = h.get("timestamp", "")[:10] if h.get("timestamp") else ""
+            score = h.get("final_score", 0)
+            risk = risk_level(score)
+            score_text = Text(f"{score}/100", style=RISK_COLORS.get(risk, "white"))
+            table.add_row(
+                ts,
+                h.get("old_version", "") or "",
+                h.get("new_version", "") or "",
+                score_text,
+                risk,
+            )
+
+        console.print(table)
+
+        if args.score_breakdown and history:
+            latest = history[0]
+            rules = get_triggered_rules(latest["id"])
+            if rules:
+                console.print("\n  [underline]Latest Score Breakdown[/]")
+                for r in rules:
+                    sign = "+"
+                    console.print(f"  {sign}{r['severity']:<10} {r['rule_id']}")
+    else:
+        for h in history:
+            print(f"{h.get('timestamp','')[:10]:<12} {str(h.get('old_version','')):<12} → {str(h.get('new_version','')):<12} Score: {h.get('final_score',0)}")
+
+
+def cmd_config(args):
+    ensure_default_configs()
+    if args.action == "show":
+        cfg = load_config()
+        print(f"Config file: {CONFIG_DIR / 'config.toml'}")
+        print()
+        llm = cfg.get("llm", {})
+        provider = llm.get("provider", "ollama")
+        model = llm.get("model", "gpt-4o-mini")
+        print(f"  provider: {provider}")
+        print(f"  model:    {model}")
+        openai_cfg = llm.get("openai", {})
+        api_key = openai_cfg.get("api_key", "")
+        base_url = openai_cfg.get("base_url", "https://api.openai.com/v1")
+        mask = api_key[:4] + "..." if len(api_key) > 8 else "(not set)"
+        print(f"  api_key:  {mask}")
+        print(f"  base_url: {base_url}")
+    elif args.action == "set":
+        if args.key in ("api_key", "base_url"):
+            set_config(f"llm.openai.{args.key}", args.value)
+            print(f"Set llm.openai.{args.key} in {CONFIG_DIR / 'config.toml'}")
+        else:
+            print(f"Unknown key: {args.key}. Use api_key or base_url.")
+            sys.exit(1)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="TrustSight - AUR Package Update Vetting Tool")
+    sub = parser.add_subparsers(dest="command")
+
+    p_review = sub.add_parser("review", help="Scan AUR and analyze outdated packages")
+    p_review.add_argument("--limit", type=int, default=0, help="Max packages to review")
+
+    p_inspect = sub.add_parser("inspect", help="Analyze a specific package")
+    p_inspect.add_argument("package", help="Package name")
+
+    p_history = sub.add_parser("history", help="Show analysis history for a package")
+    p_history.add_argument("package", help="Package name")
+    p_history.add_argument("--limit", type=int, default=20, help="Max history entries")
+    p_history.add_argument("--score-breakdown", action="store_true", help="Show score breakdown")
+
+    p_config = sub.add_parser("config", help="Manage configuration")
+    p_config_sub = p_config.add_subparsers(dest="action")
+
+    p_config_sub.add_parser("show", help="Show current configuration")
+    p_config_set = p_config_sub.add_parser("set", help="Set a configuration value")
+    p_config_set.add_argument("key", choices=["api_key", "base_url"], help="Config key")
+    p_config_set.add_argument("value", help="Config value")
+
+    args = parser.parse_args()
+
+    if args.command == "review":
+        cmd_review(args)
+    elif args.command == "inspect":
+        cmd_inspect(args)
+    elif args.command == "history":
+        cmd_history(args)
+    elif args.command == "config":
+        cmd_config(args)
+    else:
+        parser.print_help()
+
+
+if __name__ == "__main__":
+    main()