cyvest 0.1.0__py3-none-any.whl → 5.1.3__py3-none-any.whl

cyvest/observable_registry.py

@@ -1,69 +0,0 @@
-from __future__ import annotations
-
-from collections.abc import Iterable
-from typing import Any
-
-from .models import Level, Observable
-
-
-class ObservableRegistry:
-    """Central store for observables shared across result checks."""
-
-    def __init__(self) -> None:
-        self._observables: dict[str, Observable] = {}
-
-    def get(self, full_key: str) -> Observable | None:
-        return self._observables.get(full_key)
-
-    def values(self) -> Iterable[Observable]:
-        return self._observables.values()
-
-    def upsert(self, observable: Observable) -> Observable:
-        stored = self._observables.get(observable.full_key)
-        if stored:
-            stored.update(observable)
-            target = stored
-        else:
-            self._observables[observable.full_key] = observable
-            target = observable
-        self._propagate_whitelist(target)
-        return target
-
-    def mark_whitelisted(self, observable: Observable) -> None:
-        observable.whitelisted = True
-        self._propagate_whitelist(observable)
-
-    def _propagate_whitelist(self, observable: Observable) -> None:
-        if not observable.whitelisted:
-            for intel in observable.threat_intels.values():
-                extra = intel.extra or {}
-                if extra.get("whitelisted") is True:
-                    observable.whitelisted = True
-                    break
-                if extra.get("warning_lists"):
-                    observable.whitelisted = True
-                    break
-                tags = extra.get("tags") or extra.get("labels")
-                if isinstance(tags, str):
-                    tags_iterable: Iterable[Any] = [tags]
-                elif isinstance(tags, Iterable):
-                    tags_iterable = tags
-                else:
-                    tags_iterable = []
-                if any(str(tag).lower() in {"allow", "trusted", "whitelist"} for tag in tags_iterable):
-                    observable.whitelisted = True
-                    break
-                if intel.level == Level.TRUSTED:
-                    observable.whitelisted = True
-                    break
-        if observable.whitelisted:
-            for parent in observable.observables_parents.values():
-                parent.whitelisted = True
-
-    def root_observables(self) -> list[Observable]:
-        return [obs for obs in self._observables.values() if not obs.observables_parents]
-
-    def whitelisted(self, obs_type: str, obs_value: str) -> bool:
-        key = f"{obs_type}.{obs_value}"
-        observable = self._observables.get(key)
-        return bool(observable and observable.whitelisted)

cyvest/report_render.py

@@ -1,306 +0,0 @@
-from __future__ import annotations
-
-import math
-import re
-from collections.abc import Iterable
-from typing import TYPE_CHECKING, Any
-
-from logurich import logger
-from rich.align import Align
-from rich.rule import Rule
-from rich.table import Table
-from rich.tree import Tree
-
-from .models import get_color_level, get_color_score
-
-if TYPE_CHECKING:
-    from .visitors import Report
-
-
-def _log_rich(level: str, renderable: Any) -> None:
-    logger.rich(level, renderable)
-
-
-def _to_stdout_check(check: dict[str, Any], *, indent: int = 0, short_name: bool = False) -> list[str] | None:
-    level = check["level"]
-    if level == "NONE":
-        return None
-    score = check["score"]
-    color_level = get_color_level(level)
-    color_score = get_color_score(score)
-    space = indent * " "
-    name = check["short_key"] if short_name else check["full_key"]
-    return [f"{space}{name}", f"[{color_score}]{score}[/{color_score}]", f"[{color_level}]{level}[/{color_level}]"]
-
-
-def _to_stdout_rec_checks(contained: dict[str, Any], *, indent: int = 0) -> list[list[str]]:
-    rows: list[list[str]] = []
-    for value in contained.values():
-        values = value if isinstance(value, list) else [value]
-        for entry in values:
-            is_container = "container" in entry
-            row = _to_stdout_check(entry, indent=indent, short_name=not is_container)
-            if row:
-                rows.append(row)
-            if is_container:
-                rows.extend(_to_stdout_rec_checks(entry["container"], indent=indent + 1))
-    return rows
-
-
-def _to_stdout_rec_obs(tree: Tree, obs: dict[str, Any], *, indent: int = 0) -> None:
-    level = obs["level"]
-    score = obs["score"]
-    generates_by_checks = (
-        "[cyan][[/cyan]" + "[cyan]][/cyan],[cyan][[/cyan]".join(obs["generated_by"]) + "[cyan]][/cyan]"
-        if obs["generated_by"]
-        else ""
-    )
-    color_level = get_color_level(level)
-    color_score = get_color_score(score)
-    more_detail = " [green]WHITELISTED[/green]" if obs["whitelisted"] is True else ""
-    full_key = obs["full_key"]
-    data = (
-        f"{generates_by_checks} {full_key} -> "
-        f"[{color_score}]{score}[/{color_score}] [{color_level}]{level}[/{color_level}] {more_detail}"
-    )
-    child_tree = tree.add(data)
-    for child in obs["observables_children"]:
-        _to_stdout_rec_obs(child_tree, child, indent=indent + 1)
-
-
-def _get_check(full_key: str, json_data: dict[str, Any]) -> dict[str, Any] | None:
-    root = json_data["checks"]
-    match = re.match(r"^([^\.]+)\.(.*)", full_key)
-    if match is None:
-        return None
-    scope = match.group(1)
-    root = root.get(scope, {})
-    current_path = match.group(2)
-    while current_path:
-        match = re.match(r"^(([^\#\.]+)(\#[^\#]+\#)?)\.?(.*)", current_path)
-        if not match:
-            logger.error("Impossible to match full key pattern {}", full_key)
-            return None
-        current_path = match.group(2)
-        if match.group(3):
-            identifier = match.group(3).strip("#")
-            root = root.get(current_path, [])
-            root = next((x for x in root if x["identifier"] == identifier), None)
-            if root is None:
-                return None
-        else:
-            root = root.get(current_path, {})
-        if "container" in root:
-            root = root.get("container", {})
-        current_path = match.group(4)
-    return root
-
-
-def stdout_from_json(report: Report, json_data: dict[str, Any]) -> None:
-    logger.info("[cyan]### JSON CONSOLE REPORT[/cyan]")
-    checks = json_data["stats_checks"]["checks"]
-    applied = json_data["stats_checks"]["applied"]
-    table_report = Table(
-        title="Report",
-        caption=f"RESULT CHECKS: {checks} - APPLIED: {applied}",
-    )
-    table_report.add_column("Name")
-    table_report.add_column("Score", justify="right")
-    table_report.add_column("Level", justify="center")
-
-    rule = Rule("[bold magenta]CHECKS[/bold magenta]")
-    table_report.add_row(rule, "-", "-")
-    for scope_name, checks in json_data["checks"].items():
-        scope_rule = Align(f"[bold magenta]{scope_name}[/bold magenta]", align="left")
-        table_report.add_row(scope_rule, "-", "-")
-        rows = _to_stdout_rec_checks(checks, indent=1)
-        for row in rows:
-            table_report.add_row(*row)
-
-    if report.graph:
-        tree = Tree("Observables Graph")
-        logger.info("[magenta]GRAPH: {}[/magenta]", len(json_data["graph"]))
-        for obs in json_data["graph"]:
-            _to_stdout_rec_obs(tree, obs)
-        _log_rich("INFO", tree)
-
-    table_report.add_section()
-    rule = Rule("[bold magenta]BY LEVEL[/bold magenta]")
-    table_report.add_row(rule, "-", "-")
-    for level_name, list_checks in json_data["checks_by_level"].items():
-        color_level = get_color_level(level_name)
-        level_rule = Align(
-            f"[bold {color_level}]{level_name}: {len(list_checks)} check(s)[/bold {color_level}]",
-            align="center",
-        )
-        table_report.add_row(level_rule, "-", "-")
-        for check_full_key in list_checks:
-            check = _get_check(check_full_key, json_data)
-            if check:
-                row = _to_stdout_check(check)
-                if row:
-                    table_report.add_row(*row)
-
-    table_report.add_section()
-    enrichment_rule = Rule(f"[bold magenta]ENRICHMENTS[/bold magenta]: {len(report.enrichments)} enrichments")
-    table_report.add_row(enrichment_rule, "-", "-")
-    for enrichment in report.enrichments:
-        table_report.add_row(str(enrichment), "-", "-")
-
-    table_report.add_section()
-    stats_rule = Rule("[bold magenta]STATISTICS[/bold magenta]")
-    table_report.add_row(stats_rule, "-", "-")
-    for stat_name, stat_value in json_data["stats"].items():
-        table_report.add_row(" ".join(stat_name.split("_")).title(), str(stat_value), "-")
-
-    global_level = json_data["level"]
-    global_score = json_data["score"]
-    color_level = get_color_level(global_level)
-    color_score = get_color_score(global_score)
-    table_report.add_section()
-    table_report.add_row(
-        Align("[bold]GLOBAL SCORE[/bold]", align="center"),
-        f"[{color_score}]{global_score}[/{color_score}]",
-        f"[{color_level}]{global_level}[/{color_level}]",
-    )
-    _log_rich("INFO", table_report)
-
-
-def markdown_summary(json_data: dict[str, Any], *, exclude_checks: list[str] | None = None) -> str:
-    def _first(keys: Iterable[str], src: dict[str, Any], default: Any | None = None) -> Any | None:
-        for key in keys:
-            if key in src and src[key] not in (None, ""):
-                return src[key]
-        return default
-
-    def _result_of(check: dict[str, Any]) -> str:
-        value = _first(("result", "status", "outcome"), check)
-        if value is not None:
-            return str(value)
-        if isinstance(check.get("passed"), bool):
-            return "PASS" if check["passed"] else "FAIL"
-        if isinstance(check.get("ok"), bool):
-            return "OK" if check["ok"] else "NOT OK"
-        return "n/a"
-
-    def _score_of(check: dict[str, Any]) -> str:
-        score_value = check.get("score")
-        if score_value is None:
-            return "n/a"
-        if isinstance(score_value, int):
-            return str(score_value)
-        if isinstance(score_value, float):
-            return str(int(score_value)) if score_value.is_integer() else f"{score_value:.2f}"
-        return str(score_value)
-
-    def _level_of(check: dict[str, Any]) -> str:
-        level_value = _first(("level", "severity"), check, default="UNKNOWN")
-        return str(level_value).upper()
-
-    def _name_of(check: dict[str, Any], fallback_key: str) -> str:
-        return str(_first(("name", "title", "id"), check, default=fallback_key))
-
-    def _desc_of(check: dict[str, Any]) -> str:
-        desc_value = _first(("description", "desc", "details", "message"), check)
-        return str(desc_value).strip() if desc_value is not None else "_No description provided._"
-
-    def _is_check_node(node: Any) -> bool:
-        if not isinstance(node, dict):
-            return False
-        keys = ("level", "severity", "description", "result", "status", "name", "score", "passed", "ok")
-        return any(key in node for key in keys)
-
-    def _flatten_checks(
-        container: Any, path: tuple[str, ...] = ()
-    ) -> Iterable[tuple[str, dict[str, Any], tuple[str, ...]]]:
-        if isinstance(container, dict):
-            if _is_check_node(container):
-                key = path[-1] if path else "check"
-                yield (key, container, path)
-            else:
-                for key, value in container.items():
-                    yield from _flatten_checks(value, path + (str(key),))
-        elif isinstance(container, list):
-            for idx, value in enumerate(container):
-                yield from _flatten_checks(value, path + (str(idx),))
-
-    def _order_levels(levels: Iterable[str]) -> list[str]:
-        priority: dict[str, int] = {
-            "MALICIOUS": 0,
-            "SUSPICIOUS": 1,
-            "NOTABLE": 2,
-            "INFO": 3,
-            "SAFE": 4,
-        }
-        return sorted(set(levels), key=lambda level: priority.get(level, 999))
-
-    global_level = str(json_data.get("level", "UNKNOWN")).upper()
-    global_score = json_data.get("score", "n/a")
-    if isinstance(global_score, (int, float)):
-        global_score_str = (
-            str(int(global_score))
-            if isinstance(global_score, int) or (isinstance(global_score, float) and float(global_score).is_integer())
-            else str(global_score)
-        )
-    else:
-        global_score_str = str(global_score)
-
-    raw_checks_root = json_data.get("checks") or {}
-    flattened = list(_flatten_checks(raw_checks_root))
-
-    groups: dict[str, list[tuple[str, dict[str, Any], tuple[str, ...]]]] = {}
-    for key, check, path in flattened:
-        level = _level_of(check)
-        groups.setdefault(level, []).append((key, check, path))
-
-    ordered_levels = _order_levels(groups.keys())
-    md: list[str] = []
-    md.append("# Report Summary")
-    md.append("")
-    md.append(f"**Global Score:** `{global_score_str}`  |  **Global Level:** {global_level}")
-    md.append("")
-
-    if not flattened:
-        md.append("_No checks found in the report._")
-    else:
-        for level in ordered_levels:
-            entries = groups[level]
-            md.append(f"## {level} — {len(entries)} check(s)")
-            md.append("")
-
-            def _score_key(entry: tuple[str, dict[str, Any], tuple[str, ...]]) -> float:
-                score_raw = entry[1].get("score")
-                try:
-                    return -float(score_raw) if score_raw is not None else math.inf
-                except Exception:
-                    return math.inf
-
-            entries_sorted = sorted(
-                entries,
-                key=lambda entry: (_score_key(entry), _name_of(entry[1], entry[0]).lower()),
-            )
-
-            for key, check, path in entries_sorted:
-                name = _name_of(check, key)
-                if exclude_checks and name in exclude_checks:
-                    continue
-                result = _result_of(check)
-                score_value = _score_of(check)
-                desc = _desc_of(check)
-                scope = " › ".join(path[:-1]) if len(path) > 1 else (path[0] if path else "")
-                scope_note = f" _(scope: `{scope}`)_" if scope else ""
-                comment = None
-                details = check.get("details", {})
-                if details:
-                    comment = details.get("comment")
-
-                md.append(f"- **{name}**{scope_note}")
-                md.append(f"  - Level: {_level_of(check)}")
-                md.append(f"  - Result: `{result}`")
-                md.append(f"  - Score: `{score_value}`")
-                md.append(f"  - Description: {desc}")
-                if comment:
-                    md.append(f"  - Note: {comment}")
-            md.append("")
-
-    return "\n".join(md)

cyvest/report_serialization.py

@@ -1,237 +0,0 @@
-from __future__ import annotations
-
-from collections import defaultdict
-from typing import TYPE_CHECKING, Any
-
-from .models import Container, Level, Observable, ResultCheck, ThreatIntel
-
-if TYPE_CHECKING:
-    from .visitors import Report
-
-
-def _to_json_threat_intel(ti: ThreatIntel) -> dict[str, Any]:
-    return {
-        "name": ti.name,
-        "analyzer": ti.name,
-        "display_name": ti.display_name,
-        "score": ti.score,
-        "level": ti.level.name,
-        "comment": ti.comment,
-        "extra": ti.extra,
-        "taxonomies": ti.taxonomies,
-    }
-
-
-def _observable_to_json(
-    report: Report,
-    observable: Observable,
-    *,
-    max_deep_child: int | None,
-    max_deep_parent: int | None,
-) -> dict[str, Any]:
-    def _rec(
-        obj: Observable,
-        deep_child: int | None,
-        deep_parent: int | None,
-        already_seen: list[str] | None = None,
-    ) -> dict[str, Any]:
-        if already_seen is None:
-            already_seen = []
-        threat_intel_json = {k: _to_json_threat_intel(v) for k, v in obj.threat_intels.items()}
-        if obj.full_key in already_seen:
-            return {
-                "full_key": obj.full_key,
-                "score": obj.score,
-                "level": obj.level.name,
-                "obs_type": obj.obs_type.name,
-                "obs_value": obj.obs_value,
-                "whitelisted": obj.whitelisted,
-                "threat_intels": threat_intel_json,
-                "observables_children": [child for child in obj.observables_children],
-                "observables_parents": [parent for parent in obj.observables_parents],
-                "generated_by": list(obj.generated_by),
-            }
-        new_seen = [*already_seen, obj.full_key]
-        if deep_child is not None:
-            if (max_deep_child is not None and deep_child >= max_deep_child) or deep_child < 0:
-                observables_children: list[Any] = [child for child in obj.observables_children]
-            else:
-                rec_deep_child = deep_child + 1 if deep_child is not None else None
-                observables_children = [
-                    _rec(child, rec_deep_child, -1, new_seen) for child in obj.observables_children.values()
-                ]
-        else:
-            observables_children = [_rec(child, None, -1, new_seen) for child in obj.observables_children.values()]
-        if deep_parent is not None:
-            if (max_deep_parent is not None and deep_parent >= max_deep_parent) or deep_parent < 0:
-                observables_parents: list[Any] = [parent for parent in obj.observables_parents]
-            else:
-                rec_deep_parent = deep_parent + 1 if deep_parent is not None else None
-                observables_parents = [
-                    _rec(parent, -1, rec_deep_parent, new_seen) for parent in obj.observables_parents.values()
-                ]
-        else:
-            observables_parents = [_rec(parent, -1, None, new_seen) for parent in obj.observables_parents.values()]
-        return {
-            "full_key": obj.full_key,
-            "score": obj.score,
-            "level": obj.level.name,
-            "obs_type": obj.obs_type.name,
-            "obs_value": obj.obs_value,
-            "whitelisted": obj.whitelisted,
-            "threat_intels": threat_intel_json,
-            "observables_children": observables_children,
-            "observables_parents": observables_parents,
-            "generated_by": list(obj.generated_by),
-        }
-
-    if max_deep_child is None:
-        start_child = None
-    else:
-        start_child = -1 if max_deep_child < 0 else 0
-    if max_deep_parent is None:
-        start_parent = None
-    else:
-        start_parent = -1 if max_deep_parent < 0 else 0
-    return _rec(observable, start_child, start_parent)
-
-
-def _result_check_to_json(report: Report, rc: ResultCheck) -> dict[str, Any]:
-    observables_json = [
-        _observable_to_json(report, observable, max_deep_child=2, max_deep_parent=1)
-        for observable in rc.observables.values()
-    ]
-    return {
-        "full_key": rc.full_key,
-        "short_key": rc.short_key,
-        "scope": rc.scope.name if rc.scope else "UNKNOWN",
-        "path": rc.path,
-        "identifier": rc.identifier,
-        "score": rc.score,
-        "level": rc.level.name,
-        "description": rc.description,
-        "details": rc.details,
-        "observables": observables_json,
-    }
-
-
-def _container_to_json(report: Report, container: Container) -> dict[str, Any]:
-    payload = {
-        "full_key": container.full_key,
-        "scope": container.scope.name if container.scope else "UNKNOWN",
-        "path": container.path,
-        "identifier": container.identifier,
-        "score": container.score,
-        "level": container.level.name,
-        "description": container.description,
-        "details": container.details,
-        "nb_checks": container.nb_checks,
-        "container": {},
-    }
-    for child in container.children:
-        _insert_node(report, payload["container"], child)
-    return payload
-
-
-def _insert_node(report: Report, target: dict[str, Any], node: Container | ResultCheck) -> None:
-    if isinstance(node, Container):
-        serialized = _container_to_json(report, node)
-    else:
-        serialized = _result_check_to_json(report, node)
-    if node.identifier:
-        bucket = target.setdefault(node.path, [])
-        bucket.append(serialized)
-    else:
-        target[node.path] = serialized
-
-
-def report_to_json(report: Report) -> dict[str, Any]:
-    data = dict(report.json)
-    data.pop("@meta", None)
-
-    checks_root: dict[str, Any] = {}
-    for scope_name, nodes in report.tree.iter_roots():
-        scope_dict: dict[str, Any] = {}
-        for node in nodes:
-            _insert_node(report, scope_dict, node)
-        checks_root[scope_name] = scope_dict
-
-    result_checks = list(report.tree.result_checks())
-    applied_checks = [rc for rc in result_checks if rc.level > Level.NONE]
-
-    checks_by_level: dict[str, list[str]] = {}
-    grouped = defaultdict(list)
-    for rc in result_checks:
-        grouped[rc.level.name].append((rc.score, rc.full_key))
-    for level_name, entries in grouped.items():
-        entries.sort(key=lambda item: item[0], reverse=True)
-        checks_by_level[level_name] = [full_key for _, full_key in entries]
-
-    json_graph = [
-        _observable_to_json(report, observable, max_deep_child=None, max_deep_parent=-1)
-        for observable in report.observable_registry.root_observables()
-    ]
-
-    return {
-        "score": report.global_score,
-        "level": report.global_level.name,
-        "stats": report.reduce_stats({}),
-        "stats_checks": {
-            "applied": len(applied_checks),
-            "checks": len(result_checks),
-            "obs": len(report.observables),
-            "enrichments": len(report.enrichments),
-        },
-        "checks_by_level": checks_by_level,
-        "checks": checks_root,
-        "graph": json_graph,
-        "data": data,
-        "annotations": report.annotations,
-    }
-
-
-def reduce_report_json(json_report: dict[str, Any]) -> dict[str, Any]:
-    checks_root = json_report.get("checks", {})
-    flattened_checks: dict[str, Any] = {}
-    if isinstance(checks_root, dict):
-        _reduce_report_rec(checks_root, delete_none=True, all_checks=flattened_checks)
-    json_report["checks"] = flattened_checks
-    json_report.pop("graph", None)
-    json_report.pop("html_report", None)
-    return json_report
-
-
-def _reduce_report_rec(container, delete_none=False, all_checks=None):
-    if all_checks is None:
-        all_checks = {}
-    to_remove = []
-    for key, value in container.items():
-        if isinstance(value, list):
-            to_remove_in_list = []
-            for data in value:
-                if data["level"] == "NONE" and delete_none is True:
-                    to_remove_in_list.append(key)
-                elif "container" in data:
-                    _reduce_report_rec(data["container"], delete_none=delete_none, all_checks=all_checks)
-                else:
-                    all_checks[data["full_key"]] = data
-                    for obs in data["observables"]:
-                        obs["observables_children"] = [
-                            child_obs["full_key"] for child_obs in obs["observables_children"]
-                        ]
-                        obs["observables_parents"] = [
-                            parent_obs["full_key"] for parent_obs in obs["observables_parents"]
-                        ]
-                        for ti in obs["threat_intels"].values():
-                            ti.pop("extra", None)
-            for entry in to_remove_in_list:
-                value.remove(entry)
-        else:
-            if value["level"] == "NONE" and delete_none is True:
-                to_remove.append(key)
-            elif "container" in value:
-                _reduce_report_rec(value["container"], delete_none=delete_none, all_checks=all_checks)
-            else:
-                all_checks[value["full_key"]] = value
-    for key in to_remove:
-        del container[key]