scanfuse 0.2.0__py3-none-any.whl

scanfuse/parsers/base.py

@@ -0,0 +1,142 @@
+import re
+from abc import ABC, abstractmethod
+from html.parser import HTMLParser
+
+from scanfuse.models import Finding
+
+
+class PartialImportError(ValueError):
+    """Input contains usable findings and errors that must be reported."""
+
+    def __init__(self, message: str, findings: list[Finding]) -> None:
+        super().__init__(message)
+        self.findings = findings
+
+
+_CVE = re.compile(r"CVE-\d{4}-\d{4,}", re.IGNORECASE)
+_CWE_TEXT = re.compile(r"\bCWE[-\s#:]*(\d{1,5})\b", re.IGNORECASE)
+_URL = re.compile(r"https?://[^\s<>\"'\\]+", re.IGNORECASE)
+
+
+class _TextExtractor(HTMLParser):
+    """Turns the small HTML subset that scanners put into text fields into plain text."""
+
+    _BLOCK = {"p", "div", "ul", "ol", "table", "tr", "h1", "h2", "h3", "h4", "h5", "h6", "pre", "blockquote"}
+
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.parts: list[str] = []
+        self._skip = 0
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        if tag in ("script", "style"):
+            self._skip += 1
+        elif tag == "br":
+            self.parts.append("\n")
+        elif tag == "li":
+            self.parts.append("\n- ")
+        elif tag in self._BLOCK:
+            self.parts.append("\n\n")
+
+    def handle_endtag(self, tag: str) -> None:
+        if tag in ("script", "style") and self._skip:
+            self._skip -= 1
+        elif tag in self._BLOCK or tag == "li":
+            self.parts.append("\n")
+
+    def handle_data(self, data: str) -> None:
+        if not self._skip:
+            self.parts.append(data)
+
+
+def html_to_text(value: object) -> str:
+    """Plain text from a field that may contain HTML tags or HTML entities.
+
+    Line breaks and list items are kept. Scripts are dropped. Text without markup is returned
+    with entities decoded and surrounding space removed.
+    """
+    text = "" if value is None else str(value)
+    if "<" not in text and "&" not in text:
+        return text.strip()
+    parser = _TextExtractor()
+    parser.feed(text)
+    parser.close()
+    result = "".join(parser.parts).replace("\xa0", " ")
+    result = re.sub(r"[ \t]+\n", "\n", result)
+    result = re.sub(r"\n{3,}", "\n\n", result)
+    return result.strip()
+
+
+def extract_urls(*values: object) -> list[str]:
+    """http(s) URLs found in strings, HTML or lists of them, without duplicates."""
+    found: list[str] = []
+    items: list[object] = []
+    for value in values:
+        items.extend(value if isinstance(value, (list, tuple)) else [value])
+    for item in items:
+        text = str(item or "")
+        for match in _URL.findall(text):
+            url = match.rstrip(".,;:)]}")
+            if url and url not in found:
+                found.append(url)
+    return found
+
+
+def cwe_ids_from_text(*values: object) -> list[str]:
+    """CWE ids named in free text, as 'CWE-79'."""
+    found: list[str] = []
+    for value in values:
+        for number in _CWE_TEXT.findall(str(value or "")):
+            cwe = f"CWE-{int(number)}"
+            if int(number) > 0 and cwe not in found:
+                found.append(cwe)
+    return found
+
+
+def clean_ids(*values: object) -> list[str]:
+    """Return unique, non-empty identifier strings in the order given."""
+    result: list[str] = []
+    for value in values:
+        if isinstance(value, (list, tuple)):
+            items = list(value)
+        else:
+            items = [value]
+        for item in items:
+            if item is None:
+                continue
+            text = str(item).strip()
+            if text and text not in result:
+                result.append(text)
+    return result
+
+
+def cve_ids(value: object) -> list[str]:
+    """Extract CVE ids from a string or list of strings."""
+    found: list[str] = []
+    items = value if isinstance(value, (list, tuple)) else [value]
+    for item in items:
+        for match in _CVE.findall(str(item or "")):
+            cve = match.upper()
+            if cve not in found:
+                found.append(cve)
+    return found
+
+
+def cwe_id(value: object) -> str | None:
+    """Return 'CWE-<n>' for a numeric or 'CWE-n' value, otherwise None."""
+    text = str(value or "").strip().upper().removeprefix("CWE-")
+    return f"CWE-{text}" if text.isdigit() and int(text) > 0 else None
+
+
+class BaseParser(ABC):
+    """
+    Abstract base class for all scanfuse parsers.
+    """
+
+    @abstractmethod
+    def parse(self, file_path: str) -> list[Finding]:
+        """
+        Parse the given file and return a list of Finding objects.
+        Must handle malformed files gracefully and raise ValueError if the format is fundamentally wrong.
+        """
+        pass

scanfuse/parsers/burp.py

@@ -0,0 +1,135 @@
+import base64
+import xml.etree.ElementTree as ET
+
+import defusedxml.ElementTree as defused_et
+
+from scanfuse.models import Evidence, Finding, Host
+from scanfuse.parsers.base import (
+    BaseParser,
+    clean_ids,
+    cve_ids,
+    cwe_ids_from_text,
+    extract_urls,
+    html_to_text,
+)
+
+
+class BurpParser(BaseParser):
+    def parse(self, file_path: str) -> list[Finding]:
+        findings = []
+        try:
+            context = iter(defused_et.iterparse(file_path, events=("start", "end")))
+
+            try:
+                event, root = next(context)
+            except StopIteration:
+                raise ValueError("Empty XML file")
+
+            if root.tag != "issues":
+                raise ValueError("Not a valid Burp XML file (missing <issues> root tag)")
+
+            for event, issue_elem in context:
+                if event == "end" and issue_elem.tag == "issue":
+                    name = html_to_text(issue_elem.findtext("name", default="")) or "Unknown Vulnerability"
+                    severity_raw = issue_elem.findtext("severity", default="Information")
+
+                    # Map Burp severity to Finding severity model
+                    severity_map = {
+                        "High": "High",
+                        "Medium": "Medium",
+                        "Low": "Low",
+                        "Information": "Info",
+                        "False positive": "Info"
+                    }
+                    severity = severity_map.get(severity_raw, "Info")
+
+                    host_elem = issue_elem.find("host")
+                    ip = host_elem.get("ip", "unknown") if host_elem is not None else "unknown"
+                    hostname = host_elem.text if host_elem is not None else None
+                    host_model = Host(ip=ip, hostname=hostname)
+
+                    # Burp writes HTML into these fields. scanfuse keeps the text.
+                    bg = html_to_text(issue_elem.findtext("issueBackground", default=""))
+                    detail = html_to_text(issue_elem.findtext("issueDetail", default=""))
+                    desc = "\n\n".join(part for part in (detail, bg) if part)
+
+                    remediation = "\n\n".join(
+                        part for part in (
+                            html_to_text(issue_elem.findtext("remediationBackground", default="")),
+                            html_to_text(issue_elem.findtext("remediationDetail", default="")),
+                        ) if part
+                    )
+
+                    evidence_list = []
+
+                    path = issue_elem.findtext("path", default="").strip()
+                    if path:
+                        evidence_list.append(Evidence(type="path", data=path))
+                    location = html_to_text(issue_elem.findtext("location", default=""))
+                    confidence = issue_elem.findtext("confidence", default="").strip()
+                    if confidence or (location and location != path):
+                        meta = []
+                        if confidence:
+                            meta.append(f"Confidence: {confidence}")
+                        if location and location != path:
+                            meta.append(f"Location: {location}")
+                        evidence_list.append(Evidence(type="burp-details", data="\n".join(meta)))
+
+                    req_resps = issue_elem.findall("requestresponse")
+                    for rr in req_resps:
+                        req_elem = rr.find("request")
+                        res_elem = rr.find("response")
+
+                        text = ""
+                        if req_elem is not None and req_elem.text:
+                            req_data = req_elem.text
+                            if req_elem.get("base64") == "true":
+                                try:
+                                    req_data = base64.b64decode(req_data).decode("utf-8", errors="replace")
+                                except Exception:  # noqa: S110
+                                    pass
+                            text += f"Request:\n{req_data}\n\n"
+
+                        if res_elem is not None and res_elem.text:
+                            res_data = res_elem.text
+                            if res_elem.get("base64") == "true":
+                                try:
+                                    res_data = base64.b64decode(res_data).decode("utf-8", errors="replace")
+                                except Exception:  # noqa: S110
+                                    pass
+                            text += f"Response:\n{res_data}"
+
+                        if text.strip():
+                            evidence_list.append(Evidence(type="http-request", data=text.strip()))
+
+                    type_id = issue_elem.findtext("type", default="").strip()
+                    references_html = issue_elem.findtext("references", default="")
+                    classifications = issue_elem.findtext("vulnerabilityClassifications", default="")
+                    identifiers = clean_ids(
+                        f"burp-type:{type_id}" if type_id else None,
+                        cwe_ids_from_text(classifications, references_html),
+                        cve_ids(classifications + " " + references_html),
+                    )
+
+                    finding = Finding(
+                        title=name,
+                        severity=severity,
+                        description=desc,
+                        remediation=remediation if remediation else None,
+                        hosts=[host_model],
+                        evidence=evidence_list,
+                        identifiers=identifiers,
+                        references=extract_urls(references_html, classifications),
+                        source_tool="burp"
+                    )
+                    findings.append(finding)
+
+                    issue_elem.clear()
+                    root.clear()
+
+        except ET.ParseError as e:
+            raise ValueError(f"Invalid XML in {file_path}: {e}")
+        except OSError as e:
+            raise ValueError(f"File I/O error reading {file_path}: {e}")
+
+        return findings

scanfuse/parsers/nessus.py

@@ -0,0 +1,126 @@
+import xml.etree.ElementTree as ET
+
+import defusedxml.ElementTree as defused_et
+
+from scanfuse.models import Evidence, Finding, Host
+from scanfuse.parsers.base import BaseParser, clean_ids, cve_ids, cwe_id, extract_urls
+
+
+class NessusParser(BaseParser):
+    def parse(self, file_path: str) -> list[Finding]:
+        findings = []
+        try:
+            context = iter(defused_et.iterparse(file_path, events=("start", "end")))
+
+            try:
+                event, root = next(context)
+            except StopIteration:
+                raise ValueError("Empty XML file")
+
+            if root.tag != "NessusClientData_v2":
+                raise ValueError("Not a valid Nessus file (missing <NessusClientData_v2> root tag)")
+
+            severity_map = {
+                "0": "Info",
+                "1": "Low",
+                "2": "Medium",
+                "3": "High",
+                "4": "Critical"
+            }
+
+            for event, report_host in context:
+                if event == "end" and report_host.tag == "ReportHost":
+                    # Extract Host Properties
+                    host_ip = "unknown"
+                    host_fqdn = None
+                    host_os = None
+
+                    host_props = report_host.find("HostProperties")
+                    if host_props is not None:
+                        for tag in host_props.findall("tag"):
+                            name = tag.get("name")
+                            if name == "host-ip":
+                                host_ip = tag.text or "unknown"
+                            elif name == "host-fqdn":
+                                host_fqdn = tag.text
+                            elif name == "operating-system":
+                                host_os = tag.text
+
+                    # Fallback if host-ip is missing but ReportHost name is an IP
+                    if host_ip == "unknown":
+                        host_ip = report_host.get("name", "unknown")
+
+                    host_model = Host(ip=host_ip, hostname=host_fqdn, os=host_os)
+
+                    # Extract Report Items (Findings)
+                    for item in report_host.findall("ReportItem"):
+                        sev_num = item.get("severity", "0")
+                        severity = severity_map.get(sev_num, "Info")
+
+                        name = item.get("pluginName", "Unknown Plugin")
+
+                        description = item.findtext("description", default="").strip()
+                        synopsis = item.findtext("synopsis", default="").strip()
+                        if synopsis and synopsis not in description:
+                            description = f"{synopsis}\n\n{description}".strip()
+                        solution = item.findtext("solution", default="").strip()
+                        if solution.lower() in ("n/a", "none", "na"):
+                            solution = ""
+                        plugin_output = item.findtext("plugin_output", default="").strip()
+                        cvss = item.findtext("cvss_base_score", default="").strip()
+                        cvss_vector = item.findtext("cvss_vector", default="").strip()
+                        if not cvss:  # newer plugins only carry CVSS v3
+                            cvss = item.findtext("cvss3_base_score", default="").strip()
+                            cvss_vector = item.findtext("cvss3_vector", default="").strip()
+
+                        evidence_list = []
+
+                        port = item.get("port", "0")
+                        protocol = item.get("protocol", "")
+                        if port != "0":
+                            port_text = f"Port: {port}/{protocol}"
+                            svc_name = item.get("svc_name")
+                            if svc_name:
+                                port_text += f"\nService: {svc_name}"
+                            evidence_list.append(Evidence(type="port-info", data=port_text))
+
+                        if plugin_output:
+                            evidence_list.append(Evidence(type="plugin-output", data=plugin_output))
+
+                        try:
+                            cvss_val = float(cvss) if cvss else None
+                        except ValueError:
+                            cvss_val = None
+
+                        plugin_id = item.get("pluginID")
+                        identifiers = clean_ids(
+                            f"nessus-plugin:{plugin_id}" if plugin_id else None,
+                            cve_ids([c.text for c in item.findall("cve")]),
+                            cwe_id(item.findtext("cwe")),
+                        )
+
+                        finding = Finding(
+                            title=name,
+                            severity=severity,
+                            description=description,
+                            remediation=solution if solution else None,
+                            cvss_score=cvss_val,
+                            cvss_vector=cvss_vector if cvss_vector else None,
+                            hosts=[host_model],
+                            evidence=evidence_list,
+                            source_tool="nessus",
+                            identifiers=identifiers,
+                            references=extract_urls(item.findtext("see_also", default="")),
+                            port=f"{port}/{protocol}" if port != "0" else None,
+                        )
+                        findings.append(finding)
+
+                    report_host.clear()
+                    root.clear()
+
+        except ET.ParseError as e:
+            raise ValueError(f"Invalid XML in {file_path}: {e}")
+        except OSError as e:
+            raise ValueError(f"File I/O error reading {file_path}: {e}")
+
+        return findings

scanfuse/parsers/nmap.py

@@ -0,0 +1,137 @@
+import xml.etree.ElementTree as ET
+
+import defusedxml.ElementTree as defused_et
+
+from scanfuse.models import Evidence, Finding, Host
+from scanfuse.parsers.base import BaseParser, clean_ids
+
+
+def _best_os_guess(host_elem: ET.Element) -> str | None:
+    """The operating system guess with the highest accuracy, e.g. 'Linux 5.0 - 5.7 (97%)'."""
+    os_elem = host_elem.find("os")
+    if os_elem is None:
+        return None
+    best_name, best_accuracy = None, -1
+    for match in os_elem.findall("osmatch"):
+        name = match.get("name")
+        try:
+            accuracy = int(match.get("accuracy", "0"))
+        except ValueError:
+            accuracy = 0
+        if name and accuracy > best_accuracy:
+            best_name, best_accuracy = name, accuracy
+    if best_name is None:
+        return None
+    return f"{best_name} ({best_accuracy}%)"
+
+
+class NmapParser(BaseParser):
+    def parse(self, file_path: str) -> list[Finding]:
+        findings = []
+        try:
+            context = iter(defused_et.iterparse(file_path, events=("start", "end")))
+
+            try:
+                event, root = next(context)
+            except StopIteration:
+                raise ValueError("Empty XML file")
+
+            if root.tag != "nmaprun":
+                raise ValueError("Not a valid Nmap XML file (missing <nmaprun> root tag)")
+
+            for event, host_elem in context:
+                if event == "end" and host_elem.tag == "host":
+                    status = host_elem.find("status")
+                    if status is not None and status.get("state") != "up":
+                        host_elem.clear()
+                        root.clear()
+                        continue
+
+                    ip = ""
+                    for address in host_elem.findall("address"):
+                        if address.get("addrtype") in ("ipv4", "ipv6"):
+                            ip = address.get("addr")
+                            break
+
+                    if not ip:
+                        host_elem.clear()
+                        root.clear()
+                        continue
+
+                    hostname = None
+                    hostnames = host_elem.find("hostnames")
+                    if hostnames is not None:
+                        for hn in hostnames.findall("hostname"):
+                            if hn.get("name"):
+                                hostname = hn.get("name")
+                                break
+
+                    host_model = Host(ip=ip, hostname=hostname, os=_best_os_guess(host_elem))
+
+                    ports_elem = host_elem.find("ports")
+                    if ports_elem is not None:
+                        for port_elem in ports_elem.findall("port"):
+                            state_elem = port_elem.find("state")
+                            if state_elem is None or state_elem.get("state") != "open":
+                                continue
+
+                            protocol = port_elem.get("protocol", "tcp")
+                            portid = port_elem.get("portid", "")
+
+                            service_elem = port_elem.find("service")
+                            service_name = "unknown"
+                            product = ""
+                            version = ""
+                            if service_elem is not None:
+                                service_name = service_elem.get("name", service_name)
+                                product = service_elem.get("product", "")
+                                version = service_elem.get("version", "")
+
+                            cpes = (
+                                [c.text.strip() for c in service_elem.findall("cpe") if c.text and c.text.strip()]
+                                if service_elem is not None else []
+                            )
+
+                            service_desc = service_name
+                            if product:
+                                service_desc += f" ({f'{product} {version}'.strip()})"
+
+                            title = f"Open Port: {portid}/{protocol} ({service_desc})"
+
+                            evidence_text = f"Port: {portid}/{protocol}\nState: open\nService: {service_name}"
+                            if product:
+                                evidence_text += f"\nProduct: {product}"
+                            if version:
+                                evidence_text += f"\nVersion: {version}"
+
+                            for script in port_elem.findall("script"):
+                                script_id = script.get("id", "")
+                                script_output = script.get("output", "")
+                                evidence_text += f"\nScript {script_id}: {script_output}"
+
+                            evidence = Evidence(
+                                type="port-scan",
+                                data=evidence_text.strip()
+                            )
+
+                            finding = Finding(
+                                title=title,
+                                severity="Info",
+                                description=f"An open {protocol} port was discovered on port {portid} running {service_desc}.",
+                                hosts=[host_model],
+                                evidence=[evidence],
+                                source_tool="nmap",
+                                identifiers=clean_ids(cpes),
+                                port=f"{portid}/{protocol}" if portid else None,
+                            )
+                            findings.append(finding)
+
+                    host_elem.clear()
+                    root.clear()
+
+        except ET.ParseError as e:
+            raise ValueError(f"Invalid XML in {file_path}: {e}")
+        except OSError as e:
+            raise ValueError(f"File I/O error reading {file_path}: {e}")
+
+        return findings