netscraper 0.1.0__py3-none-any.whl

netscraper/core/families.py

@@ -0,0 +1,467 @@
+"""
+Orchestration layer: one function per check "family" that knows how to run
+and evaluate everything for a single qualifying host. cli.py stays generic
+-- it just gates ports, calls the right family function per host, and
+routes whatever comes back to the right folders. Adding a new family later
+(SMB, LDAP, MSSQL, ...) means adding one more function here with the same
+shape, not touching the orchestration logic in cli.py.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import List, Optional, Tuple
+
+from netscraper.core.targets import Target
+from netscraper.core.rules import Finding
+
+from netscraper.core.sslscan_runner import get_xml as sslscan_get_xml, get_ansi_text as sslscan_get_ansi
+from netscraper.core.sslscan_parser import parse_xml as sslscan_parse
+from netscraper.core.rules import evaluate as sslscan_evaluate
+
+from netscraper.core.cert_chain_runner import get_chain_text
+from netscraper.core.cert_chain_parser import parse_chain_text
+from netscraper.core.cert_chain_rules import evaluate as cert_chain_evaluate
+
+from netscraper.core.ssh_runner import get_json as ssh_get_json, get_ansi_text as ssh_get_ansi
+from netscraper.core.ssh_parser import parse_ssh_audit_json
+from netscraper.core.ssh_rules import evaluate as ssh_evaluate
+
+from netscraper.core.smb_runner import get_nmap_text as smb_get_nmap, get_smbclient_text as smb_get_smbclient
+from netscraper.core.smb_parser import parse_smb_text
+from netscraper.core.smb_rules import evaluate as smb_evaluate
+
+from netscraper.core.ldap_runner import get_anon_text as ldap_get_anon, get_signing_probe_text as ldap_get_signing, BOGUS_DN, BOGUS_PW
+from netscraper.core.ldap_parser import parse_ldap_text
+from netscraper.core.ldap_rules import evaluate as ldap_evaluate
+
+from netscraper.core.rpc_runner import get_rpcdump_text as rpc_get_rpcdump
+from netscraper.core.rpc_parser import parse_rpcdump_text
+from netscraper.core.rpc_rules import evaluate as rpc_evaluate
+
+from netscraper.core.dns_runner import get_dig_text as dns_get_dig
+from netscraper.core.dns_parser import parse_dig_text
+from netscraper.core.dns_rules import evaluate as dns_evaluate
+
+from netscraper.core.snmp_runner import get_onesixtyone_text as snmp_get_onesixtyone
+from netscraper.core.snmp_parser import parse_onesixtyone_text
+from netscraper.core.snmp_rules import evaluate as snmp_evaluate
+
+from netscraper.core.ftp_runner import get_ftp_text as ftp_get_ftp
+from netscraper.core.ftp_parser import parse_ftp_text
+from netscraper.core.ftp_rules import evaluate as ftp_evaluate
+
+from netscraper.core.mssql_runner import get_nmap_text as mssql_get_nmap
+from netscraper.core.mssql_parser import parse_mssql_text
+from netscraper.core.mssql_rules import evaluate as mssql_evaluate
+
+from netscraper.core.http_runner import (
+    get_version_text as http_get_version,
+    get_host_header_text as http_get_host_header,
+    get_trace_text as http_get_trace,
+    marker_for as http_marker_for,
+)
+from netscraper.core.http_parser import parse_http_result
+from netscraper.core.http_rules import evaluate as http_evaluate
+
+from netscraper.core.dirsearch_runner import get_dirsearch_text
+from netscraper.core.dirsearch_parser import parse_dirsearch_text
+from netscraper.core.dirsearch_rules import evaluate as dirsearch_evaluate
+
+from netscraper.core.cors_runner import get_cors_text, EVIL_ORIGIN
+from netscraper.core.cors_parser import parse_cors_text
+from netscraper.core.cors_rules import evaluate as cors_evaluate
+
+
+@dataclass
+class FamilyHostResult:
+    status: str  # "vulnerable" | "safe" | "error" -- informational; cli.py actually
+                 # branches on whether findings/manual_checks are non-empty, so a
+                 # host can carry confirmed findings AND manual-verification items
+                 # from the same family at once (e.g. LDAP: anonymous bind confirmed
+                 # vulnerable, signing enforcement flagged for manual verification).
+    # (finding, raw_text_to_render_this_finding_from, real_command_that_produced_it)
+    # -- confirmed Vulnerable/Safe verdict. The command string is shown verbatim
+    # on the POC (title bar + prompt line) so every family's screenshot honestly
+    # reflects which real tool/invocation produced it, instead of a hardcoded name.
+    findings: List[Tuple[Finding, str, str]] = field(default_factory=list)
+    # Same shape, but for checks whose result can't be verdicted automatically
+    # without a false-positive/false-negative risk (e.g. behaviorally-fuzzy
+    # protocol checks) -- routed to a "Needs Manual Verification" folder
+    # instead of being forced into Vulnerable or Safe.
+    manual_checks: List[Tuple[Finding, str, str]] = field(default_factory=list)
+    safe_raw_text: Optional[str] = None
+    safe_command: Optional[str] = None
+    error_message: Optional[str] = None
+    raw_artifacts: List[Tuple[str, str]] = field(default_factory=list)  # (label, raw_text) for audit trail
+
+
+def run_tls_family(target: Target, xml_dir: Optional[str], timeout: int) -> FamilyHostResult:
+    try:
+        xml_text = sslscan_get_xml(target, xml_dir, timeout=timeout)
+        ansi_text = sslscan_get_ansi(target, xml_dir, timeout=timeout)
+        chain_text = get_chain_text(target, xml_dir, timeout=timeout)
+    except Exception as exc:  # noqa: BLE001
+        return FamilyHostResult(status="error", error_message=str(exc))
+
+    try:
+        scan_result = sslscan_parse(xml_text, host=target.host, port=target.port)
+        chain_result = parse_chain_text(chain_text, host=target.host, port=target.port)
+    except Exception as exc:  # noqa: BLE001
+        return FamilyHostResult(status="error", error_message=f"parse error: {exc}")
+
+    sslscan_cmd = f"sslscan {target.host}:{target.port}"
+    chain_cmd = f"openssl s_client -connect {target.host}:{target.port} -servername {target.host} -showcerts"
+
+    findings: List[Tuple[Finding, str, str]] = []
+    for f in sslscan_evaluate(scan_result):
+        findings.append((f, ansi_text, sslscan_cmd))
+    for f in cert_chain_evaluate(chain_result):
+        findings.append((f, chain_text, chain_cmd))
+
+    raw_artifacts = [("sslscan", xml_text), ("sslscan_terminal", ansi_text), ("cert_chain", chain_text)]
+
+    if not findings:
+        return FamilyHostResult(status="safe", safe_raw_text=ansi_text, safe_command=sslscan_cmd, raw_artifacts=raw_artifacts)
+    return FamilyHostResult(status="vulnerable", findings=findings, raw_artifacts=raw_artifacts)
+
+
+def run_ssh_family(target: Target, xml_dir: Optional[str], timeout: int) -> FamilyHostResult:
+    try:
+        json_data = ssh_get_json(target, xml_dir, timeout=timeout)
+        ansi_text = ssh_get_ansi(target, xml_dir, timeout=timeout)
+    except Exception as exc:  # noqa: BLE001
+        return FamilyHostResult(status="error", error_message=str(exc))
+
+    try:
+        result = parse_ssh_audit_json(json_data, host=target.host, port=target.port)
+    except Exception as exc:  # noqa: BLE001
+        return FamilyHostResult(status="error", error_message=f"parse error: {exc}")
+
+    ssh_cmd = f"ssh-audit {target.host}:{target.port}"
+    findings = [(f, ansi_text, ssh_cmd) for f in ssh_evaluate(result)]
+    raw_artifacts = [("ssh_audit_terminal", ansi_text)]
+
+    if not findings:
+        return FamilyHostResult(status="safe", safe_raw_text=ansi_text, safe_command=ssh_cmd, raw_artifacts=raw_artifacts)
+    return FamilyHostResult(status="vulnerable", findings=findings, raw_artifacts=raw_artifacts)
+
+
+def run_smb_family(target: Target, xml_dir: Optional[str], timeout: int) -> FamilyHostResult:
+    try:
+        nmap_text = smb_get_nmap(target, xml_dir, timeout=timeout)
+        smbclient_text = smb_get_smbclient(target, xml_dir, timeout=timeout)
+    except Exception as exc:  # noqa: BLE001
+        return FamilyHostResult(status="error", error_message=str(exc))
+
+    try:
+        result = parse_smb_text(nmap_text, smbclient_text, host=target.host, port=target.port)
+    except Exception as exc:  # noqa: BLE001
+        return FamilyHostResult(status="error", error_message=f"parse error: {exc}")
+
+    nmap_cmd = (
+        f"nmap -p {target.port} --script smb-protocols,smb-security-mode,smb2-security-mode "
+        f"--script-args smbport={target.port} {target.host}"
+    )
+    smbclient_cmd = f"smbclient -L //{target.host} -p {target.port} -N"
+
+    confirmed, manual = smb_evaluate(result)
+    # nmap findings render from nmap_text (SMBv1 / signing); the anonymous-access
+    # finding renders from smbclient_text -- each POC highlights only its own tool's
+    # real output.
+    findings = [
+        (f, smbclient_text, smbclient_cmd) if f.rule_id == "smb_null_session_anonymous_access"
+        else (f, nmap_text, nmap_cmd)
+        for f in confirmed
+    ]
+    manual_checks = [(f, nmap_text, nmap_cmd) for f in manual]
+    raw_artifacts = [("smb_nmap", nmap_text), ("smbclient", smbclient_text)]
+
+    if not findings and not manual_checks:
+        safe_text = nmap_text + "\n\n$ " + smbclient_cmd + "\n" + smbclient_text
+        return FamilyHostResult(
+            status="safe", safe_raw_text=safe_text, safe_command=nmap_cmd, raw_artifacts=raw_artifacts,
+        )
+    return FamilyHostResult(
+        status="vulnerable" if findings else "safe",
+        findings=findings,
+        manual_checks=manual_checks,
+        raw_artifacts=raw_artifacts,
+    )
+
+
+def run_ldap_family(target: Target, xml_dir: Optional[str], timeout: int) -> FamilyHostResult:
+    try:
+        anon_text = ldap_get_anon(target, xml_dir, timeout=timeout)
+        signing_text = ldap_get_signing(target, xml_dir, timeout=timeout)
+    except Exception as exc:  # noqa: BLE001
+        return FamilyHostResult(status="error", error_message=str(exc))
+
+    try:
+        result = parse_ldap_text(anon_text, signing_text, host=target.host, port=target.port)
+    except Exception as exc:  # noqa: BLE001
+        return FamilyHostResult(status="error", error_message=f"parse error: {exc}")
+
+    anon_cmd = f"ldapsearch -x -H ldap://{target.host}:{target.port} -b \"\" -s base"
+    signing_cmd = f"ldapsearch -x -D \"{BOGUS_DN}\" -w {BOGUS_PW} -H ldap://{target.host}:{target.port} -b \"\" -s base"
+
+    confirmed, manual = ldap_evaluate(result)
+    findings = [
+        (f, signing_text, signing_cmd) if f.rule_id == "ldap_signing_not_enforced" else (f, anon_text, anon_cmd)
+        for f in confirmed
+    ]
+    manual_checks = [
+        (f, signing_text, signing_cmd) if f.rule_id == "ldap_signing_unknown" else (f, anon_text, anon_cmd)
+        for f in manual
+    ]
+    raw_artifacts = [("ldap_anon", anon_text), ("ldap_signing_probe", signing_text)]
+
+    if not findings and not manual_checks:
+        safe_text = anon_text + "\n\n$ " + signing_cmd + "\n" + signing_text
+        return FamilyHostResult(
+            status="safe", safe_raw_text=safe_text, safe_command=anon_cmd, raw_artifacts=raw_artifacts,
+        )
+    return FamilyHostResult(
+        status="vulnerable" if findings else "safe",
+        findings=findings,
+        manual_checks=manual_checks,
+        raw_artifacts=raw_artifacts,
+    )
+
+
+def run_rpc_family(target: Target, xml_dir: Optional[str], timeout: int) -> FamilyHostResult:
+    try:
+        rpcdump_text = rpc_get_rpcdump(target, xml_dir, timeout=timeout)
+    except Exception as exc:  # noqa: BLE001
+        return FamilyHostResult(status="error", error_message=str(exc))
+
+    try:
+        result = parse_rpcdump_text(rpcdump_text, host=target.host, port=target.port)
+    except Exception as exc:  # noqa: BLE001
+        return FamilyHostResult(status="error", error_message=f"parse error: {exc}")
+
+    rpc_cmd = f"rpcdump.py -port {target.port if target.port in (135, 139, 443, 445, 593) else 445} ''@{target.host}"
+    confirmed, manual = rpc_evaluate(result)
+    findings = [(f, rpcdump_text, rpc_cmd) for f in confirmed]
+    manual_checks = [(f, rpcdump_text, rpc_cmd) for f in manual]
+    raw_artifacts = [("rpcdump", rpcdump_text)]
+
+    if not findings and not manual_checks:
+        return FamilyHostResult(
+            status="safe", safe_raw_text=rpcdump_text, safe_command=rpc_cmd, raw_artifacts=raw_artifacts,
+        )
+    return FamilyHostResult(
+        status="vulnerable" if findings else "safe",
+        findings=findings,
+        manual_checks=manual_checks,
+        raw_artifacts=raw_artifacts,
+    )
+
+
+def run_dns_family(target: Target, xml_dir: Optional[str], timeout: int) -> FamilyHostResult:
+    try:
+        dig_text = dns_get_dig(target, xml_dir, timeout=timeout)
+    except Exception as exc:  # noqa: BLE001
+        return FamilyHostResult(status="error", error_message=str(exc))
+    try:
+        result = parse_dig_text(dig_text, host=target.host, port=target.port)
+    except Exception as exc:  # noqa: BLE001
+        return FamilyHostResult(status="error", error_message=f"parse error: {exc}")
+
+    cmd = f"dig @{target.host} -p {target.port} google.com A +time=3 +tries=1"
+    confirmed, manual = dns_evaluate(result)
+    findings = [(f, dig_text, cmd) for f in confirmed]
+    manual_checks = [(f, dig_text, cmd) for f in manual]
+    raw_artifacts = [("dig", dig_text)]
+
+    if not findings and not manual_checks:
+        return FamilyHostResult(status="safe", safe_raw_text=dig_text, safe_command=cmd, raw_artifacts=raw_artifacts)
+    return FamilyHostResult(
+        status="vulnerable" if findings else "safe", findings=findings, manual_checks=manual_checks,
+        raw_artifacts=raw_artifacts,
+    )
+
+
+def run_snmp_family(target: Target, xml_dir: Optional[str], timeout: int) -> FamilyHostResult:
+    try:
+        text, cmd = snmp_get_onesixtyone(target, xml_dir, timeout=timeout)
+    except Exception as exc:  # noqa: BLE001
+        return FamilyHostResult(status="error", error_message=str(exc))
+    try:
+        result = parse_onesixtyone_text(text, host=target.host, port=target.port)
+    except Exception as exc:  # noqa: BLE001
+        return FamilyHostResult(status="error", error_message=f"parse error: {exc}")
+
+    confirmed, manual = snmp_evaluate(result)
+    findings = [(f, text, cmd) for f in confirmed]
+    manual_checks = [(f, text, cmd) for f in manual]
+    raw_artifacts = [("onesixtyone", text)]
+
+    if not findings and not manual_checks:
+        return FamilyHostResult(status="safe", safe_raw_text=text, safe_command=cmd, raw_artifacts=raw_artifacts)
+    return FamilyHostResult(
+        status="vulnerable" if findings else "safe", findings=findings, manual_checks=manual_checks,
+        raw_artifacts=raw_artifacts,
+    )
+
+
+def run_ftp_family(target: Target, xml_dir: Optional[str], timeout: int) -> FamilyHostResult:
+    try:
+        text = ftp_get_ftp(target, xml_dir, timeout=timeout)
+    except Exception as exc:  # noqa: BLE001
+        return FamilyHostResult(status="error", error_message=str(exc))
+    try:
+        result = parse_ftp_text(text, host=target.host, port=target.port)
+    except Exception as exc:  # noqa: BLE001
+        return FamilyHostResult(status="error", error_message=f"parse error: {exc}")
+
+    cmd = f"ftp -inv {target.host} {target.port}   # scripted: user anonymous / ls / quit"
+    confirmed, manual = ftp_evaluate(result)
+    findings = [(f, text, cmd) for f in confirmed]
+    manual_checks = [(f, text, cmd) for f in manual]
+    raw_artifacts = [("ftp_session", text)]
+
+    if not findings and not manual_checks:
+        return FamilyHostResult(status="safe", safe_raw_text=text, safe_command=cmd, raw_artifacts=raw_artifacts)
+    return FamilyHostResult(
+        status="vulnerable" if findings else "safe", findings=findings, manual_checks=manual_checks,
+        raw_artifacts=raw_artifacts,
+    )
+
+
+def run_mssql_family(target: Target, xml_dir: Optional[str], timeout: int) -> FamilyHostResult:
+    try:
+        text = mssql_get_nmap(target, xml_dir, timeout=timeout)
+    except Exception as exc:  # noqa: BLE001
+        return FamilyHostResult(status="error", error_message=str(exc))
+    try:
+        result = parse_mssql_text(text, host=target.host, port=target.port)
+    except Exception as exc:  # noqa: BLE001
+        return FamilyHostResult(status="error", error_message=f"parse error: {exc}")
+
+    cmd = f"nmap -p {target.port} --script ms-sql-info {target.host}"
+    confirmed, manual = mssql_evaluate(result)
+    findings = [(f, text, cmd) for f in confirmed]
+    manual_checks = [(f, text, cmd) for f in manual]
+    raw_artifacts = [("mssql_nmap", text)]
+
+    if not findings and not manual_checks:
+        return FamilyHostResult(status="safe", safe_raw_text=text, safe_command=cmd, raw_artifacts=raw_artifacts)
+    return FamilyHostResult(
+        status="vulnerable" if findings else "safe", findings=findings, manual_checks=manual_checks,
+        raw_artifacts=raw_artifacts,
+    )
+
+
+def run_http_family(target: Target, xml_dir: Optional[str], timeout: int) -> FamilyHostResult:
+    marker = http_marker_for(target)
+    try:
+        version_text = http_get_version(target, xml_dir, timeout=timeout)
+        host_header_text = http_get_host_header(target, xml_dir, timeout=timeout)
+        trace_text = http_get_trace(target, xml_dir, marker, timeout=timeout)
+    except Exception as exc:  # noqa: BLE001
+        return FamilyHostResult(status="error", error_message=str(exc))
+
+    try:
+        result = parse_http_result(version_text, host_header_text, trace_text, host=target.host, port=target.port)
+    except Exception as exc:  # noqa: BLE001
+        return FamilyHostResult(status="error", error_message=f"parse error: {exc}")
+
+    version_cmd = f"curl -sS -i http://{target.host}:{target.port}/"
+    host_header_cmd = f"curl -sS -i -H \"Host: bing.com\" http://{target.host}:{target.port}/"
+    trace_cmd = f"curl -sS -i -X TRACE -H \"X-PT-XST-Marker: {marker}\" http://{target.host}:{target.port}/"
+
+    text_and_cmd_by_rule = {
+        "http_version_disclosure": (version_text, version_cmd),
+        "http_host_header_injection": (host_header_text, host_header_cmd),
+        "http_xst": (trace_text, trace_cmd),
+    }
+
+    confirmed, manual = http_evaluate(result)
+    findings = [(f, *text_and_cmd_by_rule[f.rule_id]) for f in confirmed]
+    raw_artifacts = [
+        ("http_version", version_text), ("http_hostheader", host_header_text), ("http_trace", trace_text),
+    ]
+
+    if not findings:
+        safe_text = (
+            "$ " + version_cmd + "\n" + version_text
+            + "\n\n$ " + host_header_cmd + "\n" + host_header_text
+            + "\n\n$ " + trace_cmd + "\n" + trace_text
+        )
+        return FamilyHostResult(
+            status="safe", safe_raw_text=safe_text, safe_command=version_cmd, raw_artifacts=raw_artifacts,
+        )
+    return FamilyHostResult(status="vulnerable", findings=findings, raw_artifacts=raw_artifacts)
+
+
+def run_dirsearch_family(target: Target, xml_dir: Optional[str], timeout: int) -> FamilyHostResult:
+    try:
+        text, cmd = get_dirsearch_text(target, xml_dir, timeout=timeout)
+    except Exception as exc:  # noqa: BLE001
+        return FamilyHostResult(status="error", error_message=str(exc))
+    try:
+        result = parse_dirsearch_text(text, host=target.host, port=target.port)
+    except Exception as exc:  # noqa: BLE001
+        return FamilyHostResult(status="error", error_message=f"parse error: {exc}")
+
+    confirmed, manual = dirsearch_evaluate(result)
+    findings = [(f, text, cmd) for f in confirmed]
+    raw_artifacts = [("dirsearch", text)]
+
+    if not findings:
+        return FamilyHostResult(status="safe", safe_raw_text=text, safe_command=cmd, raw_artifacts=raw_artifacts)
+    return FamilyHostResult(status="vulnerable", findings=findings, raw_artifacts=raw_artifacts)
+
+
+def run_cors_family(target: Target, xml_dir: Optional[str], timeout: int) -> FamilyHostResult:
+    try:
+        text, cmd = get_cors_text(target, xml_dir, timeout=timeout)
+    except Exception as exc:  # noqa: BLE001
+        return FamilyHostResult(status="error", error_message=str(exc))
+    try:
+        result = parse_cors_text(text, host=target.host, port=target.port, origin_sent=EVIL_ORIGIN)
+    except Exception as exc:  # noqa: BLE001
+        return FamilyHostResult(status="error", error_message=f"parse error: {exc}")
+
+    confirmed, manual = cors_evaluate(result)
+    findings = [(f, text, cmd) for f in confirmed]
+    raw_artifacts = [("cors_probe", text)]
+
+    if not findings:
+        return FamilyHostResult(status="safe", safe_raw_text=text, safe_command=cmd, raw_artifacts=raw_artifacts)
+    return FamilyHostResult(status="vulnerable", findings=findings, raw_artifacts=raw_artifacts)
+
+
+# Each family: (name, output_folder, gate_port_or_None_for_own_port, run_function)
+# gate_port=None means "gate on the target's own .port field" (e.g. TLS,
+# which defaults to 443 but can be overridden per-target via host:port).
+# gate_port="udp:<port>" means "no TCP pre-gate at all -- the real UDP
+# protocol probe on that fixed port IS the gate" (see cli.py); used for
+# families whose service has no meaningful TCP presence to "nc -z" (DNS,
+# SNMP). gate_port="prompt" means there's no fixed port at all -- the user
+# supplies it (one or several, comma-separated) only after choosing this
+# mode; cli.py's _resolve_prompt_gated_ports() handles that deferred gating
+# instead of the normal unified up-front port scan.
+# render_poc (default True when absent) -- False means cli.py never renders
+# a POC screenshot for this family's findings, only the .txt evidence file;
+# used for DIRSEARCH, whose "Found" list is what the client wants to see,
+# not a terminal screenshot.
+FAMILIES = [
+    {"name": "tls", "folder": "SSL_Scan", "gate_port": None, "run": run_tls_family},
+    {"name": "ssh", "folder": "SSH", "gate_port": 22, "run": run_ssh_family},
+    {"name": "smb", "folder": "SMB", "gate_port": 445, "run": run_smb_family},
+    {"name": "ldap", "folder": "LDAP", "gate_port": 389, "run": run_ldap_family},
+    {"name": "rpc", "folder": "RPC", "gate_port": 445, "run": run_rpc_family},
+    {"name": "dns", "folder": "DNS", "gate_port": "udp:53", "run": run_dns_family},
+    {"name": "mssql", "folder": "MSSQL", "gate_port": 1433, "run": run_mssql_family},
+    {"name": "snmp", "folder": "SNMP", "gate_port": "udp:161", "run": run_snmp_family},
+    {"name": "ftp", "folder": "FTP", "gate_port": 21, "run": run_ftp_family},
+    {"name": "http", "folder": "HTTP", "gate_port": 80, "run": run_http_family},
+    {"name": "dirsearch", "folder": "Dirsearch", "gate_port": "prompt", "render_poc": False,
+     "deferred_desc": "directory/file brute-force (pick port(s) after selecting)",
+     "run": run_dirsearch_family},
+    {"name": "cors", "folder": "CORS", "gate_port": "prompt", "render_poc": False,
+     "deferred_desc": "CORS misconfiguration check (pick port(s) after selecting)",
+     "run": run_cors_family},
+]

netscraper/core/ftp_parser.py

@@ -0,0 +1,28 @@
+"""Parses a real `ftp` client session transcript for the anonymous-access check."""
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Optional
+
+_LOGIN_SUCCESS_RE = re.compile(r"^230\b", re.MULTILINE)
+_LOGIN_FAILURE_RE = re.compile(r"^530\b", re.MULTILINE)
+
+
+@dataclass
+class FtpResult:
+    host: str
+    port: int
+    raw_text: str
+    anonymous_login_allowed: Optional[bool] = None  # None = inconclusive
+
+
+def parse_ftp_text(raw_text: str, host: str, port: int) -> FtpResult:
+    result = FtpResult(host=host, port=port, raw_text=raw_text)
+    if _LOGIN_SUCCESS_RE.search(raw_text):
+        result.anonymous_login_allowed = True
+    elif _LOGIN_FAILURE_RE.search(raw_text) or "Login failed" in raw_text or "OOPS:" in raw_text:
+        result.anonymous_login_allowed = False
+    else:
+        result.anonymous_login_allowed = None
+    return result

netscraper/core/ftp_rules.py

@@ -0,0 +1,56 @@
+"""Rule engine for the FTP family: Anonymous FTP Access (#21)."""
+from __future__ import annotations
+
+import re
+from typing import List, Optional, Tuple
+
+from netscraper.core.rules import Finding
+from netscraper.core.ftp_parser import FtpResult
+
+
+def check_anonymous_ftp(result: FtpResult) -> Optional[Finding]:
+    if result.anonymous_login_allowed is not True:
+        return None
+    return Finding(
+        rule_id="ftp_anonymous_access",
+        label="Anonymous FTP Access",
+        folder_name="Anonymous_FTP_Access",
+        severity="high",
+        evidence_lines=[
+            f"Host: {result.host}:{result.port}",
+            "A real `ftp` client session logged in as user 'anonymous' and received '230 Login successful.'.",
+            "--> Anonymous FTP allows unauthenticated access to whatever this server shares -- at minimum "
+            "directory/file listing, and file download unless further restricted.",
+        ],
+        highlight_regex=re.compile(r"^230\b", re.MULTILINE),
+    )
+
+
+def check_manual(result: FtpResult) -> Optional[Finding]:
+    if result.anonymous_login_allowed is not None:
+        return None
+    return Finding(
+        rule_id="ftp_anonymous_unknown",
+        label="Anonymous FTP Access",
+        folder_name="Needs_Manual_Verification/Anonymous_FTP",
+        severity="medium",
+        evidence_lines=[
+            f"Host: {result.host}:{result.port}",
+            "The scripted ftp session didn't return a recognized 230/530 login result (unusual banner, "
+            "connection reset mid-session, or a non-standard FTP daemon).",
+            "--> Confirm manually, e.g. `ftp <host>` and log in as 'anonymous'.",
+        ],
+        highlight_regex=None,
+    )
+
+
+def evaluate(result: FtpResult) -> Tuple[List[Finding], List[Finding]]:
+    confirmed = []
+    manual = []
+    f = check_anonymous_ftp(result)
+    if f:
+        confirmed.append(f)
+    f = check_manual(result)
+    if f:
+        manual.append(f)
+    return confirmed, manual

netscraper/core/ftp_runner.py

@@ -0,0 +1,63 @@
+"""
+Runs a real, scripted `ftp` client session against a target: log in as
+"anonymous" with a conventional anonymous-FTP identification string as the
+password, then `ls` (so a successful login also shows a real directory
+listing as evidence), then quit. The server's own real "230 Login
+successful." vs "530 Login incorrect." response is the signal -- validated
+against two real local vsftpd instances (anonymous enabled vs. disabled).
+"""
+from __future__ import annotations
+
+import shutil
+import subprocess
+from pathlib import Path
+from typing import Optional
+
+from netscraper.core.targets import Target
+
+_LOGIN_SCRIPT = "user anonymous anonymous@example.com\nls\nquit\n"
+
+
+class FtpToolNotFound(RuntimeError):
+    pass
+
+
+class FtpScanError(RuntimeError):
+    def __init__(self, target: Target, message: str):
+        super().__init__(message)
+        self.target = target
+
+
+def _require(binary: str) -> str:
+    path = shutil.which(binary)
+    if not path:
+        raise FtpToolNotFound(f"'{binary}' is not installed or not on PATH (apt install ftp).")
+    return path
+
+
+def run_live_text(target: Target, timeout: int = 20) -> str:
+    binary = _require("ftp")
+    cmd = [binary, "-inv", target.host, str(target.port)]
+    try:
+        proc = subprocess.run(cmd, input=_LOGIN_SCRIPT, capture_output=True, text=True, timeout=timeout)
+    except subprocess.TimeoutExpired as exc:
+        raise FtpScanError(target, f"ftp timed out after {timeout}s") from exc
+    text = (proc.stdout or "") + (proc.stderr or "")
+    if not text.strip():
+        raise FtpScanError(target, "ftp produced no output")
+    return text
+
+
+def run_replay_text(target: Target, xml_dir: str) -> str:
+    candidate = Path(xml_dir) / f"{target.label}_ftp.txt"
+    if not candidate.exists():
+        candidate = Path(xml_dir) / f"{target.host}_ftp.txt"
+    if not candidate.exists():
+        raise FtpScanError(target, f"No captured ftp session output found for {target.host} in {xml_dir}")
+    return candidate.read_text(encoding="utf-8")
+
+
+def get_ftp_text(target: Target, xml_dir: Optional[str], timeout: int = 20) -> str:
+    if xml_dir:
+        return run_replay_text(target, xml_dir)
+    return run_live_text(target, timeout=timeout)

netscraper/core/http_parser.py

@@ -0,0 +1,59 @@
+"""Parses the three real curl captures for the HTTP family."""
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Optional
+
+_STATUS_RE = re.compile(r"^HTTP/[\d.]+\s+(\d{3})", re.MULTILINE)
+_SERVER_HEADER_RE = re.compile(r"^Server:\s*(.+?)\s*$", re.MULTILINE | re.IGNORECASE)
+_VERSION_NUMBER_RE = re.compile(r"\d+\.\d+")
+_LOCATION_HEADER_RE = re.compile(r"^Location:\s*(.+?)\s*$", re.MULTILINE | re.IGNORECASE)
+
+XST_MARKER_HEADER_NAME = "X-PT-XST-Marker"
+
+
+@dataclass
+class HttpResult:
+    host: str
+    port: int
+    version_text: str
+    host_header_text: str
+    trace_text: str
+    server_header: Optional[str] = None
+    version_disclosed: bool = False
+    host_header_status: Optional[int] = None
+    host_header_location: Optional[str] = None
+    host_header_reflected: bool = False
+    trace_status: Optional[int] = None
+    trace_reflected: bool = False
+
+
+def parse_http_result(version_text: str, host_header_text: str, trace_text: str, host: str, port: int) -> HttpResult:
+    result = HttpResult(
+        host=host, port=port,
+        version_text=version_text, host_header_text=host_header_text, trace_text=trace_text,
+    )
+
+    m = _SERVER_HEADER_RE.search(version_text)
+    if m:
+        result.server_header = m.group(1).strip()
+        result.version_disclosed = bool(_VERSION_NUMBER_RE.search(result.server_header))
+
+    sm = _STATUS_RE.search(host_header_text)
+    if sm:
+        result.host_header_status = int(sm.group(1))
+    lm = _LOCATION_HEADER_RE.search(host_header_text)
+    if lm:
+        result.host_header_location = lm.group(1).strip()
+        result.host_header_reflected = (
+            result.host_header_status is not None and 300 <= result.host_header_status < 400
+            and "bing.com" in result.host_header_location.lower()
+        )
+
+    tm = _STATUS_RE.search(trace_text)
+    if tm:
+        result.trace_status = int(tm.group(1))
+    result.trace_reflected = XST_MARKER_HEADER_NAME.lower() in trace_text.lower()
+
+    return result