web-scanner 2.0.0__py3-none-any.whl

webscanner/__main__.py

@@ -0,0 +1,6 @@
+"""Enable ``python -m webscanner``."""
+
+from webscanner.cli import main
+
+if __name__ == "__main__":
+    main()

webscanner/cli.py

@@ -0,0 +1,24 @@
+"""Console entry point for the ``webscan`` command.
+
+    webscan example.com
+    python -m webscanner example.com
+"""
+
+from __future__ import annotations
+
+import sys
+
+from webscanner.helpers import is_valid_url
+from webscanner.ui.app import WebScannerApp
+
+
+def main() -> None:
+    target = sys.argv[1].strip() if len(sys.argv) > 1 else None
+    if target and not is_valid_url(target):
+        print(f"[!] Not a valid target: {target}")
+        raise SystemExit(1)
+    WebScannerApp(target).run()
+
+
+if __name__ == "__main__":
+    main()

webscanner/colors.py

@@ -0,0 +1,4 @@
+"""Shared semantic colours used in cell markup across modules and the UI."""
+
+GREEN = "green"
+RED = "#FF6C64"

webscanner/core/__init__.py

@@ -0,0 +1,15 @@
+"""Async scanning core: context, module contract, orchestrator, result models."""
+
+from .context import ScanContext
+from .models import ModuleResult, ModuleStatus, ScanEvent
+from .module import ScanModule
+from .scanner import AsyncScanner
+
+__all__ = [
+    "ScanContext",
+    "ScanModule",
+    "AsyncScanner",
+    "ModuleResult",
+    "ModuleStatus",
+    "ScanEvent",
+]

webscanner/core/context.py

@@ -0,0 +1,40 @@
+"""Shared per-scan context passed to every module.
+
+The orchestrator's prefetch phase populates the shared network fields (one DNS
+resolve, one HTTP GET, one TLS handshake, one geo lookup) so individual modules
+reuse them instead of refetching.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from .. import helpers
+
+
+@dataclass
+class ScanContext:
+    # inputs
+    domain: str  # bare host, e.g. "example.com"
+    url: str  # full url, e.g. "https://example.com"
+
+    # shared prefetch results (filled by AsyncScanner.prefetch)
+    ip: str | None = None
+    html: str | None = None
+    headers: dict[str, str] = field(default_factory=dict)
+    status_code: int | None = None
+    response_time_ms: float | None = None
+    final_url: str | None = None
+    tls_cert: dict[str, Any] | None = None
+    geo: dict[str, Any] | None = None
+    fetch_error: str | None = None
+
+    @classmethod
+    def from_target(cls, target: str) -> "ScanContext":
+        domain, url = helpers.normalise(target)
+        return cls(domain=domain, url=url)
+
+    @property
+    def online(self) -> bool:
+        return self.status_code is not None and self.fetch_error is None

webscanner/core/models.py

@@ -0,0 +1,79 @@
+"""Result and event data models shared across the core and UI."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any
+
+
+class ModuleStatus(str, Enum):
+    """Lifecycle state of a single scan module (also drives tab colouring)."""
+
+    PENDING = "pending"  # not started
+    RUNNING = "running"  # in flight
+    DONE = "done"  # finished with data
+    EMPTY = "empty"  # finished, no data found
+    FAILED = "failed"  # raised
+
+
+@dataclass(slots=True)
+class ModuleResult:
+    """Outcome of one module run."""
+
+    name: str
+    status: ModuleStatus
+    data: Any = None
+    error: str | None = None
+    duration_ms: float | None = None
+
+    @property
+    def ok(self) -> bool:
+        return self.status in (ModuleStatus.DONE, ModuleStatus.EMPTY)
+
+
+@dataclass(slots=True)
+class ScanEvent:
+    """Progress signal emitted by the orchestrator; the UI subscribes to these."""
+
+    name: str  # module name, or "__prefetch__" for the shared fetch phase
+    status: ModuleStatus
+    result: ModuleResult | None = None
+
+
+@dataclass(slots=True)
+class Section:
+    """One titled sub-table within a multi-table module result."""
+
+    title: str
+    data: Any
+    headers: tuple[str, str] | None = None
+    #: fixed column proportions (col1, col2), e.g. (3, 2) for 60/40; None = content-fit
+    ratio: tuple[int, int] | None = None
+    #: blank line between rows
+    spaced: bool = False
+
+
+class Sections(list):
+    """A module result rendered as several stacked, titled sub-tables.
+
+    A ``list`` of :class:`Section`. Used by modules (e.g. Security) that show more
+    than one table under a single tab.
+    """
+
+
+class Grid(list):
+    """A multi-column table result: rows (list of value-lists) + column headers.
+
+    A ``list`` of rows, so an empty ``Grid`` reports EMPTY via ``_is_empty``. The
+    first column is treated as each row's primary name.
+    """
+
+    def __init__(self, columns: list[str], rows: Any) -> None:
+        super().__init__(rows)
+        self.columns = list(columns)
+
+    @property
+    def names(self) -> list[str]:
+        """First-column values (e.g. tech names for the Server-panel summary)."""
+        return [row[0] for row in self]

webscanner/core/module.py

@@ -0,0 +1,27 @@
+"""The scan-module contract.
+
+Every module is a small, independently replaceable unit. It reads what it needs
+from the shared ``ScanContext``, does its work, and returns structured data.
+Modules must never manage their own status/timing/error handling — the
+orchestrator wraps each ``run`` call and isolates failures so one broken module
+cannot abort the whole scan.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+from .context import ScanContext
+
+
+class ScanModule(ABC):
+    #: one-word identifier; also the tab id (e.g. "dns")
+    name: str = ""
+    #: human label shown in the tab bar (e.g. "DNS")
+    label: str = ""
+
+    @abstractmethod
+    async def run(self, ctx: ScanContext) -> Any:
+        """Do the scan and return JSON-serialisable data (or raise)."""
+        raise NotImplementedError

webscanner/core/scanner.py

@@ -0,0 +1,103 @@
+"""AsyncScanner — concurrent orchestrator.
+
+Flow:
+  1. prefetch()  — resolve IP, HTTP GET, TLS handshake and geo lookup, all
+     concurrently, populating the shared ScanContext.
+  2. run()       — run every module concurrently, each failure-isolated, emitting
+     ScanEvents so the UI can animate progress and colour tabs live.
+
+Blocking libraries (pydig, requests, wappalyzer, system whois) are offloaded to
+threads via ``asyncio.to_thread`` so the event loop never stalls.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from collections.abc import Callable, Iterable
+
+from ..net import http
+from .context import ScanContext
+from .models import ModuleResult, ModuleStatus, ScanEvent
+from .module import ScanModule
+
+EventCallback = Callable[[ScanEvent], None]
+
+PREFETCH = "__prefetch__"
+
+
+def _is_empty(data: object) -> bool:
+    if data is None:
+        return True
+    if isinstance(data, (dict, list, str, tuple, set)):
+        return len(data) == 0
+    return False
+
+
+class AsyncScanner:
+    def __init__(
+        self,
+        ctx: ScanContext,
+        modules: Iterable[ScanModule],
+        on_event: EventCallback | None = None,
+    ) -> None:
+        self.ctx = ctx
+        self.modules = list(modules)
+        self.on_event = on_event
+        self.results: dict[str, ModuleResult] = {}
+
+    def _emit(self, name: str, status: ModuleStatus, result: ModuleResult | None = None) -> None:
+        if self.on_event is not None:
+            self.on_event(ScanEvent(name=name, status=status, result=result))
+
+    async def prefetch(self) -> None:
+        """Populate the shared context (IP first, then fetch/geo/tls concurrently)."""
+        self.ctx.ip = await asyncio.to_thread(http.resolve_ip, self.ctx.domain)
+
+        async def _fetch() -> None:
+            try:
+                r = await asyncio.to_thread(http.fetch, self.ctx.url)
+                self.ctx.status_code = r["status_code"]
+                self.ctx.headers = r["headers"]
+                self.ctx.html = r["html"]
+                self.ctx.response_time_ms = r["elapsed_ms"]
+                self.ctx.final_url = r["final_url"]
+            except Exception as exc:  # noqa: BLE001 - surfaced, not raised
+                self.ctx.fetch_error = repr(exc)
+
+        async def _geo() -> None:
+            if self.ctx.ip:
+                try:
+                    self.ctx.geo = await asyncio.to_thread(http.get_geo, self.ctx.ip)
+                except Exception:  # noqa: BLE001
+                    pass
+
+        async def _tls() -> None:
+            try:
+                self.ctx.tls_cert = await asyncio.to_thread(http.get_tls_cert, self.ctx.domain)
+            except Exception:  # noqa: BLE001
+                pass
+
+        await asyncio.gather(_fetch(), _geo(), _tls())
+
+    async def _run_module(self, module: ScanModule) -> ModuleResult:
+        self._emit(module.name, ModuleStatus.RUNNING)
+        start = time.perf_counter()
+        try:
+            data = await module.run(self.ctx)
+            duration = (time.perf_counter() - start) * 1000
+            status = ModuleStatus.EMPTY if _is_empty(data) else ModuleStatus.DONE
+            result = ModuleResult(module.name, status, data=data, duration_ms=duration)
+        except Exception as exc:  # noqa: BLE001 - isolate module failures
+            duration = (time.perf_counter() - start) * 1000
+            result = ModuleResult(module.name, ModuleStatus.FAILED, error=repr(exc), duration_ms=duration)
+        self.results[module.name] = result
+        self._emit(module.name, result.status, result)
+        return result
+
+    async def run(self) -> dict[str, ModuleResult]:
+        self._emit(PREFETCH, ModuleStatus.RUNNING)
+        await self.prefetch()
+        self._emit(PREFETCH, ModuleStatus.DONE)
+        await asyncio.gather(*(self._run_module(m) for m in self.modules))
+        return self.results

webscanner/helpers.py

@@ -0,0 +1,41 @@
+"""URL validation and normalisation helpers (ported/modernised from v1)."""
+
+from __future__ import annotations
+
+import re
+from urllib.parse import urlparse
+
+_URL_RE = re.compile(
+    r"^(?:http|ftp)s?://"  # scheme
+    r"|^"  # or no scheme
+    r"(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+"  # domain labels
+    r"(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|"  # tld
+    r"localhost|"
+    r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|"  # ipv4
+    r"\[?[A-F0-9]*:[A-F0-9:]+\]?)"  # ipv6
+    r"(?::\d+)?"  # port
+    r"(?:/?|[/?]\S+)?$",  # path
+    re.IGNORECASE,
+)
+
+
+def is_valid_url(url: str) -> bool:
+    """True if the string looks like a URL/host we can scan."""
+    return re.match(_URL_RE, url) is not None
+
+
+def to_domain(url: str) -> str:
+    """Reduce any URL/host to its bare registrable host (drops scheme, path, www.)."""
+    if not urlparse(url).scheme:
+        url = "https://" + url
+    domain = urlparse(url).netloc or urlparse(url).path
+    domain = domain.split("/")[0]
+    if domain.startswith("www."):
+        domain = domain[4:]
+    return domain.lower()
+
+
+def normalise(target: str) -> tuple[str, str]:
+    """Return (bare_domain, full_https_url) for a user-supplied target."""
+    domain = to_domain(target.strip())
+    return domain, f"https://{domain}"

webscanner/modules/__init__.py

@@ -0,0 +1,38 @@
+"""Scan module registry.
+
+``all_modules()`` order defines the tab order in the UI:
+DNS · Whois · Subdomains · SSL · Security · Headers · Tech · SEO · Links
+(Email auth — DMARC/DKIM — is folded into DNS. Security = ports + HTTP headers + blocklists.
+SEO = schema + content + keywords. Links = internal + external page links.)
+"""
+
+from __future__ import annotations
+
+from ..core.module import ScanModule
+from .dns import DnsModule
+from .headers import HeadersModule
+from .links import LinksModule
+from .security import SecurityModule
+from .seo import SeoModule
+from .ssl import SslModule
+from .subdomains import SubdomainsModule
+from .tech import TechModule
+from .whois import WhoisModule
+
+
+def all_modules() -> list[ScanModule]:
+    """Fresh instances in tab order."""
+    return [
+        DnsModule(),
+        WhoisModule(),
+        SubdomainsModule(),
+        SslModule(),
+        SecurityModule(),
+        HeadersModule(),
+        TechModule(),
+        SeoModule(),
+        LinksModule(),
+    ]
+
+
+__all__ = ["all_modules"]

webscanner/modules/dns.py

@@ -0,0 +1,96 @@
+"""DNS records module (also folds in email auth: DMARC + DKIM discovery).
+
+SPF and MX already surface as normal TXT/MX records, so we don't duplicate them.
+DMARC lives at ``_dmarc.<domain>`` and DKIM keys at ``<selector>._domainkey.<domain>``,
+so those aren't in the default record set — we look them up and append rows only
+when found (no "present: yes/no" noise).
+"""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+
+import pydig
+
+from ..core.module import ScanModule
+from ..core.context import ScanContext
+
+RECORD_TYPES = ("A", "AAAA", "NS", "CNAME", "SOA", "MX", "TXT", "CAA", "DS", "DNSKEY")
+
+# Known DKIM selectors across the major providers/ESPs. DKIM can't be enumerated
+# from DNS alone, so we probe this list of well-known selectors.
+DKIM_SELECTORS = (
+    # generic
+    "default", "dkim", "mail", "selector", "s", "s1", "s2", "key1", "dk",
+    # Microsoft 365 / Outlook
+    "selector1", "selector2",
+    # Google Workspace
+    "google",
+    # Amazon SES
+    "amazonses",
+    # Zoho
+    "zoho", "zmail",
+    # ProtonMail
+    "protonmail", "protonmail2", "protonmail3",
+    # Fastmail
+    "fm1", "fm2", "fm3",
+    # Mailchimp / Mandrill
+    "k1", "k2", "k3", "mandrill",
+    # SendGrid
+    "smtpapi",
+    # Postmark
+    "pm",
+    # Mailgun / generic smtp
+    "smtp", "mg", "mailo",
+    # Campaign Monitor
+    "cm",
+    # Apple iCloud
+    "sig1",
+    # HubSpot
+    "hs1", "hs2",
+    # misc common
+    "mxvault", "everlytickey1", "everlytickey2", "titan1", "titan2",
+    "turbo-smtp", "mailjet", "sendinblue", "klaviyo",
+)
+
+
+def _is_dkim(records: list[str]) -> bool:
+    return any(("dkim1" in r.lower() or "p=" in r.lower()) for r in records)
+
+
+class DnsModule(ScanModule):
+    name = "dns"
+    label = "DNS"
+
+    async def run(self, ctx: ScanContext) -> dict[str, Any]:
+        domain = ctx.domain
+
+        def records() -> dict[str, list[str]]:
+            out: dict[str, list[str]] = {}
+            for rtype in RECORD_TYPES:
+                res = pydig.query(domain, rtype)
+                if res:
+                    out[rtype] = res
+            return out
+
+        async def dmarc() -> list[str]:
+            txt = await asyncio.to_thread(pydig.query, f"_dmarc.{domain}", "TXT")
+            return [t for t in txt if "dmarc1" in t.lower()]
+
+        async def dkim_selector(sel: str) -> str | None:
+            txt = await asyncio.to_thread(pydig.query, f"{sel}._domainkey.{domain}", "TXT")
+            return sel if txt and _is_dkim(txt) else None
+
+        out, dmarc_records, dkim_hits = await asyncio.gather(
+            asyncio.to_thread(records),
+            dmarc(),
+            asyncio.gather(*(dkim_selector(s) for s in DKIM_SELECTORS)),
+        )
+
+        if dmarc_records:
+            out["DMARC"] = dmarc_records
+        found = [s for s in dkim_hits if s]
+        if found:
+            out["DKIM"] = found
+        return out

webscanner/modules/headers.py

@@ -0,0 +1,16 @@
+"""HTTP response headers (reuses the shared prefetch response)."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from ..core.module import ScanModule
+from ..core.context import ScanContext
+
+
+class HeadersModule(ScanModule):
+    name = "headers"
+    label = "Headers"
+
+    async def run(self, ctx: ScanContext) -> dict[str, Any]:
+        return dict(ctx.headers)

webscanner/modules/links.py

@@ -0,0 +1,69 @@
+"""Page links module — internal vs external links found on the page.
+
+Two tables (link text → href URL): Internal (same registrable domain, incl.
+relative links and subdomains) and External (everything else). Uses the HTML
+fetched once during prefetch.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from urllib.parse import urljoin, urlparse
+
+from bs4 import BeautifulSoup
+
+from ..core.module import ScanModule
+from ..core.context import ScanContext
+from ..core.models import Section, Sections
+
+_SKIP_PREFIXES = ("#", "javascript:", "mailto:", "tel:", "data:")
+
+
+class LinksModule(ScanModule):
+    name = "links"
+    label = "Links"
+
+    async def run(self, ctx: ScanContext) -> Sections:
+        if not ctx.html:
+            internal: list[tuple[str, str]] = []
+            external: list[tuple[str, str]] = []
+        else:
+            internal, external = await asyncio.to_thread(self._parse, ctx)
+
+        return Sections([
+            Section("Internal", internal or [("—", "no internal links found")], ("Link text", "URL")),
+            Section("External", external or [("—", "no external links found")], ("Link text", "URL")),
+        ])
+
+    @staticmethod
+    def _parse(ctx: ScanContext) -> tuple[list[tuple[str, str]], list[tuple[str, str]]]:
+        soup = BeautifulSoup(ctx.html, "html.parser")
+        base = ctx.final_url or ctx.url
+        internal: list[tuple[str, str]] = []
+        external: list[tuple[str, str]] = []
+        seen_i: set[str] = set()
+        seen_e: set[str] = set()
+
+        for tag in soup.find_all("a", href=True):
+            href = tag["href"].strip()
+            if not href or href.startswith(_SKIP_PREFIXES):
+                continue
+            url = urljoin(base, href)
+            parsed = urlparse(url)
+            if parsed.scheme not in ("http", "https"):
+                continue
+
+            text = " ".join(tag.get_text(strip=True).split()) or "-"
+            host = parsed.netloc.lower()
+            if host.startswith("www."):
+                host = host[4:]
+
+            if host == ctx.domain or host.endswith("." + ctx.domain):
+                if url not in seen_i:
+                    seen_i.add(url)
+                    internal.append((text, url))
+            elif url not in seen_e:
+                seen_e.add(url)
+                external.append((text, url))
+
+        return internal, external