web-scanner 2.1.2__tar.gz → 2.1.3__tar.gz

{web_scanner-2.1.2 → web_scanner-2.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: web-scanner
-Version: 2.1.2
+Version: 2.1.3
 Summary: An async Textual TUI for website reconnaissance — DNS, WHOIS, TLS, security, tech-stack and SEO, with no paid APIs.
 Project-URL: Homepage, https://github.com/iamramizk/web-scanner
 Project-URL: Repository, https://github.com/iamramizk/web-scanner

{web_scanner-2.1.2 → web_scanner-2.1.3}/webscanner/__init__.py

@@ -1,3 +1,3 @@
 """web-scanner v2 — async website reconnaissance TUI package."""
 
-__version__ = "2.1.2"
+__version__ = "2.1.3"

{web_scanner-2.1.2 → web_scanner-2.1.3}/webscanner/colors.py

@@ -5,3 +5,6 @@ RED = "#FF6C64"
 #: muted value/secondary text — matches the unselected-tab colour
 #: ($text-muted = white @ 60% over the dark theme bg).
 MUTED = "#9D9D9D"
+#: activity-log body text — one step down the white → MUTED ladder, so a log line
+#: reads softer than a heading but stays well clear of the dim MUTED timestamp.
+BODY = "#C6C6C6"

{web_scanner-2.1.2 → web_scanner-2.1.3}/webscanner/modules/links.py

@@ -18,6 +18,11 @@ from ..core.models import Section, Sections
 
 _SKIP_PREFIXES = ("#", "javascript:", "mailto:", "tel:", "data:")
 
+# Placeholder row shown when a section found nothing. Named so callers can tell
+# "no links" from "one link" — the section is never empty, so len() can't.
+EMPTY_INTERNAL = ("—", "no internal links found")
+EMPTY_EXTERNAL = ("—", "no external links found")
+
 
 class LinksModule(ScanModule):
     name = "links"
@@ -31,8 +36,8 @@ class LinksModule(ScanModule):
             internal, external = await asyncio.to_thread(self._parse, ctx)
 
         return Sections([
-            Section("Internal", internal or [("—", "no internal links found")], ("Link text", "URL")),
-            Section("External", external or [("—", "no external links found")], ("Link text", "URL")),
+            Section("Internal", internal or [EMPTY_INTERNAL], ("Link text", "URL")),
+            Section("External", external or [EMPTY_EXTERNAL], ("Link text", "URL")),
         ])
 
     @staticmethod

web_scanner-2.1.3/webscanner/ui/activity.py

@@ -0,0 +1,326 @@
+"""Activity-log lines — one short, informative sentence per scan event.
+
+Pure ``ScanEvent -> str``: no Textual imports, so the whole line catalogue can be
+exercised without an app, an event loop or the network. That matters because a
+healthy live scan never reaches the branches most likely to be wrong (FAILED, EMPTY,
+and the ``{"note": ...}`` "nothing to report" shape).
+
+Grammar is fixed at ``<Label>: Complete. <facts>`` so the coloured verb always lands
+in the same column. Only the verb and the odd warning word carry markup; the caller
+(``ActivityLog.add``) supplies the timestamp and the body colour.
+
+Lines are composed **at full length** — nothing here truncates to a character budget.
+``ActivityLog`` renders each line ``no_wrap`` with ``overflow="ellipsis"``, so a line
+is cut at the panel edge and a wider terminal simply shows more of it. Keep the facts
+front-loaded (the tail is what gets dropped), but don't pre-crop a value to make it
+fit some assumed width.
+
+The facts are the point — "DNS: Complete." tells you nothing the tab colour didn't.
+Each summarizer digs the headline number out of its module's own return shape, which
+is different for every module and booby-trapped in several (see the notes below).
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Callable
+
+from rich.markup import escape
+
+from ..colors import GREEN, RED
+from ..core.context import ScanContext
+from ..core.models import ModuleResult, ModuleStatus, ScanEvent
+from ..core.scanner import PREFETCH
+from ..modules import all_modules
+from ..modules.dns import RECORD_TYPES
+from ..modules.links import EMPTY_EXTERNAL, EMPTY_INTERNAL
+from ..modules.sitemap import MAX_URLS
+from .tables import _plain
+
+#: module name -> tab label ("dns" -> "DNS"), so a line's prefix matches its tab.
+_LABELS: dict[str, str] = {m.name: m.label for m in all_modules()}
+
+_DONE = f"[{GREEN}]Complete[/]"
+_ERROR = f"[{RED}]Error[/]"
+
+#: repr(exc) -> ("TimeoutError", "'timed out'")
+_ERROR_REPR = re.compile(r"^(\w+)\((.*)\)$", re.S)
+
+
+# ---- helpers --------------------------------------------------------------
+
+
+def _esc(value: Any) -> str:
+    """Collapse whitespace and escape rich markup.
+
+    Every externally-sourced value in a log line goes through here. Escaping is not
+    cosmetic: an unescaped ``[/]`` (from a registrar name, a Server header, a
+    repr(exc)) raises MarkupError inside ``RichLog.write``, which propagates out of
+    ``on_scan_progress`` and takes the handler down mid-scan.
+
+    Deliberately does **not** truncate: values are written in full and the widget
+    ellipsises at the panel edge, so a wide terminal shows the whole thing. A hard
+    cap here would crop "Web Address Registration Pty Ltd" on a 200-column display.
+    """
+    return escape(" ".join(str(value).split()))
+
+
+def _plural(n: int, singular: str, plural: str | None = None) -> str:
+    return f"{n} {singular if n == 1 else (plural or singular + 's')}"
+
+
+def _sentence(text: str) -> str:
+    """Upper-case the first letter and full-stop it, leaving the rest alone —
+    ``str.capitalize`` would flatten the modules' own wording (WHOIS -> Whois)."""
+    text = " ".join(str(text).split()).rstrip(".")
+    return f"{text[:1].upper()}{text[1:]}." if text else ""
+
+
+def _note(data: Any) -> str | None:
+    """The module's own wording when it returned a lone ``{"note": ...}``.
+
+    whois, security's port table and seo all use this shape to say "nothing to
+    report". Echoing it verbatim keeps the wording with the module that owns it.
+    """
+    if isinstance(data, dict) and set(data) == {"note"}:
+        return str(data["note"])
+    return None
+
+
+def _sections(result: ModuleResult) -> dict[str, Any]:
+    """A Sections result as {title: data}."""
+    return {section.title: section.data for section in result.data or []}
+
+
+def _short_error(error: str | None) -> str:
+    """``repr(exc)`` boiled down to ``Type: message``.
+
+    Unwrapping the repr is formatting, not trimming — the message is kept whole and
+    the panel ellipsises whatever doesn't fit.
+    """
+    text = (error or "unknown error").strip()
+    if match := _ERROR_REPR.match(text):
+        name, args = match.group(1), match.group(2).strip().strip("'\"")
+        text = f"{name}: {args}" if args else name
+    return _esc(text)
+
+
+def _date_only(value: Any) -> str:
+    """The date half of a whois timestamp — they arrive both as bare dates and as
+    ISO datetimes, so slicing [:10] is not safe."""
+    text = str(value or "").replace("T", " ").strip()
+    return text.split()[0] if text else ""
+
+
+# ---- per-module summaries -------------------------------------------------
+
+
+def _dns(result: ModuleResult) -> str:
+    # Keys are only present when non-empty, and DMARC/DKIM are folded in alongside
+    # the record types — count only the latter, then name the email auth found.
+    data = result.data or {}
+    if not data:
+        return f"{_DONE}. No records found."
+    types = [key for key in data if key in RECORD_TYPES]
+    records = sum(len(data[key]) for key in types)
+    line = f"{_DONE}. {_plural(records, 'record')} across {_plural(len(types), 'type')}"
+    auth = [key for key in ("DMARC", "DKIM") if key in data]
+    return f"{line} · {', '.join(auth)}." if auth else f"{line}."
+
+
+def _whois(result: ModuleResult) -> str:
+    data = result.data or {}
+    if not data:
+        return f"{_DONE}. No WHOIS data returned."
+    if (note := _note(data)) is not None:
+        return f"{_DONE}. {_sentence(note)}"
+    bits = []
+    if registrar := data.get("registrar"):
+        # Unlabelled: the Whois prefix and the "expires" half make it obvious enough,
+        # and the 10 chars "Registrar " costs are better spent on the name itself.
+        bits.append(_esc(registrar))
+    if expiry := _date_only(data.get("expiry_date")):
+        bits.append(f"expires {_esc(expiry)}")
+    # Sparse ccTLD output can carry neither field; say what did come back.
+    return f"{_DONE}. {' · '.join(bits) or _plural(len(data), 'field')}."
+
+
+def _subdomains(result: ModuleResult) -> str:
+    found = result.data or []
+    if not found:
+        return f"{_DONE}. None found."
+    return f"{_DONE}. Found {_plural(len(found), 'subdomain')}."
+
+
+def _ssl(result: ModuleResult) -> str:
+    data = result.data or {}
+    if not data:
+        return "No certificate found (TLS handshake failed)."
+    issuer = _esc(data.get("issuer_org") or data.get("issuer_cn") or "unknown issuer")
+    # san_count, not len(san) — the san list is capped at 20.
+    sans = _plural(data.get("san_count") or 0, "SAN")
+    if data.get("expired"):
+        return f"Certificate [{RED}]expired[/] — {issuer} · {sans}."
+    bits = [issuer]
+    # days_until_expiry is absent when the notAfter date failed to parse.
+    if (days := data.get("days_until_expiry")) is not None:
+        bits.append(f"{_plural(days, 'day')} left")
+    bits.append(sans)
+    return f"{_DONE}. {' · '.join(bits)}."
+
+
+def _security(result: ModuleResult) -> str:
+    # All three tables are fixed-length (every port/header/blocklist gets a row), so
+    # the counts have to come from the values, not len().
+    sections = _sections(result)
+    ports = sections.get("Open Ports", {})
+    headers = sections.get("HTTP Security", {})
+    blocklists = sections.get("Blocklists", {})
+
+    if (note := _note(ports)) is not None:
+        port_bit = _sentence(note).rstrip(".")  # the module's wording, leading the list
+    else:
+        port_bit = _plural(sum(1 for v in ports.values() if _plain(v) == "open"), "open port")
+    present = sum(1 for v in headers.values() if _plain(v) == "Yes")
+    # Exact match: "Not Blocked" contains "Blocked".
+    blocked = sum(1 for v in blocklists.values() if _plain(v) == "Blocked")
+    return f"{_DONE}. {port_bit}, {present}/{len(headers)} headers, {blocked} blocked."
+
+
+def _headers(result: ModuleResult) -> str:
+    data = result.data or {}
+    if not data:
+        return f"{_DONE}. No headers (fetch failed)."
+    # dict(ctx.headers) drops requests' CaseInsensitiveDict, so keys keep whatever
+    # casing the server sent.
+    server = next((v for k, v in data.items() if k.lower() == "server"), None)
+    line = f"{_DONE}. {_plural(len(data), 'header')}"
+    # Whole value, incl. "Apache/2.4.41 (Ubuntu) mod_wsgi/4.6.8" — it's the last
+    # thing on the line, so a narrow panel ellipsises only its tail.
+    return f"{line} · Server {_esc(server)}." if server else f"{line}."
+
+
+def _tech(result: ModuleResult) -> str:
+    # A tech declaring three groups appears in three Grids — dedupe on name.
+    sections = result.data or []
+    names = {name for section in sections for name in section.data.names}
+    if not names:
+        return f"{_DONE}. No technologies detected."
+    return (
+        f"{_DONE}. {_plural(len(names), 'technology', 'technologies')} "
+        f"across {_plural(len(sections), 'group')}."
+    )
+
+
+def _seo(result: ModuleResult) -> str:
+    sections = _sections(result)
+    content = sections.get("Content", {})
+    if (note := _note(content)) is not None:
+        return f"{_DONE}. {_sentence(note)}"
+    bits = []
+    if content.get("Title", "-") != "-":
+        bits.append("title")
+    if content.get("Description", "-") != "-":
+        bits.append("desc")
+    bits.append(_plural(len(content.get("H1") or []), "H1", "H1s"))
+    if _plain(sections.get("Schema", {}).get("Has Schema", "")) == "Yes":
+        bits.append("schema")
+    if _plain(sections.get("Robots", {}).get("Found", "")) == "Yes":
+        bits.append("robots")
+    return f"{_DONE}. {_sentence(', '.join(bits))}"
+
+
+def _sitemap(result: ModuleResult) -> str:
+    root = result.data
+    if root is None:
+        return f"{_DONE}. No sitemap found."
+    total = root.total or 0
+    suffix = " (truncated)" if total >= MAX_URLS else ""
+    return f"{_DONE}. {_plural(total, 'URL')} found{suffix}."
+
+
+def _links(result: ModuleResult) -> str:
+    # An empty section holds one placeholder row, so len() reads 1 for "none".
+    sections = _sections(result)
+    internal = sections.get("Internal", [])
+    external = sections.get("External", [])
+    n_internal = 0 if internal == [EMPTY_INTERNAL] else len(internal)
+    n_external = 0 if external == [EMPTY_EXTERNAL] else len(external)
+    if not n_internal and not n_external:
+        return f"{_DONE}. No links found."
+    return f"{_DONE}. {n_internal} internal, {n_external} external links."
+
+
+_SUMMARIZERS: dict[str, Callable[[ModuleResult], str]] = {
+    "dns": _dns,
+    "whois": _whois,
+    "subdomains": _subdomains,
+    "ssl": _ssl,
+    "security": _security,
+    "headers": _headers,
+    "tech": _tech,
+    "seo": _seo,
+    "sitemap": _sitemap,
+    "links": _links,
+}
+
+
+# ---- public API -----------------------------------------------------------
+
+
+def started(target: str, count: int) -> str:
+    """The opening line."""
+    return f"Scan: Started {_esc(target)} · {_plural(count, 'module')}."
+
+
+def overall(completed: int, failed: int, total: int, seconds: float) -> str:
+    """The closing line. ``seconds`` is wall-clock for the whole scan — modules run
+    concurrently, so summing their durations would overstate it several-fold."""
+    ok = completed - failed
+    took = f"in {round(seconds)}s"
+    if failed:
+        return (
+            f"Overall Scan Status: {ok}/{total} modules [{GREEN}]completed[/] {took}, "
+            f"[{RED}]{_plural(failed, 'error')}[/]."
+        )
+    return f"Overall Scan Status: {total}/{total} modules [{GREEN}]completed[/] {took}."
+
+
+def _prefetch(ctx: ScanContext | None) -> str:
+    if ctx is None:
+        return f"Prefetch: {_ERROR}. No scan context."
+    if ctx.fetch_error:
+        return f"Prefetch: [{RED}]error[/] — {_esc(ctx.fetch_error)}."
+    bits = [_esc(ctx.ip)] if ctx.ip else []
+    if ctx.status_code is not None:
+        bits.append(f"HTTP {ctx.status_code}")
+    if ctx.response_time_ms is not None:
+        bits.append(f"{round(ctx.response_time_ms)}ms")
+    return f"Prefetch: {' · '.join(bits) or 'no response'}."
+
+
+def summarize(event: ScanEvent, ctx: ScanContext | None) -> str | None:
+    """One log line for a ScanEvent, or ``None`` if it shouldn't be logged.
+
+    Module RUNNING events are dropped on purpose: all ten fire in a single
+    ``asyncio.gather`` burst, so logging them would push ten lines through a
+    four-line panel in one tick and none of them would be read.
+    """
+    if event.name == PREFETCH:
+        # Prefetch events never carry a result — the numbers live on the context.
+        return _prefetch(ctx) if event.status is ModuleStatus.DONE else None
+    if event.status is ModuleStatus.RUNNING:
+        return None
+
+    label = _LABELS.get(event.name, event.name)
+    result = event.result
+    if result is None or event.status is ModuleStatus.FAILED:
+        error = result.error if result is not None else None
+        return f"{label}: {_ERROR}. {_short_error(error)}"
+    try:
+        body = _SUMMARIZERS[event.name](result)
+    except Exception:  # noqa: BLE001
+        # Modules are failure-isolated; the handler that logs them isn't. A summary
+        # that trips over an unexpected shape must not take the scan down with it.
+        body = f"{_DONE}."
+    return f"{label}: {body}"

{web_scanner-2.1.2 → web_scanner-2.1.3}/webscanner/ui/app.py

@@ -12,6 +12,10 @@ live as each module completes.
 
 from __future__ import annotations
 
+import re
+import time
+
+from bs4 import BeautifulSoup
 from textual.app import App, ComposeResult
 from textual.binding import Binding
 from textual.containers import Grid, Horizontal, Vertical, VerticalScroll
@@ -21,9 +25,10 @@ from textual.widgets import Input, LoadingIndicator, Static
 from ..core import AsyncScanner, ModuleStatus, ScanContext, ScanEvent
 from ..core.scanner import PREFETCH
 from ..modules import all_modules
+from . import activity
 from .export import export_csvs
 from .tables import render_result
-from .widgets import MapPanel, SitemapTree, StatusPanel, TabBar, Tab
+from .widgets import ActivityLog, MapPanel, SitemapTree, StatusPanel, TabBar, Tab
 
 #: width (cells) of the footer progress bar and its dim (incomplete) colour
 _BAR_WIDTH = 22
@@ -46,6 +51,73 @@ def _cms_from_tech(data: object) -> tuple[str, str | None] | None:
     return None
 
 
+#: matches a ``<meta name="generator">`` name attribute — real pages ship
+#: ``name="Generator"`` as often as lowercase, so the value is matched case-insensitively
+_GENERATOR_ATTR = re.compile(r"^\s*generator\s*$", re.I)
+
+
+def _split_generator(content: str) -> tuple[str, str | None]:
+    """Split a ``<meta name="generator">`` content value into (name, version|None).
+
+    The version is the first token that contains a digit, plus everything after it
+    ("Sitefinity 14.4.8152.0 DX" → ``("Sitefinity", "14.4.8152.0 DX")``); a value with
+    no such token is all name ("Webflow" → ``("Webflow", None)``). A leading digit-ish
+    token is part of the name, not a version ("1C-Bitrix", "TYPO3 CMS"). Trailing
+    parentheticals are dropped ("Drupal 10 (https://www.drupal.org)" → "Drupal", "10").
+    """
+    content = re.sub(r"\s*\([^)]*\)", "", content).strip()
+    tokens = content.split()
+    for i, token in enumerate(tokens):
+        if i and any(char.isdigit() for char in token):
+            return " ".join(tokens[:i]), " ".join(tokens[i:])
+    return content, None
+
+
+def _generators(html: str | None) -> list[tuple[str, str | None]]:
+    """Every ``<meta name="generator">`` in the page, as (name, version|None) pairs."""
+    if not html:
+        return []
+    soup = BeautifulSoup(html, "html.parser")
+    found = []
+    for tag in soup.find_all("meta", attrs={"name": _GENERATOR_ATTR}):
+        content = (tag.get("content") or "").strip()
+        if content:
+            found.append(_split_generator(content))
+    return found
+
+
+def _same_cms(left: str, right: str) -> bool:
+    """Do two CMS names refer to the same product? Compared on letters/digits only,
+    either containing the other, so "Wix" matches "Wix.com Website Builder"."""
+    left, right = (re.sub(r"[^a-z0-9]", "", name.lower()) for name in (left, right))
+    return bool(left and right) and (left in right or right in left)
+
+
+def _detect_cms(tech_data: object, html: str | None) -> tuple[str, str | None] | None:
+    """The CMS (name, version) for the Server panel, or ``None`` if nothing detected.
+
+    Wappalyzer's ``CMS`` category is the primary signal — it's curated, so it won't
+    mistake a page builder or analytics tag for the CMS. The ``<meta name="generator">``
+    tag covers the two cases it misses: a CMS Wappalyzer has no fingerprint for (e.g.
+    Sitefinity, Webflow) → the generator is used outright; and a CMS it detects but
+    can't version → the generator supplies the version, but only when it names the
+    *same* product, so an "Elementor 3.x" generator can't hijack a "WordPress" hit.
+    """
+    tech = _cms_from_tech(tech_data)
+    generators = _generators(html)
+    if tech is None:
+        # Nothing in the tech stack — fall back to the generator, preferring a
+        # versioned one when the page carries several.
+        versioned = next((gen for gen in generators if gen[1]), None)
+        return versioned or (generators[0] if generators else None)
+    name, version = tech
+    if version is None:
+        for gen_name, gen_version in generators:
+            if gen_version and _same_cms(name, gen_name):
+                return name, gen_version
+    return tech
+
+
 class ScanProgress(Message):
     """A ScanEvent surfaced onto the Textual message pump."""
 
@@ -98,6 +170,9 @@ class WebScannerApp(App):
         self.completed = 0
         self.failed = 0
         self._scanning = False
+        # monotonic (not wall clock — immune to the system clock moving) start of the
+        # current scan, for the log's closing "completed in Ns".
+        self._scan_start = 0.0
         # last result rendered into the sitemap Tree, so switching tabs doesn't
         # rebuild (and re-collapse) it every visit.
         self._tree_result = None
@@ -109,10 +184,12 @@ class WebScannerApp(App):
             yield Input(value=self._target or "", placeholder="domain… (press enter to scan)", id="domain")
             yield TabBar(self.modules, id="tabs")
         with Grid(id="grid"):
-            with VerticalScroll(id="main"):
-                yield LoadingIndicator(id="main-loading")
-                yield Static("", id="main-content")
-                yield SitemapTree(id="main-tree")
+            with Vertical(id="left"):
+                with VerticalScroll(id="main"):
+                    yield LoadingIndicator(id="main-loading")
+                    yield Static("", id="main-content")
+                    yield SitemapTree(id="main-tree")
+                yield ActivityLog(id="activity")
             yield MapPanel(id="map")
             with VerticalScroll(id="status"):
                 yield StatusPanel(id="status-content")
@@ -124,6 +201,7 @@ class WebScannerApp(App):
         self.query_one("#topbar").border_title = "🌐 WebScanner"
         self.query_one("#map").border_title = "server location"
         self.query_one("#status", VerticalScroll).border_title = "Server"
+        self.query_one("#activity", ActivityLog).border_title = "Activity Log"
         self.query_one("#tabs", TabBar).set_selected(self.selected)
         self._update_main_title()
         self._set_keybar(editing=False)
@@ -141,6 +219,7 @@ class WebScannerApp(App):
         self.completed = 0
         self.failed = 0
         self._scanning = True
+        self._scan_start = time.monotonic()
         self._tree_result = None
 
         tabs = self.query_one("#tabs", TabBar)
@@ -150,6 +229,9 @@ class WebScannerApp(App):
         self._set_main_loading(True)
         self.query_one("#map", MapPanel).show_loading()
         self.query_one("#status-content", StatusPanel).show_loading(self.ctx)
+        log = self.query_one("#activity", ActivityLog)
+        log.clear()
+        log.add(activity.started(self.ctx.domain, len(self.modules)))
         self._update_progress()
 
         # Blur the input so single-key nav (←/→, q, r) reaches the app rather
@@ -174,6 +256,9 @@ class WebScannerApp(App):
 
     def on_scan_progress(self, message: ScanProgress) -> None:
         event = message.event
+        # Above the early returns below — the log wants prefetch events too.
+        if (line := activity.summarize(event, self.ctx)) is not None:
+            self.query_one("#activity", ActivityLog).add(line)
         if event.name == PREFETCH:
             if event.status is ModuleStatus.DONE and self.ctx is not None:
                 self.query_one("#map", MapPanel).set_geo(self.ctx.geo)
@@ -193,16 +278,24 @@ class WebScannerApp(App):
             self.results[event.name] = event.result
         if event.name == self.selected:
             self._refresh_main()
-        if event.name == "tech" and self.ctx is not None and event.result is not None:
+        if event.name == "tech" and self.ctx is not None:
+            # No result guard: a failed Tech scan can still surface a CMS via the
+            # page's <meta name="generator">.
+            data = event.result.data if event.result is not None else None
             self.query_one("#status-content", StatusPanel).set_ctx(
-                self.ctx, cms=_cms_from_tech(event.result.data)
+                self.ctx, cms=_detect_cms(data, self.ctx.html)
             )
         self._update_progress()
 
     def on_scan_finished(self, message: ScanFinished) -> None:
         self._scanning = False
         total = len(self.modules)
-        self.query_one("#progress", Static).update(f"done · {total}/{total}")
+        self.query_one("#activity", ActivityLog).add(
+            activity.overall(
+                self.completed, self.failed, total, time.monotonic() - self._scan_start
+            )
+        )
+        self.query_one("#progress", Static).update("")
 
     # ---- progress line ----------------------------------------------------
 

{web_scanner-2.1.2 → web_scanner-2.1.3}/webscanner/ui/app.tcss

@@ -73,9 +73,18 @@ Tab.-selected {
     height: 1fr;   /* fill remaining screen so 1fr rows split equally */
 }
 
-#main {
+/* Left column: the data table over the fixed activity log. A bare wrapper (no
+   border/padding of its own) so the grid still sees one cell here. */
+#left {
     row-span: 2;
     height: 100%;
+}
+
+#main {
+    height: 4fr;   /* 80% of the left column (activity takes the other 20%). Must
+                      stay fr, NOT 100%: that claims the whole column and silently
+                      clips #activity (Vertical is overflow: hidden, so no
+                      scrollbar, no error — the panel just never appears). */
     border: round $primary;
     padding: 0 1;
     scrollbar-size: 0 0;   /* keep wheel/key scroll, hide the bar */
@@ -112,6 +121,13 @@ Tab.-selected {
     color: $primary;
     text-style: bold;
 }
+/* The expand/collapse chevron. Not a stock Textual component class — SitemapTree
+   adds it and paints the icon span itself, because Textual renders the chevron in
+   the widget's base style, which no CSS can reach without dragging the labels
+   along. White lifts it clear of the labels (ANSI default fg, ~#c5c8c6). */
+#main-tree > .tree--toggle {
+    color: white;
+}
 /* Leaf pages are @click links (click to open the URL). Neutralise Textual's default
    link styling so they render EXACTLY like plain leaf text — no underline, no hover
    colour/background change. */
@@ -124,6 +140,22 @@ Tab.-selected {
     link-background-hover: transparent;
 }
 
+/* Activity log: 20% of the left column, so it grows with the terminal (#main is
+   the other 80%). 4 rows of that go to the border + padding (box-sizing is
+   border-box), so visible lines = height - 4: ~3 at 44 rows, ~6 at 60.
+   min-height floors it at one readable line on a short terminal, where 20% would
+   otherwise round down to nothing but chrome.
+   background: transparent overrides RichLog's $surface, which would punch an
+   opaque block through the ansi_default bg. */
+#activity {
+    height: 1fr;
+    min-height: 5;
+    border: round $primary;
+    padding: 1 1;
+    background: transparent;
+    scrollbar-size: 0 0;
+}
+
 #map {
     height: 100%;
     border: round $primary;

{web_scanner-2.1.2 → web_scanner-2.1.3}/webscanner/ui/export.py

@@ -13,10 +13,8 @@ from datetime import datetime
 from pathlib import Path
 from typing import Any, Iterator
 
-from rich.text import Text
-
 from ..core.models import Grid, ModuleStatus, Sections, TreeNode
-from .tables import TAB_HEADERS, _SMART_LABEL_TABS, _label
+from .tables import TAB_HEADERS, _SMART_LABEL_TABS, _label, _plain
 
 def _output_base() -> Path:
     """Parent directory for the ``<domain>_<ts>/`` scan folder.
@@ -34,22 +32,6 @@ def _output_base() -> Path:
     return cwd
 
 
-def _stringify(value: Any) -> str:
-    if isinstance(value, (list, tuple, set)):
-        return "\n".join(str(v) for v in value) if value else "-"
-    if isinstance(value, bool):
-        return "yes" if value else "no"
-    if value is None:
-        return "-"
-    return str(value)
-
-
-def _plain(value: Any) -> str:
-    """Stringify and strip rich markup (e.g. ``[green]Yes[/]`` -> ``Yes``)."""
-    s = _stringify(value)
-    return Text.from_markup(s).plain if "[/]" in s else s
-
-
 def _is_pairs(data: Any) -> bool:
     return (
         isinstance(data, (list, tuple))

{web_scanner-2.1.2 → web_scanner-2.1.3}/webscanner/ui/tables.py

@@ -139,6 +139,16 @@ def _stringify(value: Any) -> str:
     return str(value)
 
 
+def _plain(value: Any) -> str:
+    """Stringify and strip rich markup (e.g. ``[green]Yes[/]`` -> ``Yes``).
+
+    Shared with export (CSV cells) and activity (log counts) — both need the plain
+    text behind a module's colour markup.
+    """
+    s = _stringify(value)
+    return Text.from_markup(s).plain if "[/]" in s else s
+
+
 def _label(key: Any, mode: str = "upper") -> str:
     """Format a dict key as a column label. ``mode``: ``"upper"`` (ALL CAPS),
     ``"smart"`` (Title Case, acronyms kept upper), or ``"raw"`` (separators

{web_scanner-2.1.2 → web_scanner-2.1.3}/webscanner/ui/widgets.py

@@ -1,13 +1,18 @@
-"""Custom Textual widgets: tab bar, world-map panel, status panel."""
+"""Custom Textual widgets: tab bar, world-map panel, status panel, activity log."""
 
 from __future__ import annotations
 
+from collections import deque
+from datetime import datetime
+
 from rich.markup import escape
+from rich.text import Text
 from textual.binding import Binding
 from textual.containers import Horizontal
 from textual.message import Message
-from textual.widgets import Static, Tree
+from textual.widgets import RichLog, Static, Tree
 
+from ..colors import BODY, MUTED
 from ..core.context import ScanContext
 from ..core.models import ModuleStatus, TreeNode
 from .tables import UNSET, render_status
@@ -103,6 +108,85 @@ class StatusPanel(Static):
         self.update(render_status(ctx, cms))
 
 
+class ActivityLog(RichLog):
+    """Fixed scan log under the main panel; the newest line is always in view.
+
+    Lines are composed by ``activity.py`` — this only stamps the time, paints the
+    body, and decides how a too-long line is cut, so the wording stays testable
+    without an app.
+
+    Each line is written as a ``no_wrap`` / ``overflow="ellipsis"`` Text at the
+    panel's exact width, so it's truncated with "…" at the panel edge and a wider
+    terminal reveals more of it. Nothing is pre-cropped to an assumed width.
+
+    ``can_focus=False`` is load-bearing, not tidiness: RichLog inherits
+    ScrollableContainer's ungated up/down bindings, so a focused log would swallow
+    ↑/↓ from the Sitemap tree once it overflows. It also stops a click on the panel
+    from taking focus off the app, which is what keeps single-key nav (q/r/s/←/→)
+    working.
+    """
+
+    can_focus = False
+
+    #: scrollback kept for re-rendering on resize; matches RichLog's max_lines
+    _MAX_LINES = 200
+
+    def __init__(self, **kwargs) -> None:
+        # min_width=0: the 78 default renders every write at >=78 cells, overflowing
+        # a narrower panel. wrap=True: with wrap=False, RichLog forces
+        # overflow="ignore" on Text and the tail vanishes with no ellipsis.
+        super().__init__(markup=True, wrap=True, min_width=0, max_lines=self._MAX_LINES, **kwargs)
+        # RichLog renders to strips at write time and never re-renders, so keep the
+        # markup to replay when the width changes.
+        self._entries: deque[str] = deque(maxlen=self._MAX_LINES)
+        self._rendered_width: int | None = None
+
+    def add(self, body: str) -> None:
+        """Write one line, stamped with a muted [HH:MM:SS]."""
+        stamp = escape(f"[{datetime.now():%H:%M:%S}]")
+        markup = f"[{MUTED}]{stamp}[/] [{BODY}]{body}[/]"
+        self._entries.append(markup)
+        self._write(markup)
+
+    def _write(self, markup: str) -> None:
+        text = Text.from_markup(markup)
+        text.no_wrap = True
+        text.overflow = "ellipsis"
+        # Explicit width, because RichLog's own sizing measures against
+        # `app.console`, which is a plain 80 columns regardless of how wide the app
+        # actually is — every line would ellipsise at 80 on a wider terminal. Passing
+        # width bypasses measure/shrink/min_width entirely. Width 0 means we aren't
+        # laid out yet: let RichLog defer the write, and on_resize replays it.
+        width = self.scrollable_content_region.width
+        if width:
+            self.write(text, width=width)
+        else:
+            self.write(text)
+
+    def clear(self) -> "ActivityLog":
+        self._entries.clear()
+        super().clear()
+        return self
+
+    def on_resize(self, event) -> None:
+        # Flushes any deferred writes, once, when a size is first known.
+        super().on_resize(event)
+        width = self.scrollable_content_region.width
+        if not width or width == self._rendered_width:
+            return
+        self._rendered_width = width
+        if not self._entries:
+            return
+        # RichLog renders to strips at write time and never re-renders, so replay at
+        # the new width: a wider terminal reveals more of each line, a narrower one
+        # re-ellipsises. Also fixes the first sizing, where deferred lines were
+        # rendered by RichLog without our explicit width.
+        entries = list(self._entries)
+        super().clear()
+        for markup in entries:
+            self._write(markup)
+
+
 class SitemapTree(Tree):
     """Sitemap tab: the site's URL-path hierarchy as a collapsed tree.
 
@@ -124,6 +208,12 @@ class SitemapTree(Tree):
         Binding("space", "toggle_all", "Expand/collapse all", show=False),
     ]
 
+    #: Textual gives the expand/collapse chevron no component class of its own — it
+    #: renders with the widget's base style, so plain CSS can't reach it without
+    #: recolouring every label too. Adding one here (Textual unions COMPONENT_CLASSES
+    #: across the MRO) lets `#main-tree > .tree--toggle` in app.tcss style it alone.
+    COMPONENT_CLASSES = {"tree--toggle"}
+
     def __init__(self, **kwargs) -> None:
         super().__init__("/", **kwargs)
         self.show_root = True
@@ -152,6 +242,30 @@ class SitemapTree(Tree):
         else:
             parent.add_leaf(data.label)  # e.g. the "… (truncated)" note — not a link
 
+    def render_label(self, node, base_style, style) -> Text:
+        """Paint the chevron with ``tree--toggle``, leaving the label untouched.
+
+        Textual builds the line as ``ICON + label``, styling the icon with the
+        widget's own base style; re-styling that span is the only way to colour the
+        chevron independently. The icon is a fixed-width prefix (``"▼ "``/``"▶ "``)
+        present only on expandable nodes, so the span is the first ``len(ICON_NODE)``
+        cells. A partial style layers over the base, so only the colour changes.
+
+        The ``_component_styles`` guard is required, not defensive: this also runs
+        pre-mount via ``get_label_width`` (a reactive in ``__init__`` rebuilds the
+        tree lines), and component styles don't exist until the stylesheet is applied
+        on mount — asking for one before that raises ``KeyError``. That early call
+        only measures, and a colour never changes cell width, so skipping is safe.
+        """
+        label = super().render_label(node, base_style, style)
+        if node.allow_expand and "tree--toggle" in self._component_styles:
+            label.stylize(
+                self.get_component_rich_style("tree--toggle", partial=True),
+                0,
+                len(self.ICON_NODE),
+            )
+        return label
+
     def action_open_leaf(self, index: int) -> None:
         if 0 <= index < len(self._leaf_urls):
             self.app.open_url(self._leaf_urls[index])