web-scanner 2.2.1__tar.gz → 2.2.2__tar.gz

{web_scanner-2.2.1 → web_scanner-2.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: web-scanner
-Version: 2.2.1
+Version: 2.2.2
 Summary: An async Textual TUI for website reconnaissance — DNS, WHOIS, TLS, security, tech-stack and SEO, with no paid APIs.
 Project-URL: Homepage, https://github.com/iamramizk/web-scanner
 Project-URL: Repository, https://github.com/iamramizk/web-scanner

{web_scanner-2.2.1 → web_scanner-2.2.2}/webscanner/__init__.py

@@ -1,3 +1,3 @@
 """web-scanner v2 — async website reconnaissance TUI package."""
 
-__version__ = "2.2.1"
+__version__ = "2.2.2"

web_scanner-2.2.2/webscanner/cli.py

@@ -0,0 +1,36 @@
+"""Console entry point for the ``webscan`` command.
+
+    webscan example.com
+    python -m webscanner example.com
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+
+from webscanner.helpers import is_valid_url
+from webscanner.ui.app import WebScannerApp
+
+
+def main() -> None:
+    target = sys.argv[1].strip() if len(sys.argv) > 1 else None
+    if target and not is_valid_url(target):
+        print(f"[!] Not a valid target: {target}")
+        raise SystemExit(1)
+    WebScannerApp(target).run()
+    # Textual has already restored the terminal by the time run() returns.
+    # Quitting mid-scan cancels the scan worker's coroutine, but the blocking
+    # work it dispatched via asyncio.to_thread (requests/socket/pydig/Wappalyzer,
+    # each up to ~12–30s of timeout) keeps running in asyncio's default thread
+    # pool. Those are non-daemon threads, so concurrent.futures' atexit handler
+    # joins them on interpreter shutdown and the process hangs after the UI is
+    # gone. os._exit bypasses that join and terminates now — safe here: the
+    # terminal is restored and there is no post-scan state left to flush.
+    sys.stdout.flush()
+    sys.stderr.flush()
+    os._exit(0)
+
+
+if __name__ == "__main__":
+    main()

{web_scanner-2.2.1 → web_scanner-2.2.2}/webscanner/colors.py

@@ -5,6 +5,9 @@ RED = "#FF6C64"
 #: matches the app's primary theme blue ($primary, textual-dark) — progress bar,
 #: selected-tab chip, keybar hints.
 BLUE = "#0178D4"
+#: update-available signal — the status-bar version dot when a newer PyPI release
+#: exists (green = up to date, blue = not yet checked / couldn't check).
+ORANGE = "#F5A623"
 #: muted value/secondary text — matches the unselected-tab colour
 #: ($text-muted = white @ 60% over the dark theme bg).
 MUTED = "#9D9D9D"

{web_scanner-2.2.1 → web_scanner-2.2.2}/webscanner/core/models.py

@@ -70,13 +70,17 @@ class TreeNode:
     site's URL-path hierarchy (``/blog`` → ``/blog/post-1`` …). A node is a branch
     (expandable folder) iff it has ``children``; otherwise it's a leaf (a page). The
     UI turns the root into a Textual ``Tree`` widget. ``total`` (set on the root only)
-    is the count of page URLs behind the tree, shown in the panel subtitle. ``url`` is
-    the full page URL a node stands for (set on leaves), so the UI can make it clickable.
+    is the count of URLs behind the tree, split into ``pages`` + ``assets`` (also
+    root-only), a purely parsed classification by file extension — see
+    ``sitemap._is_asset``; shown in the panel subtitle. ``url`` is the full page URL a
+    node stands for (set on leaves), so the UI can make it clickable.
     """
 
     label: str
     children: list["TreeNode"] = field(default_factory=list)
     total: int | None = None
+    pages: int | None = None
+    assets: int | None = None
     url: str | None = None
 
 

{web_scanner-2.2.1 → web_scanner-2.2.2}/webscanner/modules/sitemap.py

@@ -37,6 +37,32 @@ MAX_URLS = 10000
 #: how deep a chain of nested <sitemapindex> files we follow
 MAX_DEPTH = 6
 
+#: File extensions that mark a sitemap ``<loc>`` as a downloadable **asset**
+#: (image / media / document / archive / static file) rather than a page.
+#: Classification is *pure parsing* — read off the URL path, never fetched.
+#: Anything not listed here (no extension, ``.html``/``.php``, a dynamic route,
+#: a trailing slash) counts as a page, so the bias is toward "page" and odd
+#: dynamic URLs aren't mislabelled.
+ASSET_EXTENSIONS = frozenset({
+    # images
+    "jpg", "jpeg", "png", "gif", "webp", "svg", "bmp", "ico",
+    "tif", "tiff", "avif", "heic", "heif",
+    # audio / video
+    "mp3", "mp4", "wav", "ogg", "oga", "webm", "mov", "avi", "mkv",
+    "flv", "m4a", "m4v", "wmv", "aac", "flac",
+    # documents
+    "pdf", "doc", "docx", "xls", "xlsx", "ppt", "pptx", "odt", "ods",
+    "odp", "rtf", "csv", "tsv", "epub",
+    # archives
+    "zip", "gz", "tgz", "tar", "rar", "7z", "bz2", "xz",
+    # web / static assets
+    "css", "js", "mjs", "map", "wasm", "rss", "atom",
+    # fonts
+    "woff", "woff2", "ttf", "otf", "eot",
+    # binaries / packages
+    "dmg", "exe", "apk", "pkg", "iso", "bin", "deb", "rpm", "msi",
+})
+
 
 class SitemapModule(ScanModule):
     name = "sitemap"
@@ -158,6 +184,17 @@ def _localname(tag: str) -> str:
     return tag.rsplit("}", 1)[-1].lower()
 
 
+def _is_asset(url: str) -> bool:
+    """True if `url`'s path names a downloadable asset (by file extension) rather
+    than a page. Pure parsing — no request is ever made. The extension is taken
+    from the last path segment only (query/fragment ignored); a missing or
+    unknown extension, or a trailing slash, counts as a page."""
+    last = urlparse(url).path.rsplit("/", 1)[-1]
+    if "." not in last:
+        return False
+    return last.rsplit(".", 1)[-1].lower() in ASSET_EXTENSIONS
+
+
 def _segments(url: str, include_host: bool) -> list[str]:
     """Path segments for `url` as tree labels: ``/blog``, ``/post`` … An empty path
     (the homepage) yields ``[]`` — it *is* the ``/`` root, not a child of it. When
@@ -196,6 +233,8 @@ def _url_tree(urls: list[str], truncated: bool) -> TreeNode:
     if truncated:
         root.children.append(TreeNode(label=f"… (truncated at {MAX_URLS} URLs)"))
     root.total = len(urls)
+    root.assets = sum(1 for u in urls if _is_asset(u))
+    root.pages = root.total - root.assets
     _sort_tree(root)
     return root
 

{web_scanner-2.2.1 → web_scanner-2.2.2}/webscanner/net/version_check.py

@@ -62,11 +62,17 @@ def _write_cache(latest: str) -> None:
     path.write_text(json.dumps({"latest": latest, "checked_at": time.time()}))
 
 
-def check_for_update(current: str) -> str | None:
-    """The latest PyPI version if newer than ``current``, else ``None``.
+def update_status(current: str) -> tuple[str, str | None]:
+    """The update state as ``(status, latest)``:
 
-    Silent on any failure — network, parse, or filesystem — since a broken update
-    check must never be visible as a scan problem.
+    - ``("outdated", latest)`` — a newer PyPI release exists;
+    - ``("latest", None)`` — confirmed up to date;
+    - ``("unknown", None)`` — couldn't check (network, parse, or filesystem failure).
+
+    The ``unknown`` case is kept distinct from ``latest`` on purpose: the status-bar
+    dot shows green only for a *confirmed* up-to-date check, and stays neutral (blue)
+    when the check couldn't complete — a broken update check must never look like a
+    scan problem, nor falsely claim "up to date".
     """
     try:
         latest = _cached_latest()
@@ -76,6 +82,18 @@ def check_for_update(current: str) -> str | None:
         if latest is None:
             latest = _fetch_latest()
             _write_cache(latest)
-        return latest if _parse(latest) > _parse(current) else None
     except Exception:  # noqa: BLE001
-        return None
+        return "unknown", None
+    if _parse(latest) > _parse(current):
+        return "outdated", latest
+    return "latest", None
+
+
+def check_for_update(current: str) -> str | None:
+    """The latest PyPI version if newer than ``current``, else ``None``.
+
+    Silent on any failure — network, parse, or filesystem — since a broken update
+    check must never be visible as a scan problem.
+    """
+    status, latest = update_status(current)
+    return latest if status == "outdated" else None

{web_scanner-2.2.1 → web_scanner-2.2.2}/webscanner/ui/activity.py

@@ -25,9 +25,12 @@ from __future__ import annotations
 import re
 from typing import Any, Callable
 
+from urllib.parse import urlparse
+
 from rich.markup import escape
 
-from ..colors import BLUE, GREEN, RED
+from ..colors import GREEN, RED
+from ..net.psl import registrable_domain
 from ..core.context import ScanContext
 from ..core.models import ModuleResult, ModuleStatus, ScanEvent
 from ..core.scanner import PREFETCH, SHARED_IP
@@ -129,6 +132,20 @@ def _dns(result: ModuleResult) -> str:
     return f"{line} · {', '.join(auth)}." if auth else f"{line}."
 
 
+def _registrar_root(url: str | None) -> str:
+    """Registrar's registrable domain from its WHOIS ``registrar_url``.
+
+    ``http://www.hostinger.com`` → ``hostinger.com``. Tolerates a scheme-less
+    value (``www.hostinger.com``) — ``urlparse`` puts that in ``path``, not
+    ``netloc``, so fall back to the whole string as the host. Returns "" when
+    nothing usable (no URL, or no registrable domain resolves)."""
+    if not url:
+        return ""
+    parsed = urlparse(url.strip())
+    host = parsed.netloc or parsed.path
+    return registrable_domain(host.split("/")[0].split("@")[-1])
+
+
 def _whois(result: ModuleResult) -> str:
     data = result.data or {}
     if not data:
@@ -136,7 +153,11 @@ def _whois(result: ModuleResult) -> str:
     if (note := _note(data)) is not None:
         return f"{_DONE}. {_sentence(note)}"
     bits = []
-    if registrar := data.get("registrar"):
+    # Prefer the registrar's root domain (registrable eTLD+1 of its URL) over its
+    # legal name — "hostinger.com" reads truer than "HOSTINGER operations, UAB".
+    # Fall back to the name when there's no URL (sparse ccTLD output).
+    registrar = _registrar_root(data.get("registrar_url")) or data.get("registrar")
+    if registrar:
         # Unlabelled: the Whois prefix and the "expires" half make it obvious enough,
         # and the 10 chars "Registrar " costs are better spent on the name itself.
         bits.append(_esc(registrar))
@@ -237,7 +258,12 @@ def _sitemap(result: ModuleResult) -> str:
         return f"{_DONE}. No sitemap found."
     total = root.total or 0
     suffix = " (truncated)" if total >= MAX_URLS else ""
-    return f"{_DONE}. {_plural(total, 'URL')} found{suffix}."
+    assets = root.assets or 0
+    pages = root.pages if root.pages is not None else total
+    tally = _plural(pages, "page")
+    if assets:
+        tally += f", {_plural(assets, 'asset')}"
+    return f"{_DONE}. {tally} found{suffix}."
 
 
 def _links(result: ModuleResult) -> str:
@@ -338,12 +364,6 @@ def waf(result: ModuleResult) -> str | None:
     return f"{line} · actively blocking." if blocked else f"{line}."
 
 
-def update_available(latest: str) -> str:
-    """A newer release exists on PyPI. Emitted once, after the scan's own closing
-    line, only when a newer version was actually found — see net/version_check.py."""
-    return f"Update: [{BLUE}]v{_esc(latest)} available[/] — run `pipx upgrade web-scanner`."
-
-
 def overall(completed: int, failed: int, total: int, seconds: float) -> str:
     """The closing line. ``seconds`` is wall-clock for the whole scan — modules run
     concurrently, so summing their durations would overstate it several-fold."""

{web_scanner-2.2.1 → web_scanner-2.2.2}/webscanner/ui/app.py

@@ -25,10 +25,11 @@ from textual.message import Message
 from textual.widgets import Input, LoadingIndicator, Static
 
 from .. import __version__
+from ..colors import BLUE, GREEN, ORANGE
 from ..core import AsyncScanner, ModuleStatus, ScanContext, ScanEvent
 from ..core.scanner import PREFETCH, SHARED_IP
 from ..modules import all_modules
-from ..net.version_check import check_for_update
+from ..net.version_check import update_status
 from . import activity
 from .export import export_csvs
 from .tables import UNSET, render_result, render_status
@@ -38,6 +39,11 @@ from .widgets import ActivityLog, MapPanel, SitemapTree, StatusPanel, TabBar, Ta
 _BAR_WIDTH = 22
 _BAR_DIM = "grey30"
 
+#: colour of the status-bar version dot per update state (see net/version_check.py):
+#: blue until the background check answers / if it couldn't, green = up to date,
+#: orange = a newer PyPI release is available.
+_DOT_COLOURS = {"unknown": BLUE, "latest": GREEN, "outdated": ORANGE}
+
 #: below this terminal width the layout collapses to a single column (the fixed map
 #: + Server panels become tabs, activity becomes a tab) — see _apply_narrow.
 _NARROW_MAX = 90
@@ -133,6 +139,20 @@ def _detect_cms(tech_data: object, html: str | None) -> tuple[str, str | None] |
     return tech
 
 
+def _sitemap_subtitle(root: object) -> str:
+    """The Sitemap tab's ``#main`` subtitle: ``"12 pages • 3 assets"`` (pages vs.
+    assets split purely by URL extension — see ``sitemap._is_asset``). Assets are
+    only shown when present; falls back to the raw total if the split is missing."""
+    pages = getattr(root, "pages", None)
+    assets = getattr(root, "assets", None) or 0
+    if pages is None:  # older/other tree without the split — show the plain count
+        return f"{getattr(root, 'total', 0) or 0} Total"
+    parts = [f"{pages} {'page' if pages == 1 else 'pages'}"]
+    if assets:
+        parts.append(f"{assets} {'asset' if assets == 1 else 'assets'}")
+    return " • ".join(parts)
+
+
 class ScanProgress(Message):
     """A ScanEvent surfaced onto the Textual message pump."""
 
@@ -146,10 +166,14 @@ class ScanFinished(Message):
 
 
 class VersionChecked(Message):
-    """Result of the background PyPI update check (see net/version_check.py)."""
+    """Result of the background PyPI update check (see net/version_check.py).
+
+    ``status`` is one of ``"unknown"`` / ``"latest"`` / ``"outdated"`` — it drives
+    the colour of the status-bar version dot.
+    """
 
-    def __init__(self, latest: str | None) -> None:
-        self.latest = latest
+    def __init__(self, status: str) -> None:
+        self.status = status
         super().__init__()
 
 
@@ -212,11 +236,9 @@ class WebScannerApp(App):
         # last result rendered into the sitemap Tree, so switching tabs doesn't
         # rebuild (and re-collapse) it every visit.
         self._tree_result = None
-        # background PyPI update check (see _check_version) — announced once, after
-        # whichever finishes last: the scan or the check itself.
-        self._update_available: str | None = None
-        self._version_checked = False
-        self._update_announced = False
+        # background PyPI update check (see _check_version) — drives the colour of the
+        # status-bar version dot. "unknown" (blue) until the check answers.
+        self._version_status = "unknown"
 
     # ---- layout -----------------------------------------------------------
 
@@ -246,6 +268,9 @@ class WebScannerApp(App):
         self.query_one("#tabs", TabBar).set_selected(self.selected)
         # Establish the layout (wide vs narrow) before the first paint / scan.
         self._apply_narrow(self.size.width < _NARROW_MAX)
+        # Show the version + (blue) update dot straight away; the dot recolours once
+        # the background check answers, and the progress bar reclaims the slot mid-scan.
+        self._refresh_version()
         # own group: the scan worker below is exclusive=True, which cancels every
         # other worker in its group — a shared/default group would kill this one.
         self.run_worker(
@@ -288,26 +313,20 @@ class WebScannerApp(App):
         self._set_keybar(editing=editing)
 
     async def _check_version(self) -> None:
-        latest = await asyncio.to_thread(check_for_update, __version__)
-        self.post_message(VersionChecked(latest))
+        status, _latest = await asyncio.to_thread(update_status, __version__)
+        self.post_message(VersionChecked(status))
 
     def on_version_checked(self, message: VersionChecked) -> None:
-        self._update_available = message.latest
-        self._version_checked = True
-        self._maybe_announce_update()
-
-    def _maybe_announce_update(self) -> None:
-        if (
-            self._update_announced
-            or self._scanning
-            or not self._version_checked
-            or self._update_available is None
-        ):
+        self._version_status = message.status
+        self._refresh_version()
+
+    def _refresh_version(self) -> None:
+        """Draw the version + update dot at the right of the footer, unless a scan is
+        in progress (then the progress bar owns that slot — restored on finish)."""
+        if self._scanning:
             return
-        self._update_announced = True
-        self.query_one("#activity", ActivityLog).add(
-            activity.update_available(self._update_available)
-        )
+        dot = _DOT_COLOURS.get(self._version_status, BLUE)
+        self.query_one("#progress", Static).update(f"v{__version__} [{dot}]•[/]")
 
     # ---- scanning ---------------------------------------------------------
 
@@ -415,8 +434,7 @@ class WebScannerApp(App):
                 self.completed, self.failed, total, time.monotonic() - self._scan_start
             )
         )
-        self.query_one("#progress", Static).update(f"v{__version__}")
-        self._maybe_announce_update()
+        self._refresh_version()
 
     # ---- progress line ----------------------------------------------------
 
@@ -542,9 +560,8 @@ class WebScannerApp(App):
             if self._tree_result is not result:
                 tree.populate(result.data)
                 self._tree_result = result
-            total = result.data.total
-            if total is not None:
-                self.query_one("#main").border_subtitle = f"{total} Total"
+            if result.data.total is not None:
+                self.query_one("#main").border_subtitle = _sitemap_subtitle(result.data)
         elif result.status is ModuleStatus.FAILED:
             self._set_main(f"[red]failed:[/] {result.error}")
         elif result.status is ModuleStatus.EMPTY:

web_scanner-2.2.1/webscanner/cli.py

@@ -1,24 +0,0 @@
-"""Console entry point for the ``webscan`` command.
-
-    webscan example.com
-    python -m webscanner example.com
-"""
-
-from __future__ import annotations
-
-import sys
-
-from webscanner.helpers import is_valid_url
-from webscanner.ui.app import WebScannerApp
-
-
-def main() -> None:
-    target = sys.argv[1].strip() if len(sys.argv) > 1 else None
-    if target and not is_valid_url(target):
-        print(f"[!] Not a valid target: {target}")
-        raise SystemExit(1)
-    WebScannerApp(target).run()
-
-
-if __name__ == "__main__":
-    main()