stackscan 2.7.2__tar.gz → 2.7.3__tar.gz

{stackscan-2.7.2 → stackscan-2.7.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: stackscan
-Version: 2.7.2
+Version: 2.7.3
 Summary: Full web stack, infrastructure, DNS, port, subdomain and CVE analyzer.
 Project-URL: Source, https://github.com/reekeer/stackscan
 Project-URL: Issues, https://github.com/reekeer/stackscan/issues
@@ -26,7 +26,6 @@ Requires-Dist: sigdb>=1.0.2
 Provides-Extra: dev
 Requires-Dist: black>=24.0; extra == 'dev'
 Requires-Dist: build>=1.0; extra == 'dev'
-Requires-Dist: cibuildwheel>=2.16; extra == 'dev'
 Requires-Dist: geoip2>=4.8; extra == 'dev'
 Requires-Dist: pyright>=1.1; extra == 'dev'
 Requires-Dist: pytest>=8.0; extra == 'dev'

{stackscan-2.7.2 → stackscan-2.7.3}/pyproject.toml

@@ -56,7 +56,6 @@ dev = [
   "pytest>=8.0",
   "twine>=5.0",
   "build>=1.0",
-  "cibuildwheel>=2.16",
   "geoip2>=4.8"
 ]
 
@@ -171,18 +170,3 @@ reportMissingTypeStubs = false
 reportOptionalMemberAccess = true
 reportOptionalSubscript = true
 reportGeneralTypeIssues = true
-
-[tool.cibuildwheel]
-build = "cp311-* cp312-* cp313-*"
-skip = "*-musllinux* *-win32"
-test-command = "pytest {project}/tests"
-test-extras = ["dev"]
-
-[tool.cibuildwheel.linux]
-archs = ["x86_64", "aarch64"]
-
-[tool.cibuildwheel.macos]
-archs = ["x86_64", "arm64"]
-
-[tool.cibuildwheel.windows]
-archs = ["AMD64"]

{stackscan-2.7.2 → stackscan-2.7.3}/src/stackscan/__init__.py

@@ -2,4 +2,4 @@ from .core import StackscanSession
 from .utils import normalize_url
 
 __all__ = ["StackscanSession", "__version__", "normalize_url"]
-__version__ = "2.7.2"
+__version__ = "2.7.3"

{stackscan-2.7.2 → stackscan-2.7.3}/src/stackscan/analyzers/cve.py

@@ -101,6 +101,16 @@ _GENERATOR_RE = re.compile(
     "<meta[^>]+name=[\\\"']generator[\\\"'][^>]+content=[\\\"']([^\\\"']+)[\\\"']", re.IGNORECASE
 )
 _SSH_RE = re.compile(r"openssh[\s_/:-](\d+\.\d+(?:p\d+)?)", re.IGNORECASE)
+_VERSION_PREFIX_RE = re.compile(r"\s*[vV]?\d+\.\d")
+_IMPRECISE_VERSION_RE = re.compile(
+    r"\b(?:or|and) (?:later|higher|earlier|newer|above|below|greater)\b", re.IGNORECASE
+)
+
+
+def _usable_port_version(version: str) -> bool:
+    if not _VERSION_PREFIX_RE.match(version):
+        return False
+    return not _IMPRECISE_VERSION_RE.search(version)
 _CORE_COMMIT_RE = re.compile(
     r"([A-Za-z][A-Za-z0-9\s-]*?)\s+Core\s+\(([a-f0-9]{4,})\)", re.IGNORECASE
 )
@@ -276,7 +286,7 @@ def software_from_ports(scan: PortScan | None) -> list[Software]:
                 )
             )
             continue
-        if port.product and port.version:
+        if port.product and port.version and _usable_port_version(port.version):
             out.append(
                 Software(
                     name=port.product.lower().split()[0],
@@ -294,6 +304,11 @@ _SEVERITY_RANK = {"CRITICAL": 0, "HIGH": 1, "MEDIUM": 2, "LOW": 3}
 
 
 def _confidence(version: str, rng: dict[str, str], source: str, backported: bool = False) -> int:
+    if source.startswith("body-text:"):
+        # a bare "name version" scraped from page prose (e.g. a blog post),
+        # not a live banner — kept below the default threshold so it is hidden
+        # unless the user asks for low-confidence matches.
+        return 30
     if backported:
         if "start_incl" in rng or "start_excl" in rng:
             return 35
@@ -337,8 +352,16 @@ def _match_entries(
         return
     if is_commit_hash(version):
         return
+    if len([c for c in re.split("[.\\-_]", version) if c[:1].isdigit()]) < 2:
+        return
+    from_prose = item.source.startswith("body-text:")
     backported = bool(item.os)
-    caveat = "distro backport likely — patchlevel not in banner" if backported else ""
+    if from_prose:
+        caveat = "version mentioned in page text — may not be the running software"
+    elif backported:
+        caveat = "distro backport likely — patchlevel not in banner"
+    else:
+        caveat = ""
     for entry in entries:
         ranges = cast("list[dict[str, str]]", entry.get("ranges") or [])
         hit_rng = next((r for r in ranges if _in_range(version, r)), None)
@@ -357,7 +380,7 @@ def _match_entries(
                 confidence=confidence,
                 summary=str(entry.get("summary", "")),
                 url=entry.get("url"),
-                unconfirmed=backported,
+                unconfirmed=backported or from_prose,
                 caveat=caveat,
             )
             agg[cve_id] = record

{stackscan-2.7.2 → stackscan-2.7.3}/src/stackscan/analyzers/exposure.py

@@ -18,6 +18,26 @@ class ExposureProbe:
     insecure: bool
 
 
+def _looks_like_robots(body: str) -> bool:
+    lowered = body.lower()
+    return any(
+        directive in lowered
+        for directive in ("user-agent:", "disallow:", "allow:", "sitemap:", "crawl-delay:")
+    )
+
+
+def _looks_like_sitemap(body: str) -> bool:
+    lowered = body.lstrip().lower()
+    return lowered.startswith("<?xml") or "<urlset" in lowered or "<sitemapindex" in lowered
+
+
+def _looks_like_security_txt(body: str) -> bool:
+    lowered = body.lower()
+    return any(
+        field in lowered for field in ("contact:", "expires:", "encryption:", "acknowledgments:")
+    )
+
+
 async def _get(session: StackscanSession, url: str, probe: ExposureProbe) -> tuple[int | None, str]:
     try:
         result = await session.fetch(
@@ -39,20 +59,20 @@ async def analyze_exposure(
     urls: dict[str, str] = {}
 
     robots_url = urljoin(base_url, "/robots.txt")
-    robots_status, _ = await _get(session, robots_url, probe)
-    robots = robots_status == 200
+    robots_status, robots_body = await _get(session, robots_url, probe)
+    robots = robots_status == 200 and _looks_like_robots(robots_body)
     if robots:
         urls["robots.txt"] = robots_url
 
     sitemap_url = urljoin(base_url, "/sitemap.xml")
-    sitemap_status, _ = await _get(session, sitemap_url, probe)
-    sitemap = sitemap_status == 200
+    sitemap_status, sitemap_body = await _get(session, sitemap_url, probe)
+    sitemap = sitemap_status == 200 and _looks_like_sitemap(sitemap_body)
     if sitemap:
         urls["sitemap.xml"] = sitemap_url
 
     sec_url = urljoin(base_url, "/.well-known/security.txt")
-    sec_status, _ = await _get(session, sec_url, probe)
-    security_txt = sec_status == 200
+    sec_status, sec_body = await _get(session, sec_url, probe)
+    security_txt = sec_status == 200 and _looks_like_security_txt(sec_body)
     if security_txt:
         urls["security.txt"] = sec_url
 

{stackscan-2.7.2 → stackscan-2.7.3}/src/stackscan/analyzers/generic.py

@@ -93,7 +93,7 @@ _POWERED_BY_PLAIN_RE = re.compile(
 _SERVER_VERSION_RE = re.compile(
     r"\b("
     + "|".join(re.escape(name) for name in sorted(SERVER_NAMES, key=len, reverse=True))
-    + r")[/ ]v?(\d+\.\d+(?:\.\d+){0,2})",
+    + r")([/ ])v?(\d+\.\d+(?:\.\d+){0,2})",
     re.IGNORECASE,
 )
 
@@ -155,6 +155,11 @@ def _software_name(name: str) -> str:
     return _normalize_name(name).lower().replace(" ", "")
 
 
+def _name_detached_from_version(name: str) -> bool:
+    """True when the version sits behind a stopword/other product, not this name."""
+    return _software_name(_clean_product_name(name)) != _software_name(name)
+
+
 def _category(name: str) -> str:
     if _software_name(name) in {n.replace(" ", "") for n in SERVER_NAMES}:
         return "infrastructure"
@@ -195,16 +200,18 @@ def extract_generic_tech(body: str) -> list[Technology]:
 
     for match in _SERVER_VERSION_RE.finditer(body):
         name = _normalize_name(match.group(1))
-        version = match.group(2)
-        remember(name, f"body:{name}/{version}", version)
+        version = match.group(3)
+        source = f"body:{name}/{version}" if match.group(2) == "/" else f"body-text:{name} {version}"
+        remember(name, source, version)
 
     powered_starts: set[int] = set()
     for match in _POWERED_BY_VERSION_RE.finditer(body):
         name = _normalize_name(match.group(1))
-        version = match.group(2)
+        version = match.group(2) if not _name_detached_from_version(name) else None
         powered_starts.add(match.start())
         if name and not _is_noise(name):
-            remember(name, f"body:powered-by {name} {version}", version)
+            evidence = f"body:powered-by {name} {version}" if version else f"body:powered-by {name}"
+            remember(name, evidence, version)
 
     for match in _POWERED_BY_PLAIN_RE.finditer(body):
         if match.start() in powered_starts:
@@ -275,16 +282,18 @@ def extract_generic_software(body: str, location: str = "") -> list[Software]:
 
     for match in _SERVER_VERSION_RE.finditer(body):
         name = _normalize_name(match.group(1))
-        version = match.group(2)
-        add(name, version, f"body:{name}/{version}")
+        version = match.group(3)
+        source = f"body:{name}/{version}" if match.group(2) == "/" else f"body-text:{name} {version}"
+        add(name, version, source)
 
     powered_starts: set[int] = set()
     for match in _POWERED_BY_VERSION_RE.finditer(body):
         name = _normalize_name(match.group(1))
-        version = match.group(2)
+        version = match.group(2) if not _name_detached_from_version(name) else None
         powered_starts.add(match.start())
         if name:
-            add(name, version, f"body:powered-by {name} {version}")
+            evidence = f"body:powered-by {name} {version}" if version else f"body:powered-by {name}"
+            add(name, version, evidence)
 
     for match in _POWERED_BY_PLAIN_RE.finditer(body):
         if match.start() in powered_starts:

{stackscan-2.7.2 → stackscan-2.7.3}/src/stackscan/cli.py

@@ -10,9 +10,10 @@ from collections.abc import Iterable
 from datetime import UTC, datetime
 from functools import lru_cache
 from pathlib import Path
-from typing import Any
+from typing import Any, cast
 
 from rich.console import Console
+from rich.markup import escape
 from rich.progress import Progress
 from rich.table import Table
 
@@ -362,7 +363,7 @@ def _bar(label: str, total: int | None = None) -> Any:
     if _reekeer_progress is None:
         return None
     try:
-        return _reekeer_progress.bar(label, total=total)
+        return cast("Any", _reekeer_progress).bar(label, total=total)
     except Exception:
         # Reporting progress is not what stackscan is for. It must never be the reason a scan fails.
         return None
@@ -710,13 +711,13 @@ def _render_table(reports: list[ScanReport], show_empty: bool) -> None:
             ips.extend(report.network.ipv4)
             ips.extend(report.network.ipv6)
         table.add_row(
-            report.final_url or report.url,
-            ", ".join(ips) if ips else "-",
+            escape(report.final_url or report.url),
+            escape(", ".join(ips)) if ips else "-",
             str(report.status) if report.status is not None else "-",
-            _infra_summary(report),
-            _format_detected(report.by_category()),
+            escape(_infra_summary(report)),
+            escape(_format_detected(report.by_category())),
             _exposure_summary(report),
-            report.error or "",
+            escape(report.error or ""),
         )
     console.print(table)
 
@@ -817,11 +818,11 @@ def _scan_summary(reports: list[ScanReport], elapsed: float) -> str:
 
 
 def _warn(console: Console, message: str) -> None:
-    console.print(f"[{theme.WARN}]{_glyphs().warn}[/] {message}", highlight=False)
+    console.print(f"[{theme.WARN}]{_glyphs().warn}[/] {escape(message)}", highlight=False)
 
 
 def _error(console: Console, message: str) -> None:
-    console.print(f"[{theme.DANGER}]{_glyphs().err}[/] {message}", highlight=False)
+    console.print(f"[{theme.DANGER}]{_glyphs().err}[/] {escape(message)}", highlight=False)
 
 
 def _set_title(title: str) -> None:

{stackscan-2.7.2 → stackscan-2.7.3}/src/stackscan/config/sources.py

@@ -83,8 +83,18 @@ def _looks_like_git(url: str) -> bool:
     return url.endswith(".git") or url.startswith(("git@", "git+", "ssh://"))
 
 
+_GIT_SCHEMES = ("https://", "http://", "ssh://", "git://", "file://")
+
+
 def _normalize_git_url(url: str) -> str:
-    return url[4:] if url.startswith("git+") else url
+    normalized = url[4:] if url.startswith("git+") else url
+    if normalized.startswith("git@") or normalized.startswith(_GIT_SCHEMES):
+        return normalized
+    if "://" not in normalized and "::" not in normalized:
+        expanded = Path(normalized).expanduser()
+        if expanded.exists():
+            return str(expanded)
+    raise SourceError(f"unsupported git url: {url}")
 
 
 def _detect_kind(url: str) -> str:
@@ -385,7 +395,7 @@ def _materialize_git(url: str, dest: Path) -> _Materialized:
     clone_url = _normalize_git_url(url)
     try:
         subprocess.run(
-            ["git", "clone", "--depth", "1", clone_url, str(checkout)],
+            ["git", "clone", "--depth", "1", "--", clone_url, str(checkout)],
             check=True,
             capture_output=True,
             text=True,

{stackscan-2.7.2 → stackscan-2.7.3}/src/stackscan/core/core.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import codecs
 from collections.abc import Iterable
 
 from aiohttp import ClientSession, ClientTimeout
@@ -7,11 +8,23 @@ from aiohttp import ClientSession, ClientTimeout
 from stackscan.net.resolver import build_connector
 from stackscan.types import FetchResult
 
+_MAX_DRAIN_BYTES = 4 * 1024 * 1024
+
 
 def _lower_headers(items: Iterable[tuple[str, str]]) -> dict[str, str]:
     return {key.lower(): value for key, value in items}
 
 
+def _charset(declared: str | None) -> str:
+    if not declared:
+        return "utf-8"
+    try:
+        codecs.lookup(declared)
+    except LookupError:
+        return "utf-8"
+    return declared
+
+
 class StackscanSession:
     def __init__(self) -> None:
         self._session: ClientSession | None = None
@@ -54,10 +67,14 @@ class StackscanSession:
             headers = _lower_headers(header_items)
             raw_headers = [f"{key.lower()}: {value}" for key, value in header_items]
             cookies = resp.headers.getall("Set-Cookie", [])
-            charset = resp.charset or "utf-8"
+            charset = _charset(resp.charset)
             body_bytes = await resp.content.read(max_bytes)
-            while await resp.content.read(8192):
-                pass
+            drained = 0
+            while drained < _MAX_DRAIN_BYTES:
+                chunk = await resp.content.read(8192)
+                if not chunk:
+                    break
+                drained += len(chunk)
             body = body_bytes.decode(charset, errors="replace")
             url_final = str(resp.url)
             version = resp.version

{stackscan-2.7.2 → stackscan-2.7.3}/src/stackscan/export.py

@@ -27,6 +27,15 @@ def to_json(payload: Payload) -> str:
     return json.dumps(payload, ensure_ascii=False, indent=2)
 
 
+def _script_json(value: Any) -> str:
+    return (
+        json.dumps(value, ensure_ascii=False)
+        .replace("<", "\\u003c")
+        .replace(">", "\\u003e")
+        .replace("&", "\\u0026")
+    )
+
+
 def build_graph(reports: list[dict[str, Any]]) -> dict[str, Any]:
     nodes: dict[str, dict[str, Any]] = {}
     edges: set[tuple[str, str, str]] = set()
@@ -407,7 +416,7 @@ def _html_graph(reports: list[dict[str, Any]]) -> str:
     if len(present_types) < 2:
         return ""
 
-    graph_json = json.dumps(
+    graph_json = _script_json(
         {
             "nodes": [
                 {
@@ -420,8 +429,7 @@ def _html_graph(reports: list[dict[str, Any]]) -> str:
                 for n in graph["nodes"]
             ],
             "edges": graph["edges"],
-        },
-        ensure_ascii=False,
+        }
     )
 
     legend_items = "".join(
@@ -438,7 +446,7 @@ def _html_graph(reports: list[dict[str, Any]]) -> str:
 </div>
 <script>
 (function(){{
-  const colors = {json.dumps(type_color)};
+  const colors = {_script_json(type_color)};
   const data = {graph_json};
   const svg = document.getElementById('netgraph');
   const viewport = svg.querySelector('.graph-viewport');
@@ -493,8 +501,14 @@ def _html_graph(reports: list[dict[str, Any]]) -> str:
   }}
 
   function showTip(ev, n) {{
-    const meta = Object.entries(n.meta || {{}}).map(([k,v]) => `${{k}}: ${{v}}`).join('<br>');
-    tooltip.innerHTML = `<strong>${{n.label}}</strong> (${{n.type}})${{meta ? '<br>' + meta : ''}}`;
+    const strong = document.createElement('strong');
+    strong.textContent = n.label;
+    const parts = [strong, document.createTextNode(` (${{n.type}})`)];
+    Object.entries(n.meta || {{}}).forEach(([k, v]) => {{
+      parts.push(document.createElement('br'));
+      parts.push(document.createTextNode(`${{k}}: ${{v}}`));
+    }});
+    tooltip.replaceChildren(...parts);
     tooltip.style.opacity = 1;
     moveTip(ev);
   }}

{stackscan-2.7.2 → stackscan-2.7.3}/src/stackscan/net/fingerprint.py

@@ -92,6 +92,16 @@ def fingerprint_http(raw_response: str) -> tuple[str | None, str | None]:
     return (server, None)
 
 
+def normalize_mysql_version(product: str | None, version: str) -> tuple[str, str, str]:
+    distro = extract_distro(version) or ""
+    if version.startswith("5.5.5-"):
+        version = version[6:]
+        product = "MariaDB"
+    elif "mariadb" in version.lower():
+        product = "MariaDB"
+    return (product or "MySQL", version.split("-", 1)[0] or version, distro)
+
+
 def fingerprint_mysql(data: bytes) -> tuple[str | None, str | None, str | None, bool]:
     if not data or len(data) < 5:
         return (None, None, None, False)
@@ -108,13 +118,5 @@ def fingerprint_mysql(data: bytes) -> tuple[str | None, str | None, str | None,
     version = version.replace("\x00", "").strip()
     if not version:
         return (None, None, None, False)
-    if version.startswith("5.5.5-"):
-        version = version[6:]
-        product = "MariaDB"
-    elif "mariadb" in version.lower():
-        product = "MariaDB"
-    else:
-        product = "MySQL"
-    clean_version = version.split("-", 1)[0]
-    distro = extract_distro(version)
-    return (product, clean_version, distro, False)
+    product, clean_version, distro = normalize_mysql_version(None, version)
+    return (product, clean_version, distro or None, False)

{stackscan-2.7.2 → stackscan-2.7.3}/src/stackscan/net/ports.py

@@ -8,6 +8,7 @@ from stackscan.net.fingerprint import (
     fingerprint_banner,
     fingerprint_http,
     fingerprint_mysql,
+    normalize_mysql_version,
     sanitize_banner,
 )
 from stackscan.types import Port, PortScan
@@ -108,9 +109,9 @@ def _run_nmap(host: str, ports: tuple[int, ...]) -> PortScan | None:
                 continue
             product = info.get("product") or None
             version = info.get("version") or None
-            extra = info.get("extrainfo") or None
-            if extra and version:
-                version = f"{version} ({extra})"
+            os_tag = ""
+            if version and (product or "").lower() in ("mysql", "mariadb"):
+                product, version, os_tag = normalize_mysql_version(product, version)
             found.append(
                 Port(
                     port=int(number),
@@ -120,6 +121,7 @@ def _run_nmap(host: str, ports: tuple[int, ...]) -> PortScan | None:
                     product=product,
                     version=version,
                     host=host,
+                    os=os_tag,
                 )
             )
     found.sort(key=lambda p: p.port)
@@ -227,9 +229,10 @@ async def _http_probe(
         await asyncio.wait_for(writer.drain(), timeout=min(timeout, 2.5))
     except (TimeoutError, OSError):
         return (None, None, "")
-    raw = await _read(reader, timeout)
-    if not raw:
+    data = await _read_bytes(reader, timeout)
+    if not data:
         return (None, None, "")
+    raw = data.decode("utf-8", "replace")
     product, version = fingerprint_http(raw)
     os = _os_from_banner(raw)
     return (product, version, os)
@@ -244,10 +247,10 @@ async def _rtsp_probe(
         await asyncio.wait_for(writer.drain(), timeout=min(timeout, 2.5))
     except (TimeoutError, OSError):
         return (None, None)
-    raw = await _read(reader, timeout)
-    if not raw:
+    data = await _read_bytes(reader, timeout)
+    if not data:
         return (None, None)
-    server = fingerprint_http(raw)
+    server = fingerprint_http(data.decode("utf-8", "replace"))
     if server != (None, None):
         return server
     return ("RTSP", None)