os-normalizer 0.3.2__py3-none-any.whl

os_normalizer/__init__.py

@@ -0,0 +1,10 @@
+from .models import OSData
+from .os_normalizer import choose_best_fact, normalize_os, merge_os, update_os
+
+__all__ = [
+    "OSData",
+    "choose_best_fact",
+    "normalize_os",
+    "merge_os",
+    "update_os",
+]

os_normalizer/constants.py

@@ -0,0 +1,87 @@
+"""Constants and static lookup tables for the OS fingerprinting package."""
+
+# Architecture synonyms
+ARCH_SYNONYMS = {
+    "x64": "x86_64",
+    "x86_64": "x86_64",
+    "amd64": "x86_64",
+    "x86": "x86",
+    "i386": "x86",
+    "i686": "x86",
+    "aarch64": "arm64",
+    "arm64": "arm64",
+    "armv8": "arm64",
+    "armv7": "arm",
+    "armv7l": "arm",
+    "ppc64le": "ppc64le",
+}
+
+# Windows build map (build number range -> product name, marketing channel)
+WINDOWS_BUILD_MAP = [
+    # Windows 10
+    (10240, 10240, "Windows 10", "1507"),
+    (10586, 10586, "Windows 10", "1511"),
+    (14393, 14393, "Windows 10", "1607"),
+    (15063, 15063, "Windows 10", "1703"),
+    (16299, 16299, "Windows 10", "1709"),
+    (17134, 17134, "Windows 10", "1803"),
+    (17763, 17763, "Windows 10", "1809"),
+    (18362, 18363, "Windows 10", "1903/1909"),
+    (19041, 19045, "Windows 10", "2004/20H2/21H1/21H2/22H2"),
+    # Windows 11
+    (22000, 22000, "Windows 11", "21H2"),
+    (22621, 22630, "Windows 11", "22H2"),
+    (22631, 25999, "Windows 11", "23H2"),
+    (26100, 26199, "Windows 11", "24H2"),
+]
+
+# Windows NT version tuple -> client product (ambiguous NT 6.x split out)
+WINDOWS_NT_CLIENT_MAP = {
+    (4, 0): "Windows NT 4.0",
+    (5, 0): "Windows 2000",
+    (5, 1): "Windows XP",
+    (5, 2): "Windows XP x64/Server 2003",  # NT 5.2 often maps to XP x64 on client
+    (6, 0): "Windows Vista",
+    (6, 1): "Windows 7",
+    (6, 2): "Windows 8",
+    (6, 3): "Windows 8.1",
+    (10, 0): "Windows 10/11",
+}
+
+# Windows NT version tuple -> server product
+WINDOWS_NT_SERVER_MAP = {
+    (4, 0): "Windows NT 4.0 Server",
+    (5, 0): "Windows 2000 Server",
+    (5, 1): "Windows XP/Server 2003",  # rarely used for server detection
+    (5, 2): "Windows Server 2003",
+    (6, 0): "Windows Server 2008",
+    (6, 1): "Windows Server 2008 R2",
+    (6, 2): "Windows Server 2012",
+    (6, 3): "Windows Server 2012 R2",
+    # NT 10.0: Server 2016/2019/2022 detected via explicit names, not NT mapping
+}
+
+# Human readable aliases (macOS codenames)
+MACOS_ALIASES = {
+    "sonoma": "macOS 14",
+    "sequoia": "macOS 15",
+    "ventura": "macOS 13",
+    "monterey": "macOS 12",
+    "big sur": "macOS 11",
+    "bigsur": "macOS 11",
+    "catalina": "macOS 10.15",
+}
+
+# macOS Darwin major version -> (product name, product version, codename)
+MACOS_DARWIN_MAP = {
+    19: ("macOS", "10.15", "Catalina"),
+    20: ("macOS", "11", "Big Sur"),
+    21: ("macOS", "12", "Monterey"),
+    22: ("macOS", "13", "Ventura"),
+    23: ("macOS", "14", "Sonoma"),
+    24: ("macOS", "15", "Sequoia"),
+}
+
+# Cisco train names (used for codename detection)
+CISCO_TRAIN_NAMES = {"Everest", "Fuji", "Gibraltar", "Amsterdam", "Denali"}
+

os_normalizer/cpe.py

@@ -0,0 +1,265 @@
+"""CPE 2.3 generation utilities for OSData.
+
+This aims for pragmatic correctness for common operating systems based on
+the fields present in OSData. Exact dictionary matching for all vendors
+is out of scope; instead we provide a curated mapping for popular OSes.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .models import OSData
+
+_ESCAPE_CHARS = set("\\: ?*(){}[]!\"#$%&'+,/:;<=>@^`|~")
+
+
+def _escape(s: str | None) -> str:
+    if s is None:
+        return "*"
+    if s == "":
+        return "-"
+    if s in ("*", "-"):
+        # Wildcard or N/A tokens are used verbatim in CPE
+        return s
+    out = []
+    for ch in s:
+        if ch in _ESCAPE_CHARS:
+            out.append("\\" + ch)
+        else:
+            out.append(ch)
+    return "".join(out)
+
+
+def _map_vendor_product(p: OSData) -> tuple[str, str, str]:
+    """Return (vendor_token, product_token, strategy).
+
+    strategy controls version/update selection rules.
+    """
+    fam = (p.family or "").lower()
+    vendor = (p.vendor or "").lower() if p.vendor else None
+    product = (p.product or "").lower() if p.product else None
+
+    # Windows
+    if fam == "windows":
+        vtok = "microsoft"
+        prod_map = {
+            "windows 7": "windows_7",
+            "windows 8.1": "windows_8.1",
+            "windows 8": "windows_8",
+            "windows 10": "windows_10",
+            "windows 11": "windows_11",
+            "windows server 2008": "windows_server_2008",
+            "windows server 2008 r2": "windows_server_2008_r2",
+            "windows server 2012": "windows_server_2012",
+            "windows server 2012 r2": "windows_server_2012_r2",
+            "windows server 2016": "windows_server_2016",
+            "windows server 2019": "windows_server_2019",
+            "windows server 2022": "windows_server_2022",
+        }
+        base = prod_map.get(product, "windows")
+        # For Windows 10/11, append channel (e.g., _23h2) when available
+        if base in ("windows_10", "windows_11") and p.channel:
+            base = f"{base}_{p.channel.lower()}"
+        return vtok, base, "windows"
+
+    # macOS
+    if fam == "macos":
+        return "apple", "macos", "macos"
+
+    # Linux distros (use distro when present)
+    if fam == "linux":
+        d = (p.distro or "").lower()
+        if d == "ubuntu":
+            return "canonical", "ubuntu_linux", "ubuntu"
+        if d == "debian":
+            return "debian", "debian_linux", "debian"
+        if d in ("rhel", "redhat", "red_hat"):
+            return "redhat", "enterprise_linux", "rhel"
+        if d in ("sles", "suse", "opensuse"):
+            # Simplify to SLES when ID is sles; otherwise opensuse
+            if d == "sles":
+                return "suse", "linux_enterprise_server", "sles"
+            return "suse", "opensuse", "opensuse"
+        if d == "fedora":
+            return "fedoraproject", "fedora", "fedora"
+        if d in ("amzn", "amazon"):
+            return "amazon", "amazon_linux", "amazon"
+        # Generic Linux fallback
+        return vendor or "linux", product or "linux", "linux"
+
+    # BSDs
+    if fam == "bsd":
+        if product and "freebsd" in product:
+            return "freebsd", "freebsd", "freebsd"
+        if product and "openbsd" in product:
+            return "openbsd", "openbsd", "openbsd"
+        if product and "netbsd" in product:
+            return "netbsd", "netbsd", "netbsd"
+        return vendor or "bsd", product or "bsd", "bsd"
+
+    # Network OS
+    if fam == "network-os":
+        if vendor == "cisco":
+            if product and ("ios xe" in product or "ios-xe" in product):
+                return "cisco", "ios_xe", "ios_xe"
+            if product and ("nx-os" in product or "nxos" in product):
+                return "cisco", "nx-os", "nx_os"
+        if vendor == "juniper":
+            return "juniper", "junos", "junos"
+        if vendor == "fortinet":
+            return "fortinet", "fortios", "fortios"
+        if vendor == "huawei":
+            return "huawei", "vrp", "vrp"
+        if vendor == "netgear":
+            return "netgear", "firmware", "firmware"
+        return vendor or "network", (product or "firmware").replace(" ", "_"), "firmware"
+
+    # Mobile
+    if fam == "android":
+        return "google", "android", "android"
+    if fam == "ios":
+        return "apple", "iphone_os", "ios"
+
+    # Fallback
+    return (vendor or fam or "unknown"), (product or (fam or "unknown")).replace(" ", "_"), fam or "unknown"
+
+
+def _fmt_version(p: OSData, strategy: str) -> tuple[str, str, str]:
+    """Return (version, update, edition) strings per strategy."""
+    maj, minr, pat = p.version_major, p.version_minor, p.version_patch
+    build = p.version_build
+    channel = (p.channel or "").lower() if p.channel else None
+    edition = (p.edition or "").lower() if p.edition else None
+
+    if strategy == "windows":
+        # Product may include channel; version should reflect NT kernel + build when known
+        if p.kernel_version:
+            ver = p.kernel_version
+        elif p.version_build:
+            nt = p.evidence.get("nt_version") if isinstance(p.evidence, dict) else None
+            ver = f"{nt}.{p.version_build}" if nt else p.version_build
+        else:
+            ver = "*"
+        return ver, "*", "*"
+
+    if strategy == "ubuntu":
+        if maj is not None and minr is not None:
+            ver = f"{maj}.{minr:02d}"
+        elif maj is not None:
+            ver = f"{maj}.00"
+        else:
+            ver = "*"
+        return ver, "*", "*"
+
+    if strategy in ("debian", "rhel", "sles", "opensuse", "fedora", "amazon"):
+        if maj is not None and minr is not None and strategy in ("opensuse",):
+            ver = f"{maj}.{minr}"
+        elif maj is not None:
+            ver = str(maj)
+        else:
+            ver = "*"
+        return ver, "*", "*"
+
+    if strategy == "macos":
+        ver = f"{maj}.{minr if minr is not None else 0}" if maj is not None else "*"
+        return ver, "*", "*"
+
+    if strategy in ("ios_xe", "nx_os", "junos"):
+        # Prefer build if present; else compose from parts
+        if build:
+            ver = build.lower()
+        elif maj is not None:
+            if minr is not None and pat is not None:
+                ver = f"{maj}.{minr}.{pat}"
+            elif minr is not None:
+                ver = f"{maj}.{minr}"
+            else:
+                ver = f"{maj}"
+        else:
+            ver = "*"
+        return ver, "*", (edition or "*")
+
+    if strategy == "fortios":
+        if maj is not None and minr is not None and pat is not None:
+            ver = f"{maj}.{minr}.{pat}"
+        elif build:
+            # Fallback if only build available
+            ver = build.split("+")[0]
+        else:
+            ver = "*"
+        return ver, "*", "*"
+
+    if strategy in ("vrp", "firmware"):
+        if build:
+            ver = build
+        elif maj is not None:
+            if minr is not None and pat is not None:
+                ver = f"{maj}.{minr}.{pat}"
+            elif minr is not None:
+                ver = f"{maj}.{minr}"
+            else:
+                ver = f"{maj}"
+        else:
+            ver = "*"
+        return ver, "*", "*"
+
+    # Generic fallback
+    if build:
+        ver = build
+    elif maj is not None:
+        if minr is not None and pat is not None:
+            ver = f"{maj}.{minr}.{pat}"
+        elif minr is not None:
+            ver = f"{maj}.{minr}"
+        else:
+            ver = f"{maj}"
+    else:
+        ver = "*"
+    return ver, "*", "*"
+
+
+def _cpe_target_hw(arch: str | None) -> str:
+    if not arch:
+        return "*"
+    a = arch.lower()
+    if a in ("x86_64", "amd64"):
+        return "x64"
+    if a in ("x86", "i386", "i686"):
+        return "x86"
+    if a in ("arm64", "aarch64"):
+        return "arm64"
+    if a.startswith("arm"):
+        return "arm"
+    return a
+
+
+def build_cpe23(p: OSData) -> str:
+    """Build a cpe:2.3 string from an OSData instance."""
+    part = "o"
+    vendor, product, strategy = _map_vendor_product(p)
+    version, update, edition = _fmt_version(p, strategy)
+
+    # Always lower-case vendor/product tokens; keep version/update as-is case except we lowered some above
+    vendor_token = _escape(vendor.lower())
+    product_token = _escape(product.lower())
+    version_token = _escape(version)
+    update_token = _escape(update)
+    edition_token = _escape(edition)
+
+    fields = [
+        "cpe:2.3",
+        part,
+        vendor_token,
+        product_token,
+        version_token,
+        update_token,
+        edition_token,
+        "*",  # language
+        "*",  # sw_edition
+        "*",  # target_sw
+        _escape(_cpe_target_hw(getattr(p, "arch", None))),  # target_hw
+        "*",  # other
+    ]
+    return ":".join(fields)

os_normalizer/helpers.py

@@ -0,0 +1,107 @@
+"""Utility functions shared across the OS fingerprinting package."""
+
+import re
+from typing import Any
+
+from .constants import ARCH_SYNONYMS
+from .models import OSData
+
+
+def norm_arch(s: str | None) -> str | None:
+    """Normalise an architecture string using ARCH_SYNONYMS."""
+    if not s:
+        return None
+    a = s.strip().lower()
+    return ARCH_SYNONYMS.get(a, a)
+
+
+def parse_semver_like(text: str) -> tuple[int | None, int | None, int | None]:
+    """Extract up to three integer components from a version-like string.
+
+    Returns (major, minor, patch) where missing parts are None.
+    """
+    m = re.search(r"\b(\d+)(?:\.(\d+))?(?:\.(\d+))?\b", text)
+    if not m:
+        return None, None, None
+    major = int(m.group(1))
+    minor = int(m.group(2)) if m.group(2) else None
+    patch = int(m.group(3)) if m.group(3) else None
+    return major, minor, patch
+
+
+def precision_from_parts(
+    major: int | None,
+    minor: int | None,
+    patch: int | None,
+    build: str | None,
+) -> str:
+    """Derive a precision label from version components."""
+    if build:
+        return "build"
+    if patch is not None:
+        return "patch"
+    if minor is not None:
+        return "minor"
+    if major is not None:
+        return "major"
+    return "product"
+
+
+def canonical_key(p: OSData) -> str:
+    """Generate a deterministic key for an OSData instance.
+
+    The function expects the object to have vendor, product, version_* and edition fields.
+    """
+    vendor = (p.vendor or "-").lower()
+    product = (p.product or "-").lower()
+    version = ".".join([str(x) for x in [p.version_major, p.version_minor, p.version_patch] if x is not None]) or "-"
+    edition = (p.edition or "-").lower()
+    codename = (p.codename or "-").lower()
+    return f"{vendor}:{product}:{version}:{edition}:{codename}"
+
+
+# Regex for extracting an architecture token from free-form text
+ARCH_TEXT_RE = re.compile(r"\b(x86_64|amd64|x64|x86|i386|i686|arm64|aarch64|armv8|armv7l?|ppc64le)\b", re.IGNORECASE)
+
+
+def extract_arch_from_text(text: str) -> str | None:
+    """Fallback architecture extraction from arbitrary text."""
+    m = ARCH_TEXT_RE.search(text)
+    if not m:
+        return None
+    raw = m.group(1).lower()
+    return ARCH_SYNONYMS.get(raw, raw)
+
+
+def parse_os_release(blob_text: str) -> dict[str, Any]:
+    """Parse the contents of an /etc/os-release style file.
+
+    Returns a dict with selected keys (ID, ID_LIKE, PRETTY_NAME, VERSION_ID, VERSION_CODENAME).
+    """
+    out: dict[str, Any] = {}
+    for line in blob_text.splitlines():
+        clean = line.strip()
+        if not clean or clean.startswith("#") or "=" not in clean:
+            continue
+        k, v = clean.split("=", 1)
+        k = k.strip().upper()
+        if k == "ID_LIKE":
+            out[k] = [s.strip().lower() for s in re.split(r"[ ,]+", v.strip("\"'").strip()) if s]
+        else:
+            out[k] = v.strip("\"'")
+    return out
+
+
+def update_confidence(p: OSData, precision: str) -> None:
+    """Boost confidence based on the determined precision level.
+
+    The mapping mirrors the original ad-hoc values used throughout the monolithic file.
+    """
+    boost_map = {
+        "build": 0.85,
+        "patch": 0.80,
+        "minor": 0.75,
+        "major": 0.70,
+        "product": 0.60,
+    }
+    p.confidence = max(p.confidence, boost_map.get(precision, 0.5))

os_normalizer/models.py

@@ -0,0 +1,159 @@
+"""Data model definitions for OS fingerprinting."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field, fields as dataclass_fields
+from typing import Any
+
+
+@dataclass
+class OSData:
+    """Structured representation of a parsed operating system."""
+
+    # Core identity
+    family: str | None = None  # windows, linux, macos, ios, android, bsd, network-os
+    vendor: str | None = None  # Microsoft, Apple, Canonical, Cisco, Juniper, Fortinet, Huawei, Netgear…
+    product: str | None = None  # Windows 11, Ubuntu, macOS, IOS XE, Junos, FortiOS, VRP, Firmware
+    edition: str | None = None  # Pro/Enterprise/LTSC; universalk9/ipbase; etc.
+    codename: str | None = None  # Sequoia; Ubuntu codename; Cisco train
+    channel: str | None = None  # LTS/Beta/GA/R3-S3 etc.
+
+    # Versions
+    version_major: int | None = None
+    version_minor: int | None = None
+    version_patch: int | None = None
+    version_build: str | None = None  # Windows build; network image tag
+
+    # Kernel / image details
+    kernel_name: str | None = None
+    kernel_version: str | None = None
+    arch: str | None = None
+    distro: str | None = None
+    like_distros: list[str] = field(default_factory=list)
+    pretty_name: str | None = None
+
+    # Network device extras
+    hw_model: str | None = None
+    build_id: str | None = None
+
+    # Meta information
+    precision: str = "unknown"  # family|product|major|minor|patch|build
+    confidence: float = 0.0
+    evidence: dict[str, Any] = field(default_factory=dict)
+
+    # Canonical key for deduplication / indexing
+    os_key: str | None = field(default=None, compare=False)
+
+    def __str__(self) -> str:  # pragma: no cover - formatting helper
+        parts: list[str] = []
+
+        # Prefer vendor + product; fallback to pretty_name; then family
+        name_bits = [x for x in (self.vendor, self.product) if x]
+        if name_bits:
+            parts.append(" ".join(name_bits))
+        elif self.pretty_name:
+            parts.append(self.pretty_name)
+        else:
+            parts.append(self.family or "Unknown OS")
+
+        # Version string (major[.minor[.patch]]) and optional build
+        ver_chunks: list[str] = []
+        if self.version_major is not None:
+            ver = str(self.version_major)
+            if self.version_minor is not None:
+                ver += f".{self.version_minor}"
+                if self.version_patch is not None:
+                    ver += f".{self.version_patch}"
+            ver_chunks.append(ver)
+        if self.version_build:
+            ver_chunks.append(f"build {self.version_build}")
+        if ver_chunks:
+            parts.append(" ".join(ver_chunks))
+
+        # Edition (e.g., Enterprise, LTSC)
+        if self.edition:
+            parts.append(self.edition)
+
+        # Codename and/or channel in parentheses
+        codchan = ", ".join([x for x in (self.codename, self.channel) if x])
+        if codchan:
+            parts.append(f"({codchan})")
+
+        # Architecture
+        if self.arch:
+            parts.append(self.arch)
+
+        # Kernel info
+        kernel_bits = " ".join([x for x in (self.kernel_name, self.kernel_version) if x])
+        if kernel_bits:
+            parts.append(f"[kernel: {kernel_bits}]")
+
+        # Hardware model (common for network devices)
+        if self.hw_model:
+            parts.append(f"[hw: {self.hw_model}]")
+
+        # Separate build identifier if distinct from version_build
+        if self.build_id and self.build_id != self.version_build:
+            parts.append(f"[build: {self.build_id}]")
+
+        # Precision/confidence summary
+        if self.precision and self.precision != "unknown":
+            parts.append(f"{{{self.precision}:{self.confidence:.2f}}}")
+        elif self.confidence:
+            parts.append(f"{{{self.confidence:.2f}}}")
+
+        return " ".join(parts)
+
+    def __repr__(self) -> str:  # pragma: no cover - formatting helper
+        # Delegate to __str__ for concise, human-friendly debug output
+        return f"OSData({str(self)})"
+
+    def full(self, none_str="<None>") -> str:  # pragma: no cover - formatting helper
+        """Return all fields in a neat two-column, aligned layout.
+
+        Example:
+        family        : linux
+        vendor        : Canonical
+        ...
+        If a field is None, prints "<None>" or none_val.
+        """
+        # Collect (name, value) pairs in declared order
+        rows: list[tuple[str, str]] = []
+        for f in dataclass_fields(self):
+            name = f.name
+            val = getattr(self, name)
+            if val is None:
+                sval = none_str
+            elif name == "confidence" and isinstance(val, (int, float)):
+                sval = f"{float(val):.2f}"
+            elif isinstance(val, list):
+                sval = ", ".join(str(x) for x in val)
+            elif isinstance(val, dict):
+                # Shallow, compact dict repr with sorted keys for stability
+                items = ", ".join(f"{k}={val[k]!r}" for k in sorted(val))
+                sval = "{" + items + "}"
+            else:
+                sval = str(val)
+            rows.append((name, sval))
+
+        width = max(len(name) for name, _ in rows) if rows else 0
+        lines = [f"{name:<{width}} : {sval}" for name, sval in rows]
+        return "\n".join(lines)
+
+
+if __name__ == "__main__":
+    x = OSData(
+        family="linux",
+        vendor="Fedora Project",
+        product="Fedora Linux",
+        version_major=33,
+        kernel_name="linux",
+        kernel_version="5.4.0-70-generic",
+        distro="fedora",
+        like_distros=[],
+        pretty_name="Fedora Linux",
+        precision="major",
+        confidence=0.7,
+        evidence={"hit": "linux"},
+    )
+    print("Normal:", x, "\nFull:", x.full(none_str=""), sep="\n", end="\n\n")