robotframework-snapshot 0.1.0__py3-none-any.whl

SnapshotLibrary/serializers.py

@@ -0,0 +1,148 @@
+"""Turn arbitrary values into stable, readable text."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Mapping, Set
+from datetime import date, datetime, time
+from decimal import Decimal
+from pathlib import PurePath
+from typing import Any, Tuple
+from xml.dom import minidom
+from xml.etree import ElementTree
+
+TEXT = "txt"
+JSON = "json"
+XML = "xml"
+FORMATS = ("auto", "text", "json", "xml")
+
+
+def normalize_text(text: str) -> str:
+    """Normalise line endings and end the text with exactly one newline."""
+    text = text.replace("\r\n", "\n").replace("\r", "\n")
+    return text.rstrip("\n") + "\n"
+
+
+def to_plain(value: Any) -> Any:
+    """Reduce ``value`` to JSON-compatible types with a deterministic layout."""
+    if value is None or isinstance(value, (bool, int, float, str)):
+        return value
+    if isinstance(value, Mapping):
+        return {str(key): to_plain(item) for key, item in value.items()}
+    if isinstance(value, (list, tuple)):
+        return [to_plain(item) for item in value]
+    if isinstance(value, Set):
+        return sorted((to_plain(item) for item in value), key=repr)
+    if isinstance(value, (datetime, date, time)):
+        return value.isoformat()
+    if isinstance(value, bytes):
+        return value.decode("utf-8", errors="replace")
+    if isinstance(value, (Decimal, PurePath)):
+        return str(value)
+    return str(value)
+
+
+def _is_scalar(value: Any) -> bool:
+    return value is None or isinstance(value, (bool, int, float, str))
+
+
+def _scalar(value: Any) -> str:
+    return json.dumps(value, ensure_ascii=False)
+
+
+def _dump(value: Any, indent: int, in_list: bool) -> str:
+    pad, inner = "  " * indent, "  " * (indent + 1)
+    if isinstance(value, dict):
+        if not value:
+            return "{}"
+        items = [
+            f"{inner}{_scalar(key)}: {_dump(value[key], indent + 1, False)}"
+            for key in sorted(value)
+        ]
+        return "{\n" + ",\n".join(items) + "\n" + pad + "}"
+    if isinstance(value, list):
+        if not value:
+            return "[]"
+        # A row of a table: a list of plain values inside another list.
+        # Kept on one line, so a changed row is one changed line in the diff.
+        if in_list and all(_is_scalar(item) for item in value):
+            return "[" + ", ".join(_scalar(item) for item in value) + "]"
+        items = [f"{inner}{_dump(item, indent + 1, True)}" for item in value]
+        return "[\n" + ",\n".join(items) + "\n" + pad + "]"
+    return _scalar(value)
+
+
+def dump_json(data: Any) -> str:
+    """Valid JSON with sorted keys, two-space indent and one table row per line."""
+    return _dump(data, 0, False) + "\n"
+
+
+def _is_xml_element(value: Any) -> bool:
+    if isinstance(value, ElementTree.ElementTree):
+        return True
+    if type(value).__module__.startswith("lxml"):
+        return hasattr(value, "tag") or hasattr(value, "getroot")
+    return isinstance(value, ElementTree.Element)
+
+
+def _element_to_string(value: Any) -> str:
+    if type(value).__module__.startswith("lxml"):
+        from lxml import etree
+
+        return etree.tostring(value, encoding="unicode")
+    if isinstance(value, ElementTree.ElementTree):
+        value = value.getroot()
+    return ElementTree.tostring(value, encoding="unicode")
+
+
+def canonical_xml(source: str) -> str:
+    """Canonical XML (C14N 2.0), indented two spaces per level.
+
+    Attributes are sorted, comments, the XML declaration and the doctype are
+    dropped, and whitespace around text and between elements is not
+    significant.
+    """
+    try:
+        canonical = ElementTree.canonicalize(source, strip_text=True)
+    except ElementTree.ParseError as error:
+        raise ValueError(f"format=xml was given but the value is not valid XML: {error}") from None
+    return minidom.parseString(canonical).documentElement.toprettyxml(indent="  ")
+
+
+def to_data(value: Any, fmt: str = "auto") -> Tuple[Any, str]:
+    """Return ``(data, extension)``.
+
+    ``data`` is a string for text and XML snapshots and a JSON-compatible
+    structure for JSON snapshots. Masking by JSONPath happens on that structure before
+    it is written out with `render`.
+    """
+    fmt = (fmt or "auto").lower()
+    if fmt not in FORMATS:
+        raise ValueError(f"Unknown snapshot format '{fmt}'. Use one of: {', '.join(FORMATS)}.")
+    if fmt in ("auto", "xml") and _is_xml_element(value):
+        return canonical_xml(_element_to_string(value)), XML
+    if fmt == "xml":
+        if not isinstance(value, (str, bytes)):
+            raise ValueError(f"format=xml needs an XML string or element, got {type(value).__name__}.")
+        # bytes go to the parser as they are, so an encoding declaration is honoured
+        return canonical_xml(value), XML
+    if isinstance(value, bytes) and fmt != "json":
+        value = value.decode("utf-8", errors="replace")
+    if fmt == "text":
+        return (value if isinstance(value, str) else str(value)), TEXT
+    if fmt == "json":
+        if isinstance(value, (str, bytes)):
+            try:
+                value = json.loads(value)
+            except ValueError as error:
+                raise ValueError(f"format=json was given but the value is not valid JSON: {error}") from None
+        return to_plain(value), JSON
+    if isinstance(value, str):
+        return value, TEXT
+    return to_plain(value), JSON
+
+
+def render(data: Any, extension: str) -> str:
+    if extension == JSON:
+        return dump_json(data)
+    return normalize_text(data)

SnapshotLibrary/store.py

@@ -0,0 +1,60 @@
+"""Where snapshots live on disk and how they are read and written."""
+
+from __future__ import annotations
+
+import os
+import re
+from pathlib import Path
+from typing import Optional
+
+SNAPSHOT_DIR_NAME = "__snapshots__"
+_UNSAFE = re.compile(r"[^\w.-]+", re.UNICODE)
+
+
+def sanitize(name: str) -> str:
+    """Reduce a suite, test or snapshot name to a safe file name part."""
+    cleaned = _UNSAFE.sub("_", str(name)).strip(".")
+    if not cleaned.strip("_"):
+        raise ValueError(f"Name '{name}' contains no characters usable in a file name.")
+    return cleaned
+
+
+def snapshot_path(
+    base_directory: Path,
+    suite: str,
+    test: str,
+    extension: str,
+    name: Optional[str] = None,
+    index: int = 1,
+    shared: bool = False,
+) -> Path:
+    """``<base>/<suite>/<test>[__<name>|__<n>].<ext>``
+
+    The first unnamed snapshot of a test has no suffix, later unnamed ones are
+    numbered from 2 in call order. A shared snapshot is stored under its name
+    alone, ``<base>/<suite>/<name>.<ext>``, so several tests can use it.
+    """
+    if shared:
+        if not name:
+            raise ValueError("A shared snapshot needs a name: use name=... together with shared=True.")
+        return Path(base_directory) / sanitize(suite) / f"{sanitize(name)}.{extension}"
+    stem = sanitize(test)
+    if name:
+        stem += "__" + sanitize(name)
+    elif index > 1:
+        stem += f"__{index}"
+    return Path(base_directory) / sanitize(suite) / f"{stem}.{extension}"
+
+
+def read(path: Path) -> str:
+    with open(path, "r", encoding="utf-8", newline="") as file:
+        return file.read().replace("\r\n", "\n").replace("\r", "\n")
+
+
+def write(path: Path, text: str) -> None:
+    """Write atomically, so a parallel process never reads a half-written snapshot."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
+    with open(temporary, "w", encoding="utf-8", newline="\n") as file:
+        file.write(text)
+    os.replace(temporary, path)

SnapshotLibrary/unused.py

@@ -0,0 +1,191 @@
+"""Find snapshot files that no test used.
+
+Every suite writes a small usage record to ``${OUTPUT_DIR}/snapshot_usage/``.
+The library warns at the end of a suite when it can judge that suite by
+itself, and ``python -m SnapshotLibrary unused <output dir>`` merges the
+records of a whole run, which is what a parallel run with pabot needs.
+
+A suite is only judged when every one of its tests ran and passed. A test
+that was filtered out, skipped or failed before reaching its snapshot would
+otherwise make that snapshot look unused.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Dict, Iterable, List, Optional, Set
+
+from . import store
+
+USAGE_DIR_NAME = "snapshot_usage"
+PASS = "PASS"
+
+
+@dataclass
+class SuiteUsage:
+    """What one suite did with snapshots, in one process."""
+
+    source: str
+    total_tests: Optional[int]
+    ran: Dict[str, str] = field(default_factory=dict)
+    used: Set[str] = field(default_factory=set)
+    directories: Set[str] = field(default_factory=set)
+    suite_passed: bool = True
+
+    @property
+    def complete(self) -> bool:
+        if self.total_tests is None or not self.suite_passed:
+            return False
+        if len(self.ran) < self.total_tests:
+            return False
+        return all(status == PASS for status in self.ran.values())
+
+    def unused(self) -> List[Path]:
+        return find_unused(self.directories, self.used)
+
+    def merge(self, other: "SuiteUsage") -> None:
+        if self.total_tests is None:
+            self.total_tests = other.total_tests
+        for name, status in other.ran.items():
+            # a rerun that passes wins over an earlier failure
+            if self.ran.get(name) != PASS:
+                self.ran[name] = status
+        self.used |= other.used
+        self.directories |= other.directories
+        self.suite_passed = self.suite_passed and other.suite_passed
+
+    def to_json(self) -> dict:
+        return {
+            "source": self.source,
+            "total_tests": self.total_tests,
+            "ran": self.ran,
+            "used": sorted(self.used),
+            "directories": sorted(self.directories),
+            "suite_passed": self.suite_passed,
+        }
+
+    @classmethod
+    def from_json(cls, data: dict) -> "SuiteUsage":
+        return cls(
+            source=data["source"],
+            total_tests=data.get("total_tests"),
+            ran=dict(data.get("ran", {})),
+            used=set(data.get("used", [])),
+            directories=set(data.get("directories", [])),
+            suite_passed=bool(data.get("suite_passed", True)),
+        )
+
+
+def _key(path) -> str:
+    return os.path.normcase(os.path.abspath(str(path)))
+
+
+def find_unused(directories: Iterable, used: Iterable) -> List[Path]:
+    """Files in ``directories`` that are not in ``used``. Hidden files are ignored."""
+    used_keys = {_key(path) for path in used}
+    unused: List[Path] = []
+    for directory in sorted({_key(d): Path(d) for d in directories}.values()):
+        if not directory.is_dir():
+            continue
+        for entry in sorted(directory.iterdir()):
+            if entry.is_file() and not entry.name.startswith(".") and _key(entry) not in used_keys:
+                unused.append(entry)
+    return unused
+
+
+def write_record(output_directory, usage: SuiteUsage) -> Path:
+    directory = Path(output_directory) / USAGE_DIR_NAME
+    name = store.sanitize(Path(usage.source).stem or "suite")
+    target = directory / f"{name}-{os.getpid()}-{abs(hash(usage.source)) % 10**8}.json"
+    store.write(target, json.dumps(usage.to_json(), indent=2) + "\n")
+    return target
+
+
+def clear_records(output_directory) -> None:
+    directory = Path(output_directory) / USAGE_DIR_NAME
+    if directory.is_dir():
+        for entry in directory.glob("*.json"):
+            try:
+                entry.unlink()
+            except OSError:
+                pass
+
+
+def read_records(output_directory) -> List[SuiteUsage]:
+    """All usage records below ``output_directory``, merged per suite file.
+
+    The search is recursive, so the records pabot leaves in
+    ``pabot_results/<n>/snapshot_usage/`` are found from the main output dir.
+    """
+    merged: Dict[str, SuiteUsage] = {}
+    for record in sorted(Path(output_directory).rglob(f"{USAGE_DIR_NAME}/*.json")):
+        try:
+            usage = SuiteUsage.from_json(json.loads(record.read_text(encoding="utf-8")))
+        except (OSError, ValueError, KeyError):
+            continue
+        key = _key(usage.source)
+        if key in merged:
+            merged[key].merge(usage)
+        else:
+            merged[key] = usage
+    return list(merged.values())
+
+
+def orphan_directories(usages: Iterable[SuiteUsage]) -> List[Path]:
+    """Snapshot directories in the default location whose suite file is gone.
+
+    Looks at every ``__snapshots__`` directory the run touched and reports
+    sub-directories that match no suite file next to it. This catches a
+    renamed or deleted suite, which leaves no usage record at all.
+    """
+    bases = set()
+    for usage in usages:
+        for directory in usage.directories:
+            base = Path(directory).parent
+            if base.name == store.SNAPSHOT_DIR_NAME:
+                bases.add(base)
+    orphans: List[Path] = []
+    for base in sorted(bases):
+        suite_directory = base.parent
+        expected = {"__init__"}
+        for entry in suite_directory.iterdir():
+            if entry.is_file():
+                try:
+                    expected.add(store.sanitize(entry.stem))
+                except ValueError:
+                    pass
+        for entry in sorted(base.iterdir()):
+            if entry.is_dir() and entry.name not in expected:
+                orphans.append(entry)
+    return orphans
+
+
+@dataclass
+class Report:
+    unused: List[Path]
+    orphan_directories: List[Path]
+    checked: List[str]
+    not_checked: List[str]
+
+    @property
+    def clean(self) -> bool:
+        return not self.unused and not self.orphan_directories
+
+
+def report(output_directory) -> Optional[Report]:
+    usages = read_records(output_directory)
+    if not usages:
+        return None
+    unused: List[Path] = []
+    checked: List[str] = []
+    not_checked: List[str] = []
+    for usage in sorted(usages, key=lambda u: u.source):
+        if usage.complete:
+            checked.append(usage.source)
+            unused.extend(usage.unused())
+        else:
+            not_checked.append(usage.source)
+    return Report(unused, orphan_directories(usages), checked, not_checked)

SnapshotLibrary/version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

SnapshotLibrary/xmlpath.py

@@ -0,0 +1,74 @@
+"""Mask parts of an XML document by XPath before a snapshot is taken.
+
+Uses the XPath subset of ``xml.etree.ElementTree``: ``.//tag``, ``a/b``,
+``*``, ``[@attr]``, ``[@attr='value']``, ``[tag]`` and ``[1]``. A path that
+ends in ``/@name`` masks that attribute instead of the element. Namespace
+prefixes declared in the document can be used in the path, for example
+``.//soap:Body``.
+"""
+
+from __future__ import annotations
+
+import io
+from typing import Dict, Iterable
+from xml.etree import ElementTree
+
+IGNORED = "<IGNORED>"
+
+
+def _namespaces(source: str) -> Dict[str, str]:
+    found: Dict[str, str] = {}
+    for _, (prefix, uri) in ElementTree.iterparse(io.StringIO(source), events=("start-ns",)):
+        found.setdefault(prefix, uri)
+    return found
+
+
+def _split(path: str):
+    path = path.strip()
+    if not path:
+        raise ValueError("An empty XPath selects nothing.")
+    if path.startswith("/"):
+        path = "." + path
+    head, _, last = path.rpartition("/")
+    if last.startswith("@"):
+        return (head or "."), last[1:]
+    return path, None
+
+
+def _expand(name: str, namespaces: Dict[str, str]) -> str:
+    prefix, colon, local = name.partition(":")
+    if colon and prefix in namespaces:
+        return f"{{{namespaces[prefix]}}}{local}"
+    return name
+
+
+def mask(source: str, paths: Iterable[str], placeholder: str = IGNORED) -> str:
+    """Return ``source`` with every element or attribute selected by ``paths`` masked.
+
+    A masked element keeps its tag and attributes, but its content is
+    replaced by ``placeholder``.
+    """
+    namespaces = _namespaces(source)
+    root = ElementTree.fromstring(source)
+    for path in paths:
+        element_path, attribute = _split(path)
+        try:
+            matches = ElementTree.ElementTree(root).findall(element_path, namespaces)
+        except (SyntaxError, KeyError, TypeError) as error:
+            raise ValueError(f"Unsupported XPath '{path}': {error}") from None
+        for element in matches:
+            if attribute is None:
+                for child in list(element):
+                    element.remove(child)
+                element.text = placeholder
+            else:
+                key = _expand(attribute, namespaces)
+                if key in element.attrib:
+                    element.set(key, placeholder)
+    # Keep the document's own prefixes instead of ns0, ns1, ...
+    for prefix, uri in namespaces.items():
+        try:
+            ElementTree.register_namespace(prefix, uri)
+        except ValueError:  # a prefix such as ns0 is reserved; ElementTree picks one itself
+            pass
+    return ElementTree.tostring(root, encoding="unicode")