python-docx-oss 0.3.2__py3-none-any.whl → 0.4.0__py3-none-any.whl

docx/__init__.py

@@ -9,14 +9,16 @@ from __future__ import annotations
 from typing import TYPE_CHECKING, Type
 
 from docx.api import Document
+from docx.inspection import FingerprintPolicy, fingerprint, inspect
+from docx.transforms import normalize, repair
 
 if TYPE_CHECKING:
     from docx.opc.part import Part
 
-__version__ = "0.3.2"
+__version__ = "0.4.0"
 
 
-__all__ = ["Document"]
+__all__ = ["Document", "FingerprintPolicy", "fingerprint", "inspect", "normalize", "repair"]
 
 
 # -- register custom Part classes with opc package reader --

docx/inspection/__init__.py

@@ -0,0 +1,24 @@
+"""Read-only package inspection and configurable content fingerprints."""
+
+from docx.inspection.fingerprint import FingerprintPolicy, fingerprint
+from docx.inspection.reader import inspect
+from docx.inspection.result import (
+    Features,
+    Finding,
+    PackageInspection,
+    PackageInventory,
+    PartInventory,
+    RelationshipInventory,
+)
+
+__all__ = [
+    "Features",
+    "Finding",
+    "FingerprintPolicy",
+    "PackageInspection",
+    "PackageInventory",
+    "PartInventory",
+    "RelationshipInventory",
+    "fingerprint",
+    "inspect",
+]

docx/inspection/features.py

@@ -0,0 +1,157 @@
+"""XML event targets keep no element tree, including for very large story parts."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+
+from docx.inspection.result import Features
+
+W_NAMESPACES = (
+    "http://schemas.openxmlformats.org/wordprocessingml/2006/main",
+    "http://purl.oclc.org/ooxml/wordprocessingml/main",
+)
+
+
+def word_tag(tag: str) -> str:
+    for namespace in W_NAMESPACES:
+        prefix = "{" + namespace + "}"
+        if tag.startswith(prefix):
+            return tag[len(prefix) :]
+    return ""
+
+
+class _FieldCommand:
+    """Match only the first token, keeping constant state across parser callbacks."""
+
+    def __init__(self):
+        self._length = 0
+        self._done = False
+        self.matched = False
+
+    def feed(self, text: str) -> None:
+        for char in text:
+            if self._done:
+                break
+            if char.isspace():
+                if self._length:
+                    self.finish()
+            elif self._length < len("HYPERLINK") and char.upper() == "HYPERLINK"[self._length]:
+                self._length += 1
+            else:
+                self._done = True
+
+    def finish(self) -> None:
+        if not self._done:
+            self.matched = self._length == len("HYPERLINK")
+            self._done = True
+
+
+class XmlFacts:
+    """Observe field instructions across runs, without retaining instruction text."""
+
+    def __init__(self, scan_features: bool = False):
+        self.scan_features = scan_features
+        self.field_hyperlink = False
+        self.split_hyperlink_runs = False
+        self._stack: list[str] = []
+        self._hyperlinks: list[tuple[int, int]] = []
+        self._instruction = False
+        self._fields: list[_FieldCommand] = []
+        self._loose_command = _FieldCommand()
+
+    def _finish_command(self, command: _FieldCommand) -> None:
+        command.finish()
+        self.field_hyperlink |= command.matched
+
+    def start(self, tag: str, attrs: Mapping[str, str]) -> None:
+        name = word_tag(tag) if self.scan_features else ""
+        self._stack.append(name)
+        if name == "hyperlink":
+            self._hyperlinks.append((len(self._stack), 0))
+        elif name == "r" and self._hyperlinks:
+            depth, count = self._hyperlinks[-1]
+            if len(self._stack) == depth + 1:
+                self._hyperlinks[-1] = (depth, count + 1)
+                self.split_hyperlink_runs |= count >= 1
+        elif name == "fldChar":
+            namespace = tag[: tag.index("}") + 1]
+            kind = attrs.get(namespace + "fldCharType")
+            if kind == "begin":
+                self._fields.append(_FieldCommand())
+            elif kind in ("separate", "end"):
+                command = self._fields[-1] if self._fields else self._loose_command
+                self._finish_command(command)
+                if kind == "end":
+                    if self._fields:
+                        self._fields.pop()
+                    self._loose_command = _FieldCommand()
+        elif name == "instrText":
+            self._instruction = True
+        elif name == "fldSimple":
+            namespace = tag[: tag.index("}") + 1]
+            command = _FieldCommand()
+            command.feed(attrs.get(namespace + "instr", ""))
+            self._finish_command(command)
+        elif name == "p":
+            self._loose_command = _FieldCommand()
+
+    def end(self, tag: str) -> None:
+        name = self._stack.pop()
+        if name == "hyperlink":
+            self._hyperlinks.pop()
+        elif name == "instrText":
+            self._instruction = False
+        elif name == "p":
+            self._finish_command(self._loose_command)
+            self._loose_command = _FieldCommand()
+
+    def data(self, text: str) -> None:
+        if self._instruction:
+            command = self._fields[-1] if self._fields else self._loose_command
+            command.feed(text)
+            self.field_hyperlink |= command.matched
+
+    def start_ns(self, prefix: str | None, uri: str) -> None:
+        pass
+
+    def end_ns(self, prefix: str | None) -> None:
+        pass
+
+    def comment(self, text: str) -> None:
+        pass
+
+    def pi(self, target: str, data: str) -> None:
+        pass
+
+    def doctype(self, name: str | None, public_id: str | None, system_id: str | None) -> None:
+        raise ValueError("DTDs are not supported in package XML")
+
+    def close(self) -> Features:
+        self._finish_command(self._loose_command)
+        for command in self._fields:
+            self._finish_command(command)
+        return Features(self.field_hyperlink, self.split_hyperlink_runs)
+
+
+class MetadataFacts(XmlFacts):
+    """Keep OPC metadata entries, never arbitrary XML text or unrelated elements."""
+
+    def __init__(self):
+        super().__init__()
+        self.elements: list[tuple[str, dict[str, str]]] = []
+        self.root_tag: str | None = None
+        self._depth = 0
+
+    def start(self, tag: str, attrs: Mapping[str, str]) -> None:
+        if self._depth == 0:
+            self.root_tag = tag
+        self._depth += 1
+        if self._depth == 2 and tag in (
+            "{http://schemas.openxmlformats.org/package/2006/content-types}Default",
+            "{http://schemas.openxmlformats.org/package/2006/content-types}Override",
+            "{http://schemas.openxmlformats.org/package/2006/relationships}Relationship",
+        ):
+            self.elements.append((tag, dict(attrs)))
+
+    def end(self, tag: str) -> None:
+        self._depth -= 1

docx/inspection/fingerprint.py

@@ -0,0 +1,150 @@
+"""Stream sorted package members into a versioned, unambiguously framed digest."""
+
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass
+from fnmatch import fnmatchcase
+from urllib.parse import unquote
+from zipfile import ZipFile
+
+from lxml import etree
+
+from docx.inspection.features import MetadataFacts
+from docx.inspection.reader import CT_NS, REL_NS, is_xml, scan_xml
+from docx.inspection.source import PackageSource, open_source
+
+
+@dataclass(frozen=True)
+class FingerprintPolicy:
+    """Explicit exclusions; the default includes every member and all XML content.
+
+    Part patterns match package-relative names using shell-style fnmatch (``*``
+    also matches ``/``). XML exclusions use expanded names, e.g. ``{namespace}tag``.
+    Relationship exclusions match complete Type URIs. Excluding a part does not
+    implicitly remove its relationships or Content-Type declaration.
+    """
+
+    exclude_parts: tuple[str, ...] = ()
+    exclude_relationship_types: tuple[str, ...] = ()
+    exclude_xml_tags: tuple[str, ...] = ()
+    exclude_xml_attributes: tuple[str, ...] = ()
+    canonicalize_xml: bool = True
+
+    def __post_init__(self) -> None:
+        if not self.canonicalize_xml and (
+            self.exclude_relationship_types or self.exclude_xml_tags or self.exclude_xml_attributes
+        ):
+            raise ValueError("XML exclusions require canonicalize_xml=True")
+
+
+class _CanonicalXml:
+    def __init__(self, write: Callable[[str], None], policy: FingerprintPolicy):
+        self._writer = etree.C14NWriterTarget(
+            write,
+            with_comments=True,
+            exclude_tags=set(policy.exclude_xml_tags),
+            exclude_attrs=set(policy.exclude_xml_attributes),
+        )
+        self._excluded_relationships = policy.exclude_relationship_types
+        self._skip_depth = 0
+
+    def start_ns(self, prefix: str | None, uri: str) -> None:
+        self._writer.start_ns(prefix or "", uri)
+
+    def end_ns(self, prefix: str | None) -> None:
+        pass
+
+    def start(self, tag: str, attrs: Mapping[str, str]) -> None:
+        if self._skip_depth or (
+            tag == f"{{{REL_NS}}}Relationship" and attrs.get("Type") in self._excluded_relationships
+        ):
+            self._skip_depth += 1
+        else:
+            self._writer.start(tag, dict(attrs))
+
+    def end(self, tag: str) -> None:
+        if self._skip_depth:
+            self._skip_depth -= 1
+        else:
+            self._writer.end(tag)
+
+    def data(self, text: str) -> None:
+        if not self._skip_depth:
+            self._writer.data(text)
+
+    def comment(self, text: str) -> None:
+        if not self._skip_depth:
+            self._writer.comment(text)
+
+    def pi(self, target: str, data: str) -> None:
+        if not self._skip_depth:
+            self._writer.pi(target, data)
+
+    def doctype(self, name: str | None, public_id: str | None, system_id: str | None) -> None:
+        raise ValueError("DTDs are not supported in package XML")
+
+    def close(self) -> None:
+        self._writer.close()
+
+
+def _xml_types(package: ZipFile) -> tuple[dict[str, str], dict[str, str]]:
+    if "[Content_Types].xml" not in package.namelist():
+        return {}, {}
+    target = MetadataFacts()
+    try:
+        scan_xml(package, package.getinfo("[Content_Types].xml"), target)
+    except (etree.XMLSyntaxError, ValueError) as exc:
+        raise ValueError(f"Cannot fingerprint XML part '[Content_Types].xml': {exc}") from exc
+    defaults: dict[str, str] = {}
+    overrides: dict[str, str] = {}
+    for tag, attrs in target.elements:
+        if tag == f"{{{CT_NS}}}Default":
+            defaults[attrs.get("Extension", "").lower()] = attrs.get("ContentType", "")
+        elif tag == f"{{{CT_NS}}}Override":
+            overrides[unquote(attrs.get("PartName", "")).lstrip("/")] = attrs.get("ContentType", "")
+    return defaults, overrides
+
+
+def fingerprint(source: PackageSource, *, policy: FingerprintPolicy | None = None) -> str:
+    """Return a SHA-256 content fingerprint, ignoring ZIP order and ZIP metadata.
+
+    XML uses C14N 2.0 with comments, preserving text whitespace and namespace
+    prefixes. This is a package-content comparison, not a semantic Word diff.
+    Malformed XML and ambiguous duplicate names raise; no partial hash is returned.
+    Stream ownership and position guarantees match inspect().
+    """
+    policy = FingerprintPolicy() if policy is None else policy
+    with open_source(source) as stream, ZipFile(stream) as package:
+        members = [member for member in package.infolist() if not member.is_dir()]
+        if len({member.filename for member in members}) != len(members):
+            raise ValueError("Cannot fingerprint a package with duplicate member names")
+        defaults, overrides = _xml_types(package) if policy.canonicalize_xml else ({}, {})
+        digest = hashlib.sha256(b"python-docx-oss package fingerprint v1\0")
+        for member in sorted(members, key=lambda member: member.filename):
+            name = member.filename
+            if any(fnmatchcase(name, pattern) for pattern in policy.exclude_parts):
+                continue
+            part_digest = hashlib.sha256()
+            content_type = overrides.get(name) or defaults.get(name.rsplit(".", 1)[-1].lower())
+            with package.open(member) as content:
+                if policy.canonicalize_xml and is_xml(name, content_type):
+                    target = _CanonicalXml(
+                        lambda text: part_digest.update(text.encode("utf-8")), policy
+                    )
+                    parser = etree.XMLParser(target=target, resolve_entities=False, no_network=True)
+                    try:
+                        while chunk := content.read(64 * 1024):
+                            parser.feed(chunk)
+                        parser.close()
+                    except (etree.XMLSyntaxError, ValueError) as exc:
+                        raise ValueError(f"Cannot fingerprint XML part {name!r}: {exc}") from exc
+                else:
+                    while chunk := content.read(64 * 1024):
+                        part_digest.update(chunk)
+            encoded_name = name.encode("utf-8")
+            digest.update(len(encoded_name).to_bytes(8, "big"))
+            digest.update(encoded_name)
+            digest.update(part_digest.digest())
+        return digest.hexdigest()

docx/inspection/reader.py

@@ -0,0 +1,245 @@
+"""Tolerant OPC inventory, independent of Document and the eager package reader."""
+
+from __future__ import annotations
+
+import posixpath
+import zlib
+from collections import deque
+from urllib.parse import unquote, urlsplit
+from zipfile import BadZipFile, ZipFile, ZipInfo
+
+from lxml import etree
+
+from docx.inspection.features import MetadataFacts, XmlFacts
+from docx.inspection.result import (
+    Features,
+    Finding,
+    PackageInspection,
+    PackageInventory,
+    PartInventory,
+    RelationshipInventory,
+)
+from docx.inspection.source import PackageSource, open_source
+
+CT_NS = "http://schemas.openxmlformats.org/package/2006/content-types"
+REL_NS = "http://schemas.openxmlformats.org/package/2006/relationships"
+OFFICE_REL_BASES = (
+    "http://schemas.openxmlformats.org/officeDocument/2006/relationships/",
+    "http://purl.oclc.org/ooxml/officeDocument/relationships/",
+)
+PACKAGE_ERRORS = (
+    BadZipFile,
+    EOFError,
+    OSError,
+    RuntimeError,
+    NotImplementedError,
+    UnicodeError,
+    zlib.error,
+)
+STORIES = {"officeDocument", "header", "footer", "footnotes", "endnotes", "comments"}
+
+
+def relationship_kind(relationship_type: str) -> str:
+    for base in OFFICE_REL_BASES:
+        if relationship_type.startswith(base):
+            return relationship_type[len(base) :]
+    return ""
+
+
+def relationship_source(name: str) -> str | None:
+    if name == "_rels/.rels":
+        return "/"
+    directory, filename = posixpath.split(name)
+    if posixpath.basename(directory) == "_rels" and filename.endswith(".rels"):
+        return posixpath.join(posixpath.dirname(directory), filename[:-5])
+    return None
+
+
+def resolve_target(source: str, target: str) -> str:
+    path = unquote(urlsplit(target).path)
+    if path.startswith("/"):
+        return posixpath.normpath(path)[1:]
+    directory = "" if source == "/" else posixpath.dirname(source)
+    return posixpath.normpath(posixpath.join(directory, path))
+
+
+def scan_xml(package: ZipFile, member: ZipInfo, target: XmlFacts) -> Features:
+    parser = etree.XMLParser(target=target, resolve_entities=False, no_network=True)
+    nonblank = False
+    with package.open(member) as stream:
+        while chunk := stream.read(64 * 1024):
+            nonblank |= bool(chunk.strip(b" \t\r\n"))
+            parser.feed(chunk)
+    if not nonblank:
+        raise EmptyXmlError("XML part is empty or contains only whitespace")
+    return parser.close()
+
+
+class EmptyXmlError(ValueError):
+    """An XML member contains no document content."""
+
+
+def is_xml(name: str, content_type: str | None) -> bool:
+    return name.endswith((".xml", ".rels")) or bool(
+        content_type
+        and (content_type.endswith("+xml") or content_type in ("text/xml", "application/xml"))
+    )
+
+
+class _Inspection:
+    def __init__(self, package: ZipFile):
+        self.package = package
+        self.findings: list[Finding] = []
+        self.members: dict[str, ZipInfo] = {}
+        self.defaults: dict[str, str] = {}
+        self.overrides: dict[str, str] = {}
+        self.relationships: list[RelationshipInventory] = []
+        for member in package.infolist():
+            name = member.filename
+            if member.is_dir():
+                continue
+            if name in self.members:
+                self.findings.append(Finding("duplicate_member", name))
+            else:
+                self.members[name] = member
+            if name.startswith("/") or "\\" in name or ".." in name.split("/"):
+                self.findings.append(Finding("unsafe_member_name", name))
+
+    def _scan(self, name: str, target: XmlFacts) -> Features:
+        member = self.members[name]
+        if member.file_size == 0:
+            self.findings.append(Finding("empty_xml_part", name))
+            return Features()
+        try:
+            return scan_xml(self.package, member, target)
+        except EmptyXmlError:
+            self.findings.append(Finding("empty_xml_part", name))
+        except (etree.XMLSyntaxError, ValueError) as exc:
+            self.findings.append(Finding("unparseable_xml_part", name, str(exc)))
+        except PACKAGE_ERRORS as exc:
+            self.findings.append(Finding("unreadable_part", name, str(exc)))
+        return Features()
+
+    def _read_metadata(self) -> None:
+        if "[Content_Types].xml" not in self.members:
+            self.findings.append(Finding("missing_content_types"))
+        else:
+            target = MetadataFacts()
+            self._scan("[Content_Types].xml", target)
+            if target.root_tag != f"{{{CT_NS}}}Types":
+                self.findings.append(Finding("invalid_content_types", "[Content_Types].xml"))
+            for tag, attrs in target.elements:
+                if tag == f"{{{CT_NS}}}Default":
+                    self.defaults[attrs.get("Extension", "").lower()] = attrs.get("ContentType", "")
+                elif tag == f"{{{CT_NS}}}Override":
+                    self.overrides[unquote(attrs.get("PartName", "")).lstrip("/")] = attrs.get(
+                        "ContentType", ""
+                    )
+        for name in sorted(self.members):
+            source = relationship_source(name)
+            if source is None:
+                continue
+            target = MetadataFacts()
+            self._scan(name, target)
+            if target.root_tag != f"{{{REL_NS}}}Relationships":
+                self.findings.append(Finding("invalid_relationships", name))
+                continue
+            ids: set[str] = set()
+            for tag, attrs in target.elements:
+                if tag != f"{{{REL_NS}}}Relationship":
+                    continue
+                raw_target = attrs.get("Target", "")
+                rel_id = attrs.get("Id", "")
+                if not raw_target or not rel_id or not attrs.get("Type") or rel_id in ids:
+                    self.findings.append(Finding("invalid_relationships", name))
+                    continue
+                ids.add(rel_id)
+                external = attrs.get("TargetMode") == "External"
+                try:
+                    resolved = raw_target if external else resolve_target(source, raw_target)
+                except ValueError as exc:
+                    self.findings.append(Finding("invalid_relationships", name, str(exc)))
+                    continue
+                relationship = RelationshipInventory(
+                    source, rel_id, attrs["Type"], resolved, external
+                )
+                self.relationships.append(relationship)
+                if not external and resolved not in self.members:
+                    self.findings.append(
+                        Finding("missing_relationship_target", name, target=resolved)
+                    )
+
+    def _reachable(self) -> set[str]:
+        edges: dict[str, list[str]] = {}
+        for rel in self.relationships:
+            if not rel.external:
+                edges.setdefault(rel.source, []).append(rel.target)
+        reached = {"/"}
+        pending = deque(["/"])
+        while pending:
+            for target in edges.get(pending.popleft(), []):
+                if target not in reached and target in self.members:
+                    reached.add(target)
+                    pending.append(target)
+        reached.add("[Content_Types].xml")
+        reached.update(name for name in self.members if relationship_source(name) in reached)
+        return reached
+
+    def content_type(self, name: str) -> str | None:
+        return self.overrides.get(name) or self.defaults.get(name.rsplit(".", 1)[-1].lower())
+
+    def report(self) -> PackageInspection:
+        self._read_metadata()
+        reachable = self._reachable()
+        main_parts = {
+            rel.target
+            for rel in self.relationships
+            if rel.source == "/"
+            and relationship_kind(rel.relationship_type) == "officeDocument"
+            and not rel.external
+            and rel.target in self.members
+        }
+        if not main_parts:
+            self.findings.append(Finding("missing_main_document"))
+        story_parts = main_parts | {
+            rel.target
+            for rel in self.relationships
+            if rel.source in reachable
+            and relationship_kind(rel.relationship_type) in STORIES
+            and not rel.external
+        }
+        features = Features()
+        for name in sorted(self.members):
+            if name == "[Content_Types].xml" or relationship_source(name) is not None:
+                continue
+            if not is_xml(name, self.content_type(name)):
+                continue
+            observed = self._scan(name, XmlFacts(name in story_parts))
+            features = Features(
+                features.field_hyperlink or observed.field_hyperlink,
+                features.split_hyperlink_runs or observed.split_hyperlink_runs,
+            )
+        inventory = PackageInventory(
+            tuple(
+                PartInventory(name, member.file_size, self.content_type(name), name in reachable)
+                for name, member in sorted(self.members.items())
+            ),
+            tuple(self.relationships),
+        )
+        return PackageInspection(inventory, tuple(self.findings), features)
+
+
+def inspect(source: PackageSource) -> PackageInspection:
+    """Observe package structure and Word story features without constructing a Document.
+
+    Format failures become findings. Source opening failures propagate, and invalid
+    argument types raise TypeError.
+    Seekable streams are read from the beginning and restored; borrowed streams stay open.
+    Binary members are inventoried without reading their payloads.
+    """
+    with open_source(source) as stream:
+        try:
+            with ZipFile(stream) as package:
+                return _Inspection(package).report()
+        except PACKAGE_ERRORS as exc:
+            return PackageInspection(findings=(Finding("not_a_package", message=str(exc)),))

docx/inspection/result.py

@@ -0,0 +1,59 @@
+"""Immutable observations of a package; findings do not prescribe repair policy."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class Finding:
+    code: str
+    partname: str | None = None
+    message: str = ""
+    target: str | None = None
+
+
+@dataclass(frozen=True)
+class PartInventory:
+    """ZIP member metadata. Names are package-relative, without a leading slash."""
+
+    partname: str
+    size: int
+    content_type: str | None
+    reachable: bool
+
+
+@dataclass(frozen=True)
+class RelationshipInventory:
+    """A relationship with its resolved internal target or original external URI."""
+
+    source: str
+    relationship_id: str
+    relationship_type: str
+    target: str
+    external: bool
+
+
+@dataclass(frozen=True)
+class PackageInventory:
+    parts: tuple[PartInventory, ...] = ()
+    relationships: tuple[RelationshipInventory, ...] = ()
+
+
+@dataclass(frozen=True)
+class Features:
+    field_hyperlink: bool = False
+    split_hyperlink_runs: bool = False
+
+
+@dataclass(frozen=True)
+class PackageInspection:
+    inventory: PackageInventory = PackageInventory()
+    findings: tuple[Finding, ...] = ()
+    features: Features = Features()
+
+    def has_finding(self, code: str) -> bool:
+        return any(finding.code == code for finding in self.findings)
+
+    def findings_by_code(self, code: str) -> tuple[Finding, ...]:
+        return tuple(finding for finding in self.findings if finding.code == code)