doc-control 0.0.1.dev0__py3-none-any.whl

doc_control/ops/diff.py

@@ -0,0 +1,177 @@
+"""``diff``: paragraph alignment between two versions of a document, read-only.
+
+Request: ``old`` and ``new`` (paths to ``.docx`` files), optional ``limit`` and
+``cursor`` (the envelope's ``next`` of the previous page; the cursor is tied to the old
+file's hash and to the new file's, so a changed file is not paged on silently).
+
+Result: ``old`` / ``new`` (path, sha256, paragraphs), ``summary`` (paragraphs by kind),
+``total`` (records), ``changes`` (one page, capped like every list) and ``address_map``
+(old address -> new address, or ``null`` for a paragraph that is gone, for the records
+of that page). The envelope's ``doc`` is the old file.
+
+Records, in the order of the old version (an insertion right after its predecessor):
+
+* ``{"kind": "modified", "old": ["p3#..."], "new": ["p3#..."], "similarity": 0.93}``
+  (``"moved": true`` when it also changed place); ``moved``, ``split``
+  (one old, several new), ``merged`` (several old, one new), ``deleted`` (``new`` empty)
+  and ``inserted`` (``old`` empty) look alike.
+* ``{"kind": "unchanged", "old": [a], "new": [b]}``: an unchanged paragraph whose address
+  changed (its number shifted). Unchanged paragraphs that kept their address are not
+  listed. With ``collapse_runs: true`` consecutive unchanged paragraphs shifted by the
+  same amount become one record ``{"kind": "unchanged", "old": [first], "new": [first],
+  "count": n}`` and are left out of ``address_map`` (``p<k>#<fp>`` is ``p<k + shift>#<fp>``).
+  By default every paragraph whose address changed is in ``address_map``.
+
+Address map for a split: the old address maps to the first new part (the record lists
+all parts); for a merge every old part maps to the merged paragraph.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from doc_control.envelope import Reply, success
+from doc_control.model.align import UNCHANGED, Alignment, Change, align
+from doc_control.model.document import Document
+from doc_control.model.index import DocumentIndex
+from doc_control.ops.paging import (
+    PAGING_PROPERTIES,
+    RESPONSE_BUDGET,
+    make_cursor,
+    read_cursor,
+    size,
+)
+from doc_control.registry import operation
+
+
+def diff_documents(old: Path, new: Path) -> tuple[DocumentIndex, DocumentIndex, Alignment]:
+    """Index both files (read-only) and align their paragraphs."""
+    index_old, index_new = Document(str(old)).index, Document(str(new)).index
+    return index_old, index_new, align(index_old.paragraphs, index_new.paragraphs)
+
+
+def address_map(
+    alignment: Alignment, old: DocumentIndex, new: DocumentIndex
+) -> dict[str, str | None]:
+    """The full cross-version map: every old address to its new address, or ``None``."""
+    out: dict[str, str | None] = {}
+    for change in alignment.changes:
+        target = str(new.paragraphs[change.new[0]].address) if change.new else None
+        for i in change.old:
+            out[str(old.paragraphs[i].address)] = target
+    return out
+
+
+def _records(
+    alignment: Alignment, old: DocumentIndex, new: DocumentIndex, *, collapse: bool = False
+) -> list[dict[str, Any]]:
+    """Output records: unchanged paragraphs that kept their address are left out."""
+    out: list[dict[str, Any]] = []
+    run: dict[str, Any] | None = None
+    next_old = shift = 0
+    for change in alignment.changes:
+        if change.kind != UNCHANGED:
+            run = None
+            out.append(_record(change, old, new))
+            continue
+        i, j = change.old[0], change.new[0]
+        if i == j:
+            run = None
+        elif not collapse:
+            out.append(_record(change, old, new))
+        elif run is not None and j - i == shift and next_old == i:
+            run["count"] += 1
+            next_old = i + 1
+        else:
+            shift, next_old = j - i, i + 1
+            run = {
+                "kind": UNCHANGED,
+                "old": [str(old.paragraphs[i].address)],
+                "new": [str(new.paragraphs[j].address)],
+                "count": 1,
+            }
+            out.append(run)
+    return out
+
+
+def _record(change: Change, old: DocumentIndex, new: DocumentIndex) -> dict[str, Any]:
+    rec: dict[str, Any] = {
+        "kind": change.kind,
+        "old": [str(old.paragraphs[i].address) for i in change.old],
+        "new": [str(new.paragraphs[j].address) for j in change.new],
+    }
+    if change.similarity is not None:
+        rec["similarity"] = change.similarity
+    if change.moved:
+        rec["moved"] = True
+    return rec
+
+
+def _page_map(records: list[dict[str, Any]]) -> dict[str, str | None]:
+    out: dict[str, str | None] = {}
+    for rec in records:
+        if "count" in rec:  # a collapsed run
+            continue
+        target = rec["new"][0] if rec["new"] else None
+        for address in rec["old"]:
+            out[address] = target
+    return out
+
+
+@operation(
+    "diff",
+    "read",
+    "Paragraph alignment between two versions of a document, with a cross-version address map.",
+    {
+        "type": "object",
+        "properties": {
+            "old": {"type": "string", "description": "Path of the earlier version (.docx)."},
+            "new": {"type": "string", "description": "Path of the later version (.docx)."},
+            "collapse_runs": {
+                "type": "boolean",
+                "description": "List runs of shifted unchanged paragraphs as one record each.",
+            },
+            **PAGING_PROPERTIES,
+        },
+        "required": ["old", "new"],
+        "additionalProperties": False,
+    },
+)
+def diff(request: dict[str, Any]) -> Reply:
+    old, new = Document(request["old"]), Document(request["new"])
+    alignment = align(old.index.paragraphs, new.index.paragraphs)
+    records = _records(
+        alignment, old.index, new.index, collapse=request.get("collapse_runs", False)
+    )
+
+    # The cursor must also notice the new file changing: fold its hash into the scope.
+    scoped = {**request, "new_sha256": new.sha256}
+    start = read_cursor("diff", old, scoped)
+    limit = request.get("limit", len(records))
+    result: dict[str, Any] = {
+        "old": old.envelope_doc(),
+        "new": new.envelope_doc(),
+        "summary": alignment.counts(),
+        "total": len(records),
+        "changes": [],
+        "address_map": {},
+    }
+    placeholder = make_cursor("diff", old, scoped, 10**12)
+    room = RESPONSE_BUDGET - size(
+        success("diff", result, doc=old.envelope_doc(), next_cursor=placeholder)
+    )
+    taken: list[dict[str, Any]] = []
+    used = 0
+    for rec in records[start : start + limit]:
+        cost = 2 * len(json.dumps(rec, ensure_ascii=False).encode("utf-8")) + 4  # record + map
+        if taken and used + cost > room:
+            break
+        taken.append(rec)
+        used += cost
+    end = start + len(taken)
+    result["changes"] = taken
+    result["address_map"] = _page_map(taken)
+    cursor = make_cursor("diff", old, scoped, end) if end < len(records) else None
+    return Reply(result, doc=old.envelope_doc(), next_cursor=cursor)

doc_control/ops/dotx.py

@@ -0,0 +1,206 @@
+"""``.dotx`` templates: ``styleset.export-dotx`` and ``styleset.import-dotx``.
+
+Export writes a minimal, valid Word template package (template main content type, an
+empty body, ``styles.xml`` holding the exported definitions, the package and document
+relationships and ``docProps``) to a new file; the source document is never touched.
+Import reads the styles of a ``.dotx`` back into a style set and installs it with the
+``styleset.install`` semantics (``on_conflict``), as one verified write.
+"""
+
+from __future__ import annotations
+
+import io
+import tempfile
+import zipfile
+from pathlib import Path
+from typing import Any
+from xml.sax.saxutils import escape
+
+from doc_control.envelope import Reply
+from doc_control.model.document import Document
+from doc_control.model.styles import load_styles
+from doc_control.ooxml.styledef import STYLES_CONTENT_TYPE, invalid
+from doc_control.ooxml.zipio import Package
+from doc_control.ops.styleset import (
+    _DOC,
+    _install,
+    _validated,
+    select_styleset,
+    style_entry,
+    write_new_file,
+)
+from doc_control.ops.stylewrite import StyleSession, commit_styles
+from doc_control.registry import operation
+
+TEMPLATE_CONTENT_TYPE = (
+    "application/vnd.openxmlformats-officedocument.wordprocessingml.template.main+xml"
+)
+_XML = '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n'
+_W = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
+_OD = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/"
+_PKG_REL = "http://schemas.openxmlformats.org/package/2006/relationships"
+_CORE_REL = _PKG_REL + "/metadata/core-properties"
+_EPOCH = (1980, 1, 1, 0, 0, 0)
+
+
+def _rels(rows: list[tuple[str, str, str]]) -> str:
+    body = "".join(f'<Relationship Id="{i}" Type="{t}" Target="{g}"/>' for i, t, g in rows)
+    return f'{_XML}<Relationships xmlns="{_PKG_REL}">{body}</Relationships>'
+
+
+def _skeleton(title: str) -> dict[str, bytes]:
+    """The template package without its styles definitions."""
+    types = (
+        f'{_XML}<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">'
+        '<Default Extension="rels" '
+        'ContentType="application/vnd.openxmlformats-package.relationships+xml"/>'
+        '<Default Extension="xml" ContentType="application/xml"/>'
+        f'<Override PartName="/word/document.xml" ContentType="{TEMPLATE_CONTENT_TYPE}"/>'
+        f'<Override PartName="/word/styles.xml" ContentType="{STYLES_CONTENT_TYPE}"/>'
+        '<Override PartName="/docProps/core.xml" '
+        'ContentType="application/vnd.openxmlformats-package.core-properties+xml"/>'
+        '<Override PartName="/docProps/app.xml" '
+        'ContentType="application/vnd.openxmlformats-officedocument.extended-properties+xml"/>'
+        "</Types>"
+    )
+    document = (
+        f'{_XML}<w:document xmlns:w="{_W}"><w:body><w:p/>'
+        '<w:sectPr><w:pgSz w:w="11906" w:h="16838"/>'
+        '<w:pgMar w:top="1440" w:right="1440" w:bottom="1440" w:left="1440" '
+        'w:header="708" w:footer="708" w:gutter="0"/></w:sectPr></w:body></w:document>'
+    )
+    core = (
+        f'{_XML}<cp:coreProperties xmlns:cp="http://schemas.openxmlformats.org/package/2006/'
+        'metadata/core-properties" xmlns:dc="http://purl.org/dc/elements/1.1/">'
+        f"<dc:title>{escape(title)}</dc:title><dc:creator>doc-control</dc:creator>"
+        "</cp:coreProperties>"
+    )
+    app = (
+        f'{_XML}<Properties xmlns="http://schemas.openxmlformats.org/officeDocument/2006/'
+        'extended-properties"><Application>doc-control</Application></Properties>'
+    )
+    parts = {
+        "[Content_Types].xml": types,
+        "_rels/.rels": _rels(
+            [
+                ("rId1", _OD + "officeDocument", "word/document.xml"),
+                ("rId2", _CORE_REL, "docProps/core.xml"),
+                ("rId3", _OD + "extended-properties", "docProps/app.xml"),
+            ]
+        ),
+        "word/document.xml": document,
+        "word/_rels/document.xml.rels": _rels([("rId1", _OD + "styles", "styles.xml")]),
+        "docProps/core.xml": core,
+        "docProps/app.xml": app,
+    }
+    return {name: text.encode("utf-8") for name, text in parts.items()}
+
+
+def _zip(parts: dict[str, bytes]) -> bytes:
+    buf = io.BytesIO()
+    with zipfile.ZipFile(buf, "w") as zf:
+        for name, data in parts.items():
+            info = zipfile.ZipInfo(name, _EPOCH)
+            info.compress_type = zipfile.ZIP_DEFLATED
+            info.external_attr = 0o600 << 16
+            zf.writestr(info, data)
+    return buf.getvalue()
+
+
+def build_dotx(styleset: dict[str, Any]) -> bytes:
+    """A minimal Word template package holding the style set's definitions."""
+    entries = _validated(styleset)
+    ids = {e["id"] for e in entries}
+    # A subset can leave a reference to a style that is not in the template: drop it.
+    entries = [
+        {k: v for k, v in e.items() if k not in ("based_on", "next", "link") or v in ids}
+        for e in entries
+    ]
+    parts = _skeleton(str(styleset.get("styleset") or "styles"))
+    parts["word/styles.xml"] = f'{_XML}<w:styles xmlns:w="{_W}"></w:styles>'.encode()
+    with tempfile.TemporaryDirectory() as tmp:
+        scratch = Path(tmp) / "styles.dotx"
+        scratch.write_bytes(_zip(parts))
+        session = StyleSession(Package(scratch))
+        _install(session, entries, "replace")
+        session.check()
+        parts["word/styles.xml"] = session.styles.data()
+    return _zip(parts)
+
+
+@operation(
+    "styleset.export-dotx",
+    "write",
+    "Write the document's styles as a new .dotx template; never touches the document.",
+    {
+        "type": "object",
+        "properties": {
+            "doc": _DOC,
+            "out": {"type": "string", "minLength": 1, "description": "New .dotx to create."},
+            "name": {"type": "string", "minLength": 1, "description": "Set name."},
+            "styles": {
+                "type": "array",
+                "items": {"type": "string", "minLength": 1},
+                "description": "Only these styles (ids or names).",
+            },
+            "custom_only": {"type": "boolean", "description": "Only user-defined styles."},
+        },
+        "required": ["doc", "out"],
+        "additionalProperties": False,
+    },
+)
+def styleset_export_dotx(request: dict[str, Any]) -> Reply:
+    doc = Document(request["doc"])
+    styleset = select_styleset(doc, request)
+    data = build_dotx(styleset)
+    write_new_file(Path(request["out"]), Path(request["doc"]), data)
+    result = {"path": request["out"], "count": len(styleset["styles"]), "bytes": len(data)}
+    return Reply(result, doc=doc.envelope_doc())
+
+
+def read_dotx(path: Path) -> list[dict[str, Any]]:
+    """The style entries of a .dotx (or any Word package) as style set entries."""
+    return [style_entry(s) for s in load_styles(Package(path)).styles]
+
+
+@operation(
+    "styleset.import-dotx",
+    "write",
+    "Install the styles of a .dotx template into the document: new styles are added, "
+    "existing ones replaced (or skipped); dry_run previews.",
+    {
+        "type": "object",
+        "properties": {
+            "doc": _DOC,
+            "path": {"type": "string", "minLength": 1, "description": "The .dotx to read."},
+            "on_conflict": {
+                "enum": ["replace", "skip", "error"],
+                "description": "A style id already in the document (default replace).",
+            },
+            "expect": {
+                "type": "object",
+                "properties": {"sha256": {"type": "string", "minLength": 1}},
+                "additionalProperties": False,
+            },
+            "dry_run": {"type": "boolean", "description": "Preview only; write nothing."},
+        },
+        "required": ["doc", "path"],
+        "additionalProperties": False,
+    },
+)
+def styleset_import_dotx(request: dict[str, Any]) -> Reply:
+    source, target = Path(request["path"]), Path(request["doc"])
+    if source.exists() and target.exists() and source.samefile(target):
+        raise invalid("path is the document itself")
+    styles = read_dotx(source)
+    if not styles:
+        raise invalid(f"no styles to import in {source.name}")
+    entries = _validated({"version": 1, "styles": styles})
+    mode = request.get("on_conflict", "replace")
+    result = commit_styles(
+        target,
+        lambda session: _install(session, entries, mode),
+        expect_sha256=request.get("expect", {}).get("sha256"),
+        dry_run=request.get("dry_run", False),
+    )
+    return Reply(result, doc=Document(request["doc"]).envelope_doc())

doc_control/ops/find.py

@@ -0,0 +1,190 @@
+"""`find`: literal or regex search, returned as addressable ranges.
+
+Each match is ``{"at": "p142#a7b2a7b2:10-15", "text": ..., "context": ...}``: the range
+is a paragraph address plus code-point offsets into the paragraph's visible text.
+Output is capped by ``limit``; when more matches exist the envelope's ``next`` is a
+cursor that continues the same search (it only works on the same document bytes and
+the same query options). Regex search runs in a child process killed at the time
+budget (Python's regex engine cannot be interrupted); a literal search checks the budget
+between paragraphs. A search that outlives the budget returns what it found with warning
+``TIME_BUDGET`` and a cursor at the unfinished paragraph; if not even that paragraph
+could be searched, ``SCHEMA_INVALID`` asks for a simpler pattern.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import hashlib
+import json
+import re
+import time
+from pathlib import Path
+from typing import Any
+
+from doc_control.envelope import Reply
+from doc_control.errors import DocControlError, ErrorCode
+from doc_control.model.address import Address
+from doc_control.model.index import DocumentIndex
+from doc_control.model.search import IsolatedScan, LocalScan, Matcher, SearchOptions
+from doc_control.ooxml.zipio import Package
+from doc_control.registry import operation
+
+DEFAULT_LIMIT = 20
+MAX_LIMIT = 200
+DEFAULT_CONTEXT = 30
+MAX_CONTEXT = 200
+MAX_TEXT = 200  # characters of a match echoed back; longer matches are cut
+TIME_BUDGET_SECONDS = 10.0
+
+_CURSOR = re.compile(r"c:find:([0-9a-f]{8}):([1-9][0-9]{0,11}):(0|[1-9][0-9]{0,11})")
+
+SCHEMA: dict[str, Any] = {
+    "type": "object",
+    "properties": {
+        "doc": {"type": "string", "minLength": 1, "description": "Path to the .docx file."},
+        "query": {
+            "type": "string",
+            "minLength": 1,
+            "description": "Text to find (at most 1000 characters), or a Python regex if `regex`.",
+        },
+        "regex": {"type": "boolean", "default": False, "description": "Treat `query` as a regex."},
+        "ignore_case": {"type": "boolean", "default": False, "description": "Ignore case."},
+        "ignore_accents": {
+            "type": "boolean",
+            "default": False,
+            "description": "Ignore accents: `maria` finds `María`. A regex must be written"
+            " without accents then.",
+        },
+        "whole_word": {
+            "type": "boolean",
+            "default": False,
+            "description": "Match only whole words (no letter, digit or mark on either side).",
+        },
+        "limit": {
+            "type": "integer",
+            "minimum": 1,
+            "maximum": MAX_LIMIT,
+            "default": DEFAULT_LIMIT,
+            "description": "Most matches to return; more are reached with `cursor`.",
+        },
+        "context": {
+            "type": "integer",
+            "minimum": 0,
+            "maximum": MAX_CONTEXT,
+            "default": DEFAULT_CONTEXT,
+            "description": "Characters of surrounding text on each side of a match.",
+        },
+        "cursor": {"type": "string", "description": "The `next` value of the previous page."},
+    },
+    "required": ["doc", "query"],
+    "additionalProperties": False,
+}
+
+
+@operation(
+    "find",
+    "read",
+    "Find text or a regex; matches come back as paragraph ranges, capped, with a cursor.",
+    SCHEMA,
+)
+def find(request: dict[str, Any]) -> Reply:
+    options = SearchOptions(
+        query=request["query"],
+        regex=request.get("regex", False),
+        ignore_case=request.get("ignore_case", False),
+        ignore_accents=request.get("ignore_accents", False),
+        whole_word=request.get("whole_word", False),
+    )
+    matcher = Matcher(options)
+    limit: int = request.get("limit", DEFAULT_LIMIT)
+    context: int = request.get("context", DEFAULT_CONTEXT)
+    pkg = Package(Path(request["doc"]))
+    index = DocumentIndex.from_package(pkg)
+    token = _token(pkg.sha256, options)
+    first, skip = _parse_cursor(request.get("cursor"), token)
+
+    if first > len(index):
+        raise _bad_cursor()
+    items = [(p.number, p.text) for p in index.paragraphs[first - 1 :] if p.text]
+    deadline = time.monotonic() + TIME_BUDGET_SECONDS
+    scan: LocalScan | IsolatedScan
+    # Regexes run in a child process that can be killed; literals cannot backtrack.
+    scan = (
+        IsolatedScan(options, items, deadline)
+        if options.regex
+        else LocalScan(matcher, items, deadline)
+    )
+    matches: list[dict[str, Any]] = []
+    warnings: list[str] = []
+    next_cursor: str | None = None
+    with contextlib.closing(scan.__iter__()) as found:
+        for number, spans in found:
+            para = index.paragraph(number)
+            skipping = skip if number == first else 0
+            for k, (start, end) in enumerate(spans):
+                if k < skipping:
+                    continue
+                if len(matches) == limit:
+                    next_cursor = f"c:find:{token}:{number}:{k}"
+                    break
+                matches.append(_match(para.address, para.text, start, end, context))
+            if next_cursor is not None:
+                break
+    stopped = scan.stopped_at
+    if stopped is not None and next_cursor is None:
+        if stopped == first and not matches:
+            raise DocControlError(
+                ErrorCode.SCHEMA_INVALID,
+                f"search of p{stopped} did not finish within {TIME_BUDGET_SECONDS:g}s;"
+                " simplify the pattern (avoid nested or overlapping repeats)",
+                path="$.query",
+                paragraph=stopped,
+            )
+        warnings.append(
+            f"TIME_BUDGET: stopped at p{stopped} after {TIME_BUDGET_SECONDS:g}s;"
+            " continue with `next`"
+        )
+        next_cursor = f"c:find:{token}:{stopped}:0"
+    return Reply(
+        {"matches": matches},
+        doc={"path": request["doc"], "sha256": pkg.sha256, "paragraphs": len(index)},
+        warnings=warnings,
+        next_cursor=next_cursor,
+    )
+
+
+def _match(address: Address, text: str, start: int, end: int, context: int) -> dict[str, Any]:
+    lo, hi = max(0, start - context), min(len(text), end + context)
+    shown = text[start : min(end, start + MAX_TEXT)]
+    out: dict[str, Any] = {
+        "at": f"{address}:{start}-{end}",
+        "text": shown,
+        "context": ("…" if lo > 0 else "") + text[lo:hi] + ("…" if hi < len(text) else ""),
+    }
+    if end - start > MAX_TEXT:
+        out["text_truncated"] = True
+        out["context"] = ("…" if lo > 0 else "") + text[lo:start] + shown + "…"
+    return out
+
+
+def _token(sha256: str, options: SearchOptions) -> str:
+    """Ties a cursor to one document and one query, so it cannot resume a different search."""
+    key = json.dumps([sha256, *vars(options).values()], ensure_ascii=True)
+    return hashlib.blake2s(key.encode("ascii")).hexdigest()[:8]
+
+
+def _parse_cursor(cursor: str | None, token: str) -> tuple[int, int]:
+    if cursor is None:
+        return 1, 0
+    m = _CURSOR.fullmatch(cursor)
+    if m is None or m[1] != token:
+        raise _bad_cursor()
+    return int(m[2]), int(m[3])
+
+
+def _bad_cursor() -> DocControlError:
+    return DocControlError(
+        ErrorCode.SCHEMA_INVALID,
+        "cursor does not belong to this document and query; repeat the search without it",
+        path="$.cursor",
+    )