extract-cli 0.1.3__tar.gz → 0.1.4__tar.gz

{extract_cli-0.1.3 → extract_cli-0.1.4}/CHANGELOG.md

@@ -6,6 +6,32 @@ to [Semantic Versioning](https://semver.org/). Per the suite convention
 (see [`docs/INTEROP.md`](docs/INTEROP.md)), **backward-incompatible changes to
 the output schema require a major version bump**; new optional fields are minor.
 
+## [0.1.4] - 2026-05-21
+
+DOCX clause detection, driven by testing against 20 real `.docx` contracts
+(Common Paper / Bonterms / YC templates via open-agreements, plus government
+samples) — the format we expect most.
+
+### Fixed
+- **The DOCX reader now honors Word heading styles.** Real Word contracts carry
+  their clause structure in `Heading1`–`Heading9`/`Title` paragraph styles with
+  *auto-generated* numbers (absent from the raw text), so the prior cascade
+  found almost no clauses. Heading-styled paragraphs are now emitted as `##`
+  headings (detected by the strongest tier); run-in headings
+  (`Payment.  Customer will pay …`) are split into title + body, and a full
+  sentence that merely carries a heading style is rejected (not a clause).
+  Across the 20-doc sample this took heading-styled agreements from ~0 clauses
+  to a clean 14–21 distinct suite-vocabulary clauses each.
+- Binary DOCX test fixtures are now generated deterministically (fixed zip
+  timestamp) so their sha256 — and the goldens — are stable across regenerations.
+
+### Known limitations (documented)
+- DOCX that auto-number clauses via `numbering.xml` with **no heading style and
+  no bold lead** (some Bonterms/older templates use a flat `Plain`/`ListParagraph`
+  style) still yield no clause map: the heading text carries no detectable
+  signal without reconstructing Word's numbering counters. Parties/dates/
+  governing-law still extract.
+
 ## [0.1.3] - 2026-05-21
 
 Clause-map de-noising and party cleanup, driven by testing against 10 more
@@ -140,6 +166,7 @@ Initial release — the open-loop front door of the contract-ops CLI suite.
   intentionally *not* governed by the output schema (the schema describes the
   full default output).
 
+[0.1.4]: https://github.com/DrBaher/extract-cli/releases/tag/v0.1.4
 [0.1.3]: https://github.com/DrBaher/extract-cli/releases/tag/v0.1.3
 [0.1.2]: https://github.com/DrBaher/extract-cli/releases/tag/v0.1.2
 [0.1.1]: https://github.com/DrBaher/extract-cli/releases/tag/v0.1.1

{extract_cli-0.1.3 → extract_cli-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: extract-cli
-Version: 0.1.3
+Version: 0.1.4
 Summary: Open-loop front door of the contract-ops CLI suite: ingest any contract (.md/.txt/.html/.docx/.pdf) and emit structured JSON.
 Project-URL: Homepage, https://cli.drbaher.com/
 Project-URL: Repository, https://github.com/DrBaher/extract-cli

{extract_cli-0.1.3 → extract_cli-0.1.4}/extract_cli.py

@@ -43,11 +43,11 @@ import urllib.request
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
 
-__version__ = "0.1.3"
+__version__ = "0.1.4"
 
 # Bumped independently of the package version when the *extraction logic*
 # changes in a way downstream consumers should notice. Embedded in `_meta`.
-EXTRACTOR_VERSION = "0.1.3"
+EXTRACTOR_VERSION = "0.1.4"
 
 # JSON Schema version of the output contract (docs/spec/extract-output.schema.json).
 SCHEMA_VERSION = 1
@@ -950,6 +950,39 @@ def _read_docx(path: Path, raw: bytes, prefer_optional: bool = True) -> Tuple[st
         return "", warnings
 
 
+def _docx_paragraph_style(ppr: Any, w: str) -> Optional[str]:
+    if ppr is None:
+        return None
+    st = ppr.find(w + "pStyle")
+    return st.get(w + "val") if st is not None else None
+
+
+def _is_heading_style(style: Optional[str]) -> bool:
+    """True for Word built-in heading/title styles (Heading1-9, Title, and the
+    'H1'/'H2' shorthands). These mark clause headings whose visible numbers are
+    auto-generated and absent from the raw text."""
+    if not style:
+        return False
+    s = style.lower()
+    return "heading" in s or s == "title" or bool(re.fullmatch(r"h[1-9]", s))
+
+
+def _docx_heading_title(text: str) -> Optional[str]:
+    """Pull the clause title out of a heading paragraph. Many contracts use a
+    run-in heading -- 'Performing Services.  Contractor will ...' -- where the
+    title is the lead before the first sentence break; a standalone header
+    ('Services & Restrictions') has no such break and is used whole.
+
+    Returns None when the paragraph is really a full sentence that merely
+    carries a heading style (no run-in title) -- those would otherwise become
+    garbage clause titles and mis-map under substring matching."""
+    m = re.match(r"\s*(.{2,80}?)[.:]\s+[A-Z(\"“]", text)
+    title = m.group(1).strip() if m else text.strip()
+    if len(title) > 70 or len(title.split()) > 9:
+        return None
+    return title
+
+
 def _read_docx_stdlib(raw: bytes) -> str:
     import io
     import zipfile
@@ -962,6 +995,7 @@ def _read_docx_stdlib(raw: bytes) -> str:
     paras: List[str] = []
     # iter over w:p in document order (includes paragraphs inside table cells).
     for p in root.iter(w + "p"):
+        style = _docx_paragraph_style(p.find(w + "pPr"), w)
         run_texts: List[str] = []
         any_text = False
         all_bold = True
@@ -978,6 +1012,17 @@ def _read_docx_stdlib(raw: bytes) -> str:
         if not line:
             paras.append("")
             continue
+        # Word heading styles carry the clause structure (their numbers are
+        # auto-generated, so absent from text). Emit them as H2 so the clause
+        # cascade's strongest tier detects them; keep any run-in body too.
+        if _is_heading_style(style):
+            title = _docx_heading_title(line)
+            if title is not None:
+                paras.append(f"## {title}")
+                if len(title) < len(line):
+                    paras.append(line[len(title):].lstrip(" .:\t"))
+                continue
+            # Sentence carrying a heading style -> treat as ordinary body text.
         if any_text and all_bold:
             line = f"**{line}**"
         paras.append(line)

{extract_cli-0.1.3 → extract_cli-0.1.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "extract-cli"
-version = "0.1.3"
+version = "0.1.4"
 description = "Open-loop front door of the contract-ops CLI suite: ingest any contract (.md/.txt/.html/.docx/.pdf) and emit structured JSON."
 readme = "README.md"
 requires-python = ">=3.9"

{extract_cli-0.1.3 → extract_cli-0.1.4}/tests/_fixtures_build.py

@@ -42,14 +42,36 @@ _DOCX_PARAS = [
 _W = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
 
 
-def _docx_paragraph(text: str, bold: bool) -> str:
+def _docx_paragraph(text: str, bold: bool = False, style: str = "") -> str:
+    ppr = f'<w:pPr><w:pStyle w:val="{style}"/></w:pPr>' if style else ""
     rpr = "<w:rPr><w:b/></w:rPr>" if bold else ""
-    return (f"<w:p><w:r>{rpr}"
+    return (f"<w:p>{ppr}<w:r>{rpr}"
             f'<w:t xml:space="preserve">{escape(text)}</w:t></w:r></w:p>')
 
 
-def build_docx() -> bytes:
-    body = "".join(_docx_paragraph(t, b) for t, b in _DOCX_PARAS)
+# A Word-styled agreement: clause structure carried by Heading1 styles (their
+# numbers are auto-generated, absent from text), including a run-in heading and
+# a full sentence that merely carries the heading style (must be rejected).
+_HEADING_DOCX_PARAS = [
+    ('Cloud Service Agreement', False, "Title"),
+    ('This Cloud Service Agreement is entered into as of April 4, 2024, by and '
+     'between Initech Software, Inc. (the "Provider") and Globex Corporation '
+     '(the "Customer").', False, ""),
+    ('Confidentiality', False, "Heading1"),
+    ('Each party will protect the other party’s Confidential Information.', False, ""),
+    ('Payment.  Customer will pay the fees set out in the Order Form within '
+     'thirty (30) days.', False, "Heading1"),
+    ('Term & Termination', False, "Heading1"),
+    ('The term of this Agreement is two (2) years and will automatically renew '
+     'for successive one-year terms.', False, ""),
+    ('Either party may terminate this Agreement upon material breach that '
+     'remains uncured for thirty days after written notice.', False, "Heading1"),
+    ('Governing Law', False, "Heading1"),
+    ('This Agreement is governed by the laws of the State of New York.', False, ""),
+]
+
+
+def _docx_package(body: str) -> bytes:
     document = (
         '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>'
         f'<w:document xmlns:w="{_W}"><w:body>{body}<w:sectPr/></w:body></w:document>'
@@ -70,14 +92,29 @@ def build_docx() -> bytes:
         'Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" '
         'Target="word/document.xml"/></Relationships>'
     )
+    # Deterministic: a fixed timestamp on every entry so regenerating the
+    # fixture produces byte-identical output (stable sha256 -> stable goldens).
     buf = io.BytesIO()
     with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as z:
-        z.writestr("[Content_Types].xml", content_types)
-        z.writestr("_rels/.rels", rels)
-        z.writestr("word/document.xml", document)
+        for name, data in (("[Content_Types].xml", content_types),
+                           ("_rels/.rels", rels),
+                           ("word/document.xml", document)):
+            info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0))
+            info.compress_type = zipfile.ZIP_DEFLATED
+            z.writestr(info, data)
     return buf.getvalue()
 
 
+def build_docx() -> bytes:
+    return _docx_package("".join(_docx_paragraph(t, b) for t, b in _DOCX_PARAS))
+
+
+def build_heading_docx() -> bytes:
+    return _docx_package(
+        "".join(_docx_paragraph(t, b, style=s) for t, b, s in _HEADING_DOCX_PARAS)
+    )
+
+
 # --- PDF: a software license with ALL-CAPS headings (Tier 3) ----------------
 
 _PDF_TEXT = """SOFTWARE LICENSE AGREEMENT
@@ -156,6 +193,7 @@ def build_scanned_pdf() -> bytes:
 
 _BINARY_FIXTURES = {
     "employment_docx.docx": build_docx,
+    "heading_docx.docx": build_heading_docx,
     "license_pdf.pdf": build_pdf,
     "scanned.pdf": build_scanned_pdf,
 }

{extract_cli-0.1.3 → extract_cli-0.1.4}/tests/_make_goldens.py

@@ -20,8 +20,8 @@ from tests._fixtures_build import ensure_binary_fixtures  # noqa: E402
 FIXTURES = Path(__file__).resolve().parent / "fixtures"
 
 DOCS = ["nda_h2.md", "services_bold.txt", "lease_allcaps.txt",
-        "employment_docx.docx", "license_pdf.pdf", "services_html.html",
-        "scanned.pdf"]
+        "employment_docx.docx", "heading_docx.docx", "license_pdf.pdf",
+        "services_html.html", "scanned.pdf"]
 
 
 def golden_for(name: str) -> dict:

{extract_cli-0.1.3 → extract_cli-0.1.4}/tests/conftest.py

@@ -25,6 +25,7 @@ CORPUS: Tuple[Tuple[str, str, str], ...] = (
     ("services_bold.txt", "bold-numbered", "text"),
     ("lease_allcaps.txt", "all-caps", "text"),
     ("employment_docx.docx", "bold-numbered", "docx"),
+    ("heading_docx.docx", "h2", "docx"),
     ("license_pdf.pdf", "all-caps", "pdf"),
     ("services_html.html", "numbered", "html"),
 )

{extract_cli-0.1.3 → extract_cli-0.1.4}/tests/fixtures/employment_docx.docx.expected.json

@@ -2,7 +2,7 @@
   "document": {
     "title": "EMPLOYMENT AGREEMENT",
     "format": "docx",
-    "sha256": "1ba94a7bfd5a32a6d080cc6704cefb786e26dc16a8985857d562182da5f7298f",
+    "sha256": "f50e4b9b0cb77250280eb4c26225009de063b5f4a2318e9e53784d3730d20bd1",
     "source_path": "employment_docx.docx"
   },
   "parties": [
@@ -138,7 +138,7 @@
     "source": "deterministic"
   },
   "_meta": {
-    "extractor_version": "0.1.3",
+    "extractor_version": "0.1.4",
     "tiers_used": [
       "deterministic"
     ],

extract_cli-0.1.4/tests/fixtures/heading_docx.docx.expected.json

@@ -0,0 +1,142 @@
+{
+  "document": {
+    "title": "Cloud Service Agreement",
+    "format": "docx",
+    "sha256": "23a3b14196cdca6b58d14c7a6836fe28ff6d2be6c2fd852badb03ab6b6e84056",
+    "source_path": "heading_docx.docx"
+  },
+  "parties": [
+    {
+      "name": "Initech Software, Inc.",
+      "confidence": 0.9,
+      "source": "deterministic",
+      "role": "Provider"
+    },
+    {
+      "name": "Globex Corporation",
+      "confidence": 0.9,
+      "source": "deterministic",
+      "role": "Customer"
+    }
+  ],
+  "dates": {
+    "effective": {
+      "value": "2024-04-04",
+      "confidence": 0.85,
+      "source": "deterministic"
+    },
+    "expiration": {
+      "value": null,
+      "confidence": 0.0,
+      "source": "none"
+    }
+  },
+  "term": {
+    "length": {
+      "value": "2 years",
+      "confidence": 0.7,
+      "source": "deterministic"
+    },
+    "auto_renew": {
+      "value": true,
+      "confidence": 0.65,
+      "source": "deterministic"
+    },
+    "notice_period_days": {
+      "value": null,
+      "confidence": 0.0,
+      "source": "none"
+    }
+  },
+  "governing_law": {
+    "value": "State of New York",
+    "confidence": 0.85,
+    "source": "deterministic"
+  },
+  "clauses": [
+    {
+      "canonical_title": "Cloud Service Agreement",
+      "detected_title": "## Cloud Service Agreement",
+      "tier": "h2",
+      "span": {
+        "start": 0,
+        "end": 191
+      },
+      "confidence": 0.71,
+      "source": "deterministic",
+      "mapped": false
+    },
+    {
+      "canonical_title": "Confidentiality",
+      "detected_title": "## Confidentiality",
+      "tier": "h2",
+      "span": {
+        "start": 191,
+        "end": 280
+      },
+      "confidence": 0.95,
+      "source": "deterministic",
+      "mapped": true
+    },
+    {
+      "canonical_title": "Payment",
+      "detected_title": "## Payment",
+      "tier": "h2",
+      "span": {
+        "start": 280,
+        "end": 371
+      },
+      "confidence": 0.95,
+      "source": "deterministic",
+      "mapped": true
+    },
+    {
+      "canonical_title": "Termination",
+      "detected_title": "## Term & Termination",
+      "tier": "h2",
+      "span": {
+        "start": 371,
+        "end": 622
+      },
+      "confidence": 0.95,
+      "source": "deterministic",
+      "mapped": true
+    },
+    {
+      "canonical_title": "Governing Law",
+      "detected_title": "## Governing Law",
+      "tier": "h2",
+      "span": {
+        "start": 622,
+        "end": 704
+      },
+      "confidence": 0.95,
+      "source": "deterministic",
+      "mapped": true
+    }
+  ],
+  "defined_terms": [
+    {
+      "term": "Provider",
+      "confidence": 0.6,
+      "source": "deterministic"
+    },
+    {
+      "term": "Customer",
+      "confidence": 0.6,
+      "source": "deterministic"
+    }
+  ],
+  "value": {
+    "value": null,
+    "confidence": 0.0,
+    "source": "none"
+  },
+  "_meta": {
+    "extractor_version": "0.1.4",
+    "tiers_used": [
+      "deterministic"
+    ],
+    "llm_used": false
+  }
+}

{extract_cli-0.1.3 → extract_cli-0.1.4}/tests/fixtures/lease_allcaps.txt.expected.json

@@ -133,7 +133,7 @@
     "source": "deterministic"
   },
   "_meta": {
-    "extractor_version": "0.1.3",
+    "extractor_version": "0.1.4",
     "tiers_used": [
       "deterministic"
     ],

{extract_cli-0.1.3 → extract_cli-0.1.4}/tests/fixtures/license_pdf.pdf.expected.json

@@ -133,7 +133,7 @@
     "source": "deterministic"
   },
   "_meta": {
-    "extractor_version": "0.1.3",
+    "extractor_version": "0.1.4",
     "tiers_used": [
       "deterministic"
     ],

{extract_cli-0.1.3 → extract_cli-0.1.4}/tests/fixtures/nda_h2.md.expected.json

@@ -143,7 +143,7 @@
     "source": "none"
   },
   "_meta": {
-    "extractor_version": "0.1.3",
+    "extractor_version": "0.1.4",
     "tiers_used": [
       "deterministic"
     ],

{extract_cli-0.1.3 → extract_cli-0.1.4}/tests/fixtures/scanned.pdf.expected.json

@@ -48,7 +48,7 @@
     "source": "none"
   },
   "_meta": {
-    "extractor_version": "0.1.3",
+    "extractor_version": "0.1.4",
     "tiers_used": [
       "deterministic"
     ],

{extract_cli-0.1.3 → extract_cli-0.1.4}/tests/fixtures/services_bold.txt.expected.json

@@ -133,7 +133,7 @@
     "source": "deterministic"
   },
   "_meta": {
-    "extractor_version": "0.1.3",
+    "extractor_version": "0.1.4",
     "tiers_used": [
       "deterministic"
     ],

{extract_cli-0.1.3 → extract_cli-0.1.4}/tests/fixtures/services_html.html.expected.json

@@ -148,7 +148,7 @@
     "source": "deterministic"
   },
   "_meta": {
-    "extractor_version": "0.1.3",
+    "extractor_version": "0.1.4",
     "tiers_used": [
       "deterministic"
     ],

{extract_cli-0.1.3 → extract_cli-0.1.4}/tests/test_misc.py

@@ -142,6 +142,37 @@ def test_pdf_unescape() -> None:
     assert ex._pdf_unescape(r"\101\102") == "AB"  # octal escapes
 
 
+def test_docx_heading_style_helpers() -> None:
+    assert ex._is_heading_style("Heading1")
+    assert ex._is_heading_style("Heading 2".replace(" ", ""))
+    assert ex._is_heading_style("Title")
+    assert ex._is_heading_style("h3")
+    assert not ex._is_heading_style("Plain")
+    assert not ex._is_heading_style(None)
+    # Run-in heading: title is the lead before the sentence body.
+    assert ex._docx_heading_title("Payment.  Customer will pay the fees.") == "Payment"
+    assert ex._docx_heading_title("Governing Law") == "Governing Law"
+    # A full sentence carrying a heading style is rejected (not a clause title).
+    assert ex._docx_heading_title(
+        "Either party may terminate this Agreement upon material breach that "
+        "remains uncured for thirty days.") is None
+
+
+def test_docx_heading_styles_drive_clause_map() -> None:
+    """The Word-styled fixture's clauses come from Heading1 styles (their
+    numbers are auto-generated), detected via the H2 tier; the sentence that
+    merely carries a heading style is not a clause."""
+    raw, text, fmt, _w = ex.load_source(FIXTURES / "heading_docx.docx", prefer_optional=False)
+    result = ex.build_extraction(text, raw, fmt, "heading_docx.docx")
+    assert result["clauses"], "heading-styled docx should yield clauses"
+    canon = {c["canonical_title"] for c in result["clauses"]}
+    assert {"Confidentiality", "Payment", "Governing Law"} <= canon
+    assert all(c["tier"] == "h2" for c in result["clauses"])
+    # The full-sentence "Either party may terminate ..." must not appear.
+    assert not any("terminate this Agreement" in c["detected_title"] for c in result["clauses"])
+    assert [p["name"] for p in result["parties"]] == ["Initech Software, Inc.", "Globex Corporation"]
+
+
 def test_html_extraction() -> None:
     raw, text, fmt, _w = ex.load_source(FIXTURES / "services_html.html")
     assert fmt == "html"