extract-cli 0.1.2__tar.gz → 0.1.3__tar.gz

{extract_cli-0.1.2 → extract_cli-0.1.3}/CHANGELOG.md

@@ -6,6 +6,31 @@ to [Semantic Versioning](https://semver.org/). Per the suite convention
 (see [`docs/INTEROP.md`](docs/INTEROP.md)), **backward-incompatible changes to
 the output schema require a major version bump**; new optional fields are minor.
 
+## [0.1.3] - 2026-05-21
+
+Clause-map de-noising and party cleanup, driven by testing against 10 more
+contracts (SEC EDGAR credit, loan, employment, lease, asset-purchase, and
+consulting HTML exhibits; Apache PDFs).
+
+### Fixed
+- **Clause map drops structural noise** common in dense real documents:
+  a heading whose title repeats 3+ times is treated as a running header/footer
+  (one lease's `Ks 112708-2` page code went from 44 "clauses" to 0), and
+  front/back-matter (`Table of Contents`, `Exhibit B`, `Schedule 2.1`) and
+  document codes/page numbers (4+ consecutive digits) are filtered out.
+- **Party-name cleanup** extended: trailing `together with …`, `, as
+  administrative agent`, and a dangling unclosed parenthetical
+  (`(each of them being`) are trimmed.
+
+### Notes
+- On dense documents the deterministic clause map can still surface a few
+  non-clause headings (e.g. address lines in a notices block); consumers
+  wanting only suite-vocabulary clauses should filter on `mapped == true`,
+  which isolates the real clauses (the noise is always `mapped == false`).
+- Known best-effort edge cases on varied real paper: a bare role word as a
+  party name ("Landlord"), and a middle-initial period truncating a personal
+  name ("John C." → "John C"). Best-effort fields carry confidence/source.
+
 ## [0.1.2] - 2026-05-21
 
 More real-world hardening, driven by testing against five additional contracts
@@ -115,6 +140,7 @@ Initial release — the open-loop front door of the contract-ops CLI suite.
   intentionally *not* governed by the output schema (the schema describes the
   full default output).
 
+[0.1.3]: https://github.com/DrBaher/extract-cli/releases/tag/v0.1.3
 [0.1.2]: https://github.com/DrBaher/extract-cli/releases/tag/v0.1.2
 [0.1.1]: https://github.com/DrBaher/extract-cli/releases/tag/v0.1.1
 [0.1.0]: https://github.com/DrBaher/extract-cli/releases/tag/v0.1.0

{extract_cli-0.1.2 → extract_cli-0.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: extract-cli
-Version: 0.1.2
+Version: 0.1.3
 Summary: Open-loop front door of the contract-ops CLI suite: ingest any contract (.md/.txt/.html/.docx/.pdf) and emit structured JSON.
 Project-URL: Homepage, https://cli.drbaher.com/
 Project-URL: Repository, https://github.com/DrBaher/extract-cli

{extract_cli-0.1.2 → extract_cli-0.1.3}/extract_cli.py

@@ -43,11 +43,11 @@ import urllib.request
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
 
-__version__ = "0.1.2"
+__version__ = "0.1.3"
 
 # Bumped independently of the package version when the *extraction logic*
 # changes in a way downstream consumers should notice. Embedded in `_meta`.
-EXTRACTOR_VERSION = "0.1.2"
+EXTRACTOR_VERSION = "0.1.3"
 
 # JSON Schema version of the output contract (docs/spec/extract-output.schema.json).
 SCHEMA_VERSION = 1
@@ -629,18 +629,25 @@ _PARTY_CUT_MARKERS: Tuple[str, ...] = (
     r"\s+organized\b",
     r"\s+incorporated\b",
     r"\s+whose\b",
+    r"\s+together\b",
+    r",\s+as\s+\w",                                   # ", as administrative agent"
     r"\s+(?:as\s+of|dated|effective)\b",
 )
 
 
 def _clean_party_name(s: str) -> str:
     """Trim a captured party name down to the entity name, dropping trailing
-    descriptors ('a Delaware corporation', 'd/b/a ...', 'as of ...')."""
+    descriptors ('a Delaware corporation', 'd/b/a ...', 'together with ...',
+    'as of ...') and any dangling unclosed parenthetical ('(each of them ...')."""
     s = re.sub(r"\s+", " ", s).strip().strip(",").strip()
     for pat in _PARTY_CUT_MARKERS:
         m = re.search(pat, s, re.IGNORECASE)
         if m:
             s = s[: m.start()].strip().strip(",").strip()
+    # Drop a trailing parenthetical that was opened but never closed (the close
+    # fell outside the captured span), e.g. "Glenn Rufrano (each of them being".
+    if "(" in s and ")" not in s:
+        s = s[: s.index("(")].strip().strip(",").strip()
     return s.strip("\"“”").strip()
 
 
@@ -741,9 +748,45 @@ def extract_defined_terms(text: str) -> List[JSON]:
     return [{"term": t, "confidence": 0.6, "source": "deterministic"} for t in seen]
 
 
+# Detected-heading titles that are almost never real clauses: front/back-matter,
+# page/document codes, exhibit & schedule references.
+_NOISE_TITLE_PREFIX_RE = re.compile(
+    r"^(?:table\s+of\s+contents|exhibit|schedule|annex|appendix|attachment|"
+    r"signature\s+page|page)\b",
+    re.IGNORECASE,
+)
+
+
+def _is_noise_clause_title(title: str) -> bool:
+    """True for detected 'headings' that are structural noise rather than
+    clauses -- document codes/page numbers (4+ consecutive digits, e.g.
+    'Ks 112708-2'), and front/back-matter like 'Table of Contents' or
+    'Exhibit B'. Safe filters only; kept conservative to avoid dropping real
+    clauses."""
+    t = title.strip()
+    if re.search(r"\d{4,}", t):
+        return True
+    if _NOISE_TITLE_PREFIX_RE.match(t):
+        return True
+    return False
+
+
 def extract_clauses(text: str) -> List[JSON]:
+    detected = detect_clauses(text)
+    # A heading whose title repeats 3+ times across the document is almost
+    # always a running header/footer (e.g. a page code), not that many distinct
+    # clauses -- drop every occurrence. (Counted on the normalized title.)
+    counts: Dict[str, int] = {}
+    for c in detected:
+        k = _norm_clause_key(c["title"])
+        counts[k] = counts.get(k, 0) + 1
+
     out: List[JSON] = []
-    for c in detect_clauses(text):
+    for c in detected:
+        if counts[_norm_clause_key(c["title"])] >= 3:
+            continue
+        if _is_noise_clause_title(c["title"]):
+            continue
         canonical, mapped = _canonicalize_clause(c["title"])
         tier = c["tier"]
         base = {"h2": 0.95, "bold-numbered": 0.85, "numbered": 0.8,

{extract_cli-0.1.2 → extract_cli-0.1.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "extract-cli"
-version = "0.1.2"
+version = "0.1.3"
 description = "Open-loop front door of the contract-ops CLI suite: ingest any contract (.md/.txt/.html/.docx/.pdf) and emit structured JSON."
 readme = "README.md"
 requires-python = ">=3.9"

{extract_cli-0.1.2 → extract_cli-0.1.3}/tests/fixtures/employment_docx.docx.expected.json

@@ -138,7 +138,7 @@
     "source": "deterministic"
   },
   "_meta": {
-    "extractor_version": "0.1.2",
+    "extractor_version": "0.1.3",
     "tiers_used": [
       "deterministic"
     ],

{extract_cli-0.1.2 → extract_cli-0.1.3}/tests/fixtures/lease_allcaps.txt.expected.json

@@ -133,7 +133,7 @@
     "source": "deterministic"
   },
   "_meta": {
-    "extractor_version": "0.1.2",
+    "extractor_version": "0.1.3",
     "tiers_used": [
       "deterministic"
     ],

{extract_cli-0.1.2 → extract_cli-0.1.3}/tests/fixtures/license_pdf.pdf.expected.json

@@ -133,7 +133,7 @@
     "source": "deterministic"
   },
   "_meta": {
-    "extractor_version": "0.1.2",
+    "extractor_version": "0.1.3",
     "tiers_used": [
       "deterministic"
     ],

{extract_cli-0.1.2 → extract_cli-0.1.3}/tests/fixtures/nda_h2.md.expected.json

@@ -143,7 +143,7 @@
     "source": "none"
   },
   "_meta": {
-    "extractor_version": "0.1.2",
+    "extractor_version": "0.1.3",
     "tiers_used": [
       "deterministic"
     ],

{extract_cli-0.1.2 → extract_cli-0.1.3}/tests/fixtures/scanned.pdf.expected.json

@@ -48,7 +48,7 @@
     "source": "none"
   },
   "_meta": {
-    "extractor_version": "0.1.2",
+    "extractor_version": "0.1.3",
     "tiers_used": [
       "deterministic"
     ],

{extract_cli-0.1.2 → extract_cli-0.1.3}/tests/fixtures/services_bold.txt.expected.json

@@ -133,7 +133,7 @@
     "source": "deterministic"
   },
   "_meta": {
-    "extractor_version": "0.1.2",
+    "extractor_version": "0.1.3",
     "tiers_used": [
       "deterministic"
     ],

{extract_cli-0.1.2 → extract_cli-0.1.3}/tests/fixtures/services_html.html.expected.json

@@ -148,7 +148,7 @@
     "source": "deterministic"
   },
   "_meta": {
-    "extractor_version": "0.1.2",
+    "extractor_version": "0.1.3",
     "tiers_used": [
       "deterministic"
     ],

{extract_cli-0.1.2 → extract_cli-0.1.3}/tests/test_clause_map.py

@@ -69,6 +69,31 @@ def test_trailing_period_stripped_from_titles() -> None:
     assert ex._canonicalize_clause("Survival.") == ("Survival", True)
 
 
+def test_repeated_heading_treated_as_boilerplate() -> None:
+    # A "heading" that repeats 3+ times is a running header/footer, not clauses.
+    body = "\n\n".join("## Ks 99-2\n\nfoo" for _ in range(4))
+    text = "## Confidentiality\n\nreal body\n\n" + body
+    clauses = ex.extract_clauses(text)
+    titles = [c["canonical_title"] for c in clauses]
+    assert "Confidentiality" in titles
+    assert not any("Ks" in (t or "") for t in titles)
+
+
+def test_noise_clause_titles_filtered() -> None:
+    assert ex._is_noise_clause_title("Ks 112708-2")       # 4+ digit code
+    assert ex._is_noise_clause_title("Table of Contents")
+    assert ex._is_noise_clause_title("Exhibit B")
+    assert ex._is_noise_clause_title("Schedule 2.1")
+    assert not ex._is_noise_clause_title("Confidentiality")
+    assert not ex._is_noise_clause_title("Term and Termination")
+
+
+def test_party_cuts_together_as_agent_and_unclosed_paren() -> None:
+    assert ex._clean_party_name("Foo LLC, together with its affiliates") == "Foo LLC"
+    assert ex._clean_party_name("GE Capital Corporation, as administrative agent") == "GE Capital Corporation"
+    assert ex._clean_party_name("Glenn Rufrano (each of them being") == "Glenn Rufrano"
+
+
 def test_cascade_priority_h2_wins() -> None:
     # An H2 present means the bold/all-caps fallbacks must not fire.
     text = "## Real Heading\n\n**1. Not A Heading**\n\nALSO NOT A HEADING\n\nbody"