openom-cli 0.1.0__py3-none-any.whl

openom_cli/buildout.py

@@ -0,0 +1,207 @@
+# SPDX-License-Identifier: MIT
+"""Deterministic Buildout listing -> openOM payload mapper (the connector->manifest bridge).
+
+Grounded in the REAL Buildout `get_listing` shape (nested ``core.research_property_attributes.*`` +
+``custom_fields.*`` + ``financials``). Pure + zero-inference: it normalizes names/units and omits
+anything absent - it never guesses. The human/CLI supplies the assertion identity (assertedBy,
+assertedDate, noiType, noiAsOfDate); those are never inferred from Buildout. The output is a schema-
+valid openOM payload ready for ``om embed-batch``.
+
+Note the two cap rates: ``cap_rate`` is Buildout's stated "Average CAP Rate" over the term (often
+absent); ``cap_rate_derived`` is current NOI/price. We map the derived one because it is what the
+openOM consistency check (NOI/price vs capRate) expects and what the OM's headline cap reflects.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+NS = "https://openom.app/ns/0.1"
+
+
+def _num(v: Any) -> float | None:
+    try:
+        return float(str(v).replace(",", "").strip())
+    except (TypeError, ValueError):
+        return None
+
+
+def _int(v: Any) -> int | None:
+    n = _num(v)
+    return int(n) if n is not None else None
+
+
+def _pct_to_fraction(v: Any) -> float | None:
+    n = _num(v)
+    return round(n / 100, 6) if n is not None else None
+
+
+def _iso_date(mdy: Any) -> str | None:
+    """'10/1/2026' -> '2026-10-01'. Returns None if not an M/D/Y date."""
+    if not mdy:
+        return None
+    parts = str(mdy).strip().split("/")
+    if len(parts) != 3:
+        return None
+    try:
+        m, d, y = (int(p) for p in parts)
+    except ValueError:
+        return None
+    if not (1 <= m <= 12 and 1 <= d <= 31 and y > 1900):
+        return None
+    return f"{y:04d}-{m:02d}-{d:02d}"
+
+
+def _state_code(v: Any) -> str | None:
+    """'GA - Georgia' -> 'GA'; 'GA' -> 'GA'."""
+    if not v:
+        return None
+    head = str(v).split("-")[0].strip()
+    return head.upper() if len(head) == 2 else None
+
+
+def _lease_type(v: Any) -> str | None:
+    """Map Buildout's free-text lease type to the openOM asserted value."""
+    if not v:
+        return None
+    s = str(v).upper()
+    if "NNN" in s:
+        return "NNN"
+    if "NN" in s:
+        return "NN"
+    if "GROSS" in s:
+        return "gross"
+    return str(v)
+
+
+def _compact(d: dict[str, Any]) -> dict[str, Any]:
+    # drop absent values, incl. empty strings (schema forbids ""), but keep 0/0.0/False
+    return {k: v for k, v in d.items() if v not in (None, "", {}, [])}
+
+
+def _months_between(start_iso: str | None, end_iso: str | None) -> int | None:
+    """Whole months between two ISO (YYYY-MM-DD) dates, or None. Deterministic (no clock)."""
+    if not start_iso or not end_iso:
+        return None
+    try:
+        sy, sm, sd = (int(x) for x in start_iso.split("-"))
+        ey, em, ed = (int(x) for x in end_iso.split("-"))
+    except ValueError:
+        return None
+    months = (ey - sy) * 12 + (em - sm)
+    if ed < sd:  # a partial trailing month doesn't count
+        months -= 1
+    return months if months >= 0 else None
+
+
+# Canonical fields tracked for a back-catalog coverage report (the numbers a buyer underwrites).
+# Used to flag near-empty payloads before a bulk embed (Rule 6: review at scale).
+_COVERAGE_FIELDS: tuple[tuple[str, tuple[str, ...]], ...] = (
+    ("askingPrice", ("deal", "askingPrice")),
+    ("capRate", ("deal", "capRate")),
+    ("noi", ("deal", "noi")),
+    ("address", ("property", "address")),
+    ("buildingSF", ("property", "buildingSF")),
+    ("tenant", ("lease", "tenantEntity")),
+    ("leaseType", ("lease", "leaseTypeAsserted")),
+    ("expiration", ("lease", "expiration")),
+)
+
+
+def payload_coverage(payload: dict[str, Any]) -> dict[str, Any]:
+    """Which tracked fields a mapped payload actually carries - for a pre-embed coverage report."""
+    present: list[str] = []
+    missing: list[str] = []
+    for name, path in _COVERAGE_FIELDS:
+        cur: Any = payload
+        for seg in path:
+            cur = cur.get(seg) if isinstance(cur, dict) else None
+        (present if cur not in (None, "", {}, []) else missing).append(name)
+    return {
+        "filled": len(present), "of": len(_COVERAGE_FIELDS),
+        "present": present, "missing": missing,
+    }
+
+
+def listing_to_payload(
+    listing: dict[str, Any],
+    *,
+    asserted_by: dict[str, str],
+    asserted_date: str,
+    noi_type: str,
+    noi_as_of: str | None = None,
+) -> dict[str, Any]:
+    """Map one Buildout ``get_listing`` object to a schema-valid openOM payload (partial fields only
+    where Buildout has them). ``asserted_by``/``asserted_date``/``noi_type``/``noi_as_of`` are the
+    human's assertion identity and are stamped verbatim, never inferred."""
+    core: dict[str, Any] = listing.get("core", {})
+    cf: dict[str, Any] = listing.get("custom_fields", {})
+    fin: dict[str, Any] = listing.get("financials", {})
+
+    def rp(attr: str) -> Any:
+        return core.get(f"research_property_attributes.{attr}")
+
+    address = _compact({
+        "streetAddress": rp("address"),
+        "addressLocality": rp("city"),
+        "addressRegion": _state_code(rp("state")),
+        "postalCode": rp("zip"),
+        "addressCountry": "US" if str(rp("country_id")) == "1" else None,
+    })
+    lat, lng = _num(rp("latitude")), _num(rp("longitude"))
+    geo = {"latitude": lat, "longitude": lng} if lat is not None and lng is not None else None
+    lot = _num(rp("lot_size")) if str(rp("lot_size_units")).lower().startswith("acre") else None
+    building_sf = _int(rp("building_size"))
+    units = _int(rp("number_of_units"))
+    # propertyType ([M4]): the primary asset-class filter, trivially mappable and never auto-filled
+    # before. Buildout exposes it as a research attribute; omitted (never guessed) when absent.
+    prop_type = rp("property_type") or rp("property_sub_type") or cf.get("Property type")
+    property_ = _compact({
+        "propertyType": str(prop_type).strip().lower() if prop_type else None,
+        "address": address or None,
+        "geo": geo,
+        "buildingSF": building_sf,
+        "yearBuilt": _int(rp("year_built")),
+        "lotAcres": lot,
+        "units": units,
+        "occupancy": _pct_to_fraction(rp("occupancy_pct")),
+    })
+
+    price = _int(fin.get("sale_price"))
+    deal = _compact({
+        "askingPrice": price,
+        "capRate": _pct_to_fraction(fin.get("cap_rate_derived") or fin.get("cap_rate")),
+        "noi": _int(fin.get("noi") or cf.get("NOI")),
+        # Deterministically derived from mapped values ([M4]); not in Buildout, computed here.
+        "pricePerUnit": round(price / units) if price and units else None,
+        "pricePerSF": round(price / building_sf, 2) if price and building_sf else None,
+        "noiType": noi_type,
+        "noiAsOfDate": noi_as_of or asserted_date,
+        "status": "active",
+    })
+
+    commencement = _iso_date(cf.get("Lease start date"))
+    expiration = _iso_date(cf.get("Lease expiration date"))
+    guarantor_name = cf.get("Lease guarantor")
+    lease = _compact({
+        "tenantEntity": cf.get("Tenant"),
+        "leaseTypeAsserted": _lease_type(cf.get("Lease type")),
+        "commencement": commencement,
+        "expiration": expiration,
+        # termMonths ([M4]): derived from the two dates above, deterministic (no clock).
+        "termMonths": _months_between(commencement, expiration),
+        "guarantor": {"name": guarantor_name, "type": "corporate"} if guarantor_name else None,
+    })
+
+    return _compact({
+        "@context": ["https://schema.org", NS],
+        "@type": "RealEstateListing",
+        "specVersion": "0.1",
+        "assertedBy": _compact(dict(asserted_by)),
+        "assertedDate": asserted_date,
+        "property": property_ or None,
+        "deal": deal or None,
+        "lease": lease or None,
+        # A fresh assertion has no prior; a re-embed records supersedes = prior payload hash (core).
+        "meta": {"supersedes": None},
+    })

openom_cli/buildout_pull.py

@@ -0,0 +1,235 @@
+# SPDX-License-Identifier: MIT
+"""Pull a Buildout back-catalog into local files for ``om buildout-manifest`` (#B3).
+
+Closes the acquisition gap: instead of hand-extracting one ``get_listing`` JSON per listing and
+downloading every OM PDF by hand, this fetches them in one authenticated pass. Deterministic and
+zero-inference (a data fetch). The pure orchestrator ``pull`` is transport-injected so it is fully
+unit-testable with a fake; the real MCP Streamable-HTTP transport (``mcp_http_call_tool``) mirrors
+the extension's ``buildout-http.ts`` (initialize -> initialized -> tools/call, JSON or SSE).
+
+The real live endpoint run is environment-gated (a Buildout MCP endpoint + token), like the
+extension's real-Prompt-API check - stated, not faked. The transport wire-format and SSE parser ARE
+unit-tested here.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import urllib.request
+from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from typing import Any
+
+# (tool_name, arguments) -> the tool's result object (the listing dict).
+CallTool = Callable[[str, dict[str, Any]], dict[str, Any]]
+# url -> raw PDF bytes.
+FetchBytes = Callable[[str], bytes]
+
+_PDF_URL = re.compile(r"https?://\S+?\.pdf(?:\?\S*)?", re.IGNORECASE)
+
+
+def om_url_of(listing: dict[str, Any]) -> str | None:
+    """Best-effort find the OM PDF URL inside a listing object, else None.
+
+    Checks the obvious explicit places, then falls back to the first ``…\\.pdf`` URL anywhere in the
+    record. Never guesses a non-PDF link. A None result means "no OM PDF found" - the caller records
+    it and moves on (the listing JSON is still written)."""
+    for key in ("om_url", "offering_memorandum_url", "document_url"):
+        v = listing.get(key)
+        if isinstance(v, str) and v.lower().split("?")[0].endswith(".pdf"):
+            return v
+    docs = listing.get("documents")
+    if isinstance(docs, list):
+        for d in docs:
+            url = d.get("url") if isinstance(d, dict) else None
+            if isinstance(url, str) and url.lower().split("?")[0].endswith(".pdf"):
+                return url
+    m = _PDF_URL.search(json.dumps(listing))
+    return m.group(0) if m else None
+
+
+def ids_from_search_result(result: Any) -> list[str]:
+    """Extract listing ids from a search tool result (best-effort over common shapes).
+
+    Handles a bare list of ids, a list of listing objects (``id``/``listing_id``), or a wrapper
+    ``{listings|results|data: [...]}``. Ids are stringified + de-duplicated, order preserved."""
+    rows: Any = result
+    if isinstance(result, dict):
+        for key in ("listings", "results", "data", "items"):
+            if isinstance(result.get(key), list):
+                rows = result[key]
+                break
+    out: list[str] = []
+    seen: set[str] = set()
+    for row in rows if isinstance(rows, list) else []:
+        rid: Any = None
+        if isinstance(row, str | int):
+            rid = row
+        elif isinstance(row, dict):
+            rid = row.get("id") or row.get("listing_id") or row.get("ref")
+        if rid is None:
+            continue
+        s = str(rid)
+        if s not in seen:
+            seen.add(s)
+            out.append(s)
+    return out
+
+
+def _pull_one(
+    lid: str,
+    *,
+    get_listing: CallTool,
+    fetch_pdf: FetchBytes,
+    out_listings_dir: Path,
+    out_pdf_dir: Path,
+    listing_tool: str,
+    om_url: Callable[[dict[str, Any]], str | None],
+    skip_existing: bool,
+) -> dict[str, str]:
+    pdf_path = out_pdf_dir / f"{lid}.pdf"
+    if skip_existing and pdf_path.exists() and (out_listings_dir / f"{lid}.json").exists():
+        return {"id": lid, "status": "exists"}
+    try:
+        listing = get_listing(listing_tool, {"ref": lid})
+    except Exception as e:  # noqa: BLE001 - report per-listing, keep going
+        return {"id": lid, "status": "listing-error", "detail": str(e)}
+    (out_listings_dir / f"{lid}.json").write_text(
+        json.dumps(listing, indent=2, ensure_ascii=False), encoding="utf-8"
+    )
+    url = om_url(listing)
+    if not url:
+        return {"id": lid, "status": "no-om"}
+    try:
+        pdf_path.write_bytes(fetch_pdf(url))
+        return {"id": lid, "status": "ok"}
+    except Exception as e:  # noqa: BLE001
+        return {"id": lid, "status": "pdf-error", "detail": str(e)}
+
+
+def pull(
+    ids: list[str],
+    *,
+    get_listing: CallTool,
+    fetch_pdf: FetchBytes,
+    out_listings_dir: Path,
+    out_pdf_dir: Path,
+    listing_tool: str = "get_listing",
+    om_url: Callable[[dict[str, Any]], str | None] = om_url_of,
+    skip_existing: bool = False,
+    jobs: int = 1,
+) -> dict[str, Any]:
+    """Fetch each listing id -> write ``<id>.json`` and download its OM PDF -> ``<id>.pdf``.
+
+    Returns ``{pulled, of, counts, results}`` (results in input order). Pure except the injected
+    effects (get_listing / fetch_pdf / filesystem), so it is deterministic + testable. A per-listing
+    error is captured (never aborts the run); ``skip_existing`` avoids re-pulling already-downloaded
+    OMs (resume); ``jobs>1`` downloads concurrently (I/O-bound) while keeping input order."""
+    out_listings_dir.mkdir(parents=True, exist_ok=True)
+    out_pdf_dir.mkdir(parents=True, exist_ok=True)
+
+    def do(lid: str) -> dict[str, str]:
+        return _pull_one(
+            lid, get_listing=get_listing, fetch_pdf=fetch_pdf,
+            out_listings_dir=out_listings_dir, out_pdf_dir=out_pdf_dir,
+            listing_tool=listing_tool, om_url=om_url, skip_existing=skip_existing,
+        )
+
+    if jobs > 1 and len(ids) > 1:
+        with ThreadPoolExecutor(max_workers=jobs) as ex:
+            results = list(ex.map(do, ids))  # ex.map preserves input order
+    else:
+        results = [do(lid) for lid in ids]
+
+    counts: dict[str, int] = {}
+    for r in results:
+        counts[r["status"]] = counts.get(r["status"], 0) + 1
+    pulled = counts.get("ok", 0)
+    return {"pulled": pulled, "of": len(ids), "counts": counts, "results": results}
+
+
+# --- real MCP Streamable-HTTP transport (network; parser unit-tested, live call env-gated) ---
+def parse_rpc(content_type: str, body: str) -> dict[str, Any]:
+    """Parse an MCP HTTP response body that is application/json OR an SSE (text/event-stream)."""
+    if "text/event-stream" in content_type:
+        data = [
+            ln[5:].strip()
+            for ln in body.splitlines()
+            if ln.startswith("data:") and ln[5:].strip()
+        ]
+        if not data:
+            raise ValueError("empty SSE response")
+        parsed: dict[str, Any] = json.loads(data[-1])
+        return parsed
+    body_parsed: dict[str, Any] = json.loads(body)
+    return body_parsed
+
+
+def listing_from_result(rpc: dict[str, Any]) -> dict[str, Any]:
+    """Pull the listing out of a tools/call result (structuredContent, or a JSON text block)."""
+    if "error" in rpc and rpc["error"]:
+        err = rpc["error"]
+        raise RuntimeError(f"Buildout MCP error {err.get('code')}: {err.get('message')}")
+    result = rpc.get("result") or {}
+    sc = result.get("structuredContent")
+    if isinstance(sc, dict):
+        return sc
+    for block in result.get("content") or []:
+        if isinstance(block, dict) and block.get("type") == "text" and block.get("text"):
+            text_parsed: dict[str, Any] = json.loads(block["text"])
+            return text_parsed
+    raise RuntimeError("Buildout MCP returned no listing content")
+
+
+def mcp_http_call_tool(
+    endpoint: str,
+    token: str | None,
+    tool: str,
+    arguments: dict[str, Any],
+    *,
+    opener: Callable[[urllib.request.Request], Any] = urllib.request.urlopen,
+) -> dict[str, Any]:
+    """One tools/call over MCP Streamable HTTP: initialize -> initialized -> tools/call. Network."""
+
+    def post(session_id: str | None, payload: dict[str, Any]) -> Any:
+        headers = {
+            "Content-Type": "application/json",
+            "Accept": "application/json, text/event-stream",
+        }
+        if token:
+            headers["Authorization"] = f"Bearer {token}"
+        if session_id:
+            headers["Mcp-Session-Id"] = session_id
+        req = urllib.request.Request(
+            endpoint, data=json.dumps(payload).encode(), headers=headers, method="POST"
+        )
+        return opener(req)
+
+    init = post(
+        None,
+        {
+            "jsonrpc": "2.0", "id": 1, "method": "initialize",
+            "params": {
+                "protocolVersion": "2024-11-05", "capabilities": {},
+                "clientInfo": {"name": "openom-cli", "version": "0.1"},
+            },
+        },
+    )
+    session_id = init.headers.get("mcp-session-id")
+    post(session_id, {"jsonrpc": "2.0", "method": "notifications/initialized"})
+    resp = post(
+        session_id,
+        {"jsonrpc": "2.0", "id": 2, "method": "tools/call",
+         "params": {"name": tool, "arguments": arguments}},
+    )
+    body = resp.read().decode()
+    return listing_from_result(parse_rpc(resp.headers.get("content-type") or "", body))
+
+
+def http_fetch_pdf(url: str, *, opener: Callable[[str], Any] = urllib.request.urlopen) -> bytes:
+    """Download PDF bytes from an https URL (network)."""
+    with opener(url) as r:
+        data: bytes = r.read()
+        return data

openom_cli/humanize.py

@@ -0,0 +1,57 @@
+# SPDX-License-Identifier: MIT
+"""Plain-English rendering of validation findings for the terminal, so "all validation errors, haha"
+never happens again.
+
+Lives in the CLI, NOT the deterministic core (the core stays a pure library emitting stable codes).
+The machine-readable JSON still goes to stdout unchanged; these strings are the friendly stderr
+coaching. Every line leads with what to do and keeps the raw code in a trailing parenthetical.
+"""
+
+from __future__ import annotations
+
+import re
+
+_FOOTER = (
+    "Fix these in your payload and re-run. Starting from scratch? `om init` writes a valid "
+    "template. Not a developer? https://openom.app/embed/ builds the payload for you."
+)
+
+
+def _word(w: str) -> str:
+    # keep acronyms (PSF, SF, NOI, APN) as-is; lowercase ordinary words (Rate -> rate)
+    return w if w.isupper() else w.lower()
+
+
+def humanize_path(path: str) -> str:
+    """'/deal/capRate' -> 'deal > cap rate' (acronyms like PSF/NOI/SF kept uppercase)."""
+    parts = [p for p in path.strip("/").split("/") if p]
+    out: list[str] = []
+    for p in parts:
+        if p.isdigit():
+            out.append(f"#{int(p) + 1}")
+        else:
+            words = re.sub(r"(?<=[a-z])(?=[A-Z])", " ", p).split(" ")
+            out.append(" ".join(_word(w) for w in words))
+    return " > ".join(out) or "(payload root)"
+
+
+def humanize_finding(code: str, path: str, message: str) -> str:
+    """One plain-English line for a finding, keyed on the codes a broker actually hits."""
+    label = humanize_path(path)
+    if code == "OMV-E001" and path == "/deal/capRate":
+        return (
+            "Cap rate must be a decimal fraction between 0 and 1 - enter 6.25% as 0.0625 "
+            "(not 6.25). Fix deal.capRate. (OMV-E001)"
+        )
+    if code == "OMV-E002":
+        return (
+            "You set an NOI, so you must also say whether it's 'in-place' or 'pro-forma' "
+            "(deal.noiType) and its as-of date (deal.noiAsOfDate). Add both. (OMV-E002)"
+        )
+    if "currency" in path.lower():
+        return f"{label}: currency must be a 3-letter ISO 4217 code like USD. ({code})"
+    return f"{label}: {message} ({code})"
+
+
+def footer() -> str:
+    return _FOOTER