openom-cli 0.1.0__tar.gz → 0.1.1__tar.gz

{openom_cli-0.1.0 → openom_cli-0.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: openom-cli
-Version: 0.1.0
+Version: 0.1.1
 Summary: openOM CLI - the `om` command over openom-core. Zero inference.
 Project-URL: Homepage, https://openom.app
 Project-URL: Documentation, https://openom.app/docs/

{openom_cli-0.1.0 → openom_cli-0.1.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "openom-cli"
-version = "0.1.0"
+version = "0.1.1"
 description = "openOM CLI - the `om` command over openom-core. Zero inference."
 readme = "README.md"
 requires-python = ">=3.11"

openom_cli-0.1.1/src/openom_cli/csv_map.py

@@ -0,0 +1,171 @@
+# SPDX-License-Identifier: MIT
+"""Deterministic CSV row -> openOM payload mapper (the spreadsheet on-ramp for bulk seeding).
+
+A broker with a back catalog usually has a spreadsheet + a folder of PDFs, not Buildout JSON. This
+maps one CSV row (a documented canonical column set) to a schema-valid openOM payload, ready for
+``om embed-batch`` - the same output shape as the Buildout bridge, different input. Pure + zero
+inference: every value comes from the broker's own cell (that IS the assertion); absent cells are
+omitted, never guessed. The assertion identity (assertedBy / assertedDate / noiType / noiAsOfDate)
+is supplied by the caller and stamped verbatim.
+
+Numeric/date/state normalization reuses the SAME helpers as the Buildout mapper, so a number, a
+percent, or an M/D/Y date maps identically on both on-ramps. Percentage columns are named ``*Pct``
+(e.g. ``capRatePct`` = 6.25, not 0.0625) so a non-technical broker can't misread the unit.
+"""
+
+from __future__ import annotations
+
+import csv
+import io
+from collections.abc import Mapping
+from typing import Any
+
+from .buildout import (
+    NS,
+    _compact,
+    _int,
+    _iso_date,
+    _lease_type,
+    _months_between,
+    _num,
+    _pct_to_fraction,
+    _state_code,
+)
+
+# The canonical header vocabulary a broker fills in. Order is the template's column order. Every
+# column is optional except that a row must name its PDF (an ``id`` -> <id>.pdf, or an explicit
+# ``pdf`` filename) and produce a schema-valid payload. ``*Pct`` columns are percentages.
+CANONICAL_COLUMNS: tuple[str, ...] = (
+    "id", "pdf",
+    "broker", "brokerage", "license", "noiType", "noiAsOfDate",
+    "streetAddress", "city", "state", "postalCode", "country",
+    "propertyType", "buildingSF", "yearBuilt", "lotAcres", "units", "occupancyPct",
+    "latitude", "longitude",
+    "askingPrice", "capRatePct", "noi", "status",
+    "tenant", "leaseType", "commencement", "expiration", "guarantor",
+)
+
+# Per-row overrides of the assertion identity (a catalog can span brokers / NOI types).
+_OVERRIDE_COLUMNS: tuple[str, ...] = ("broker", "brokerage", "license", "noiType", "noiAsOfDate")
+
+_EXAMPLE_ROW: dict[str, str] = {
+    "id": "123-main", "pdf": "123-main.pdf",
+    "streetAddress": "123 Main St", "city": "Austin", "state": "TX", "postalCode": "78701",
+    "propertyType": "retail", "buildingSF": "9100", "yearBuilt": "2019",
+    "askingPrice": "1850000", "capRatePct": "6.25", "noi": "115625",
+    "tenant": "Example Retail, LLC", "leaseType": "NNN",
+    "commencement": "5/1/2019", "expiration": "4/30/2034",
+}
+
+
+def _cell(row: Mapping[str, str], key: str) -> str | None:
+    """A trimmed cell value, or None when the column is absent/blank (so _compact drops it)."""
+    v = row.get(key)
+    if v is None:
+        return None
+    s = str(v).strip()
+    return s or None
+
+
+def _date(v: str | None) -> str | None:
+    """Accept either an ISO date (pass-through) or an M/D/Y spreadsheet date."""
+    if not v:
+        return None
+    s = str(v).strip()
+    parts = s.split("-")
+    if len(parts) == 3 and len(parts[0]) == 4:  # already ISO-ish -> validate via round-trip
+        try:
+            y, m, d = (int(p) for p in parts)
+        except ValueError:
+            return None
+        if 1 <= m <= 12 and 1 <= d <= 31 and y > 1900:
+            return f"{y:04d}-{m:02d}-{d:02d}"
+        return None
+    return _iso_date(s)
+
+
+def override_identity(row: Mapping[str, str]) -> dict[str, str]:
+    """The per-row assertion-identity overrides present in this row (subset of the override set)."""
+    return {k: v for k in _OVERRIDE_COLUMNS if (v := _cell(row, k)) is not None}
+
+
+def row_to_payload(
+    row: Mapping[str, str],
+    *,
+    asserted_by: dict[str, str],
+    asserted_date: str,
+    noi_type: str,
+    noi_as_of: str | None = None,
+) -> dict[str, Any]:
+    """Map one canonical CSV row to a schema-valid openOM payload (only the fields the row carries).
+    ``asserted_by``/``asserted_date``/``noi_type``/``noi_as_of`` are the assertion identity and are
+    stamped verbatim, never inferred from the row."""
+    address = _compact({
+        "streetAddress": _cell(row, "streetAddress"),
+        "addressLocality": _cell(row, "city"),
+        "addressRegion": _state_code(_cell(row, "state")),
+        "postalCode": _cell(row, "postalCode"),
+        "addressCountry": _cell(row, "country") or ("US" if _cell(row, "state") else None),
+    })
+    lat, lng = _num(_cell(row, "latitude")), _num(_cell(row, "longitude"))
+    geo = {"latitude": lat, "longitude": lng} if lat is not None and lng is not None else None
+    building_sf = _int(_cell(row, "buildingSF"))
+    units = _int(_cell(row, "units"))
+    prop_type = _cell(row, "propertyType")
+    property_ = _compact({
+        "propertyType": prop_type.lower() if prop_type else None,
+        "address": address or None,
+        "geo": geo,
+        "buildingSF": building_sf,
+        "yearBuilt": _int(_cell(row, "yearBuilt")),
+        "lotAcres": _num(_cell(row, "lotAcres")),
+        "units": units,
+        "occupancy": _pct_to_fraction(_cell(row, "occupancyPct")),
+    })
+
+    price = _int(_cell(row, "askingPrice"))
+    deal = _compact({
+        "askingPrice": price,
+        "capRate": _pct_to_fraction(_cell(row, "capRatePct")),
+        "noi": _int(_cell(row, "noi")),
+        "pricePerUnit": round(price / units) if price and units else None,
+        "pricePerSF": round(price / building_sf, 2) if price and building_sf else None,
+        "noiType": noi_type,
+        "noiAsOfDate": noi_as_of or asserted_date,
+        "status": _cell(row, "status") or "active",
+    })
+
+    commencement = _date(_cell(row, "commencement"))
+    expiration = _date(_cell(row, "expiration"))
+    guarantor_name = _cell(row, "guarantor")
+    lease = _compact({
+        "tenantEntity": _cell(row, "tenant"),
+        "leaseTypeAsserted": _lease_type(_cell(row, "leaseType")),
+        "commencement": commencement,
+        "expiration": expiration,
+        "termMonths": _months_between(commencement, expiration),
+        "guarantor": {"name": guarantor_name, "type": "corporate"} if guarantor_name else None,
+    })
+
+    return _compact({
+        "@context": ["https://schema.org", NS],
+        "@type": "RealEstateListing",
+        "specVersion": "0.1",
+        "assertedBy": _compact(dict(asserted_by)),
+        "assertedDate": asserted_date,
+        "property": property_ or None,
+        "deal": deal or None,
+        "lease": lease or None,
+        "meta": {"supersedes": None},
+    })
+
+
+def template_csv() -> str:
+    """A blank template: the canonical header row + one worked example, so a broker knows exactly
+    what to fill in. Assertion identity (broker/brokerage/license/noiType) normally comes from the
+    command flags; the columns exist only for a catalog that spans brokers."""
+    buf = io.StringIO()
+    writer = csv.writer(buf, lineterminator="\n")  # csv.writer quotes cells containing commas
+    writer.writerow(CANONICAL_COLUMNS)
+    writer.writerow([_EXAMPLE_ROW.get(c, "") for c in CANONICAL_COLUMNS])
+    return buf.getvalue()

{openom_cli-0.1.0 → openom_cli-0.1.1}/src/openom_cli/main.py

@@ -11,9 +11,11 @@ affect the exit code.
 
 from __future__ import annotations
 
+import csv
 import dataclasses
 import datetime
 import functools
+import io
 import json
 import sys
 import time
@@ -40,6 +42,7 @@ from openom_core.validate import validate as _validate
 from openom_cli import profile as _profile
 from openom_cli import scaffold as _scaffold
 from openom_cli.buildout import listing_to_payload, payload_coverage
+from openom_cli.csv_map import CANONICAL_COLUMNS, override_identity, row_to_payload, template_csv
 from openom_cli.humanize import footer as _err_footer
 from openom_cli.humanize import humanize_finding as _humanize
 
@@ -635,6 +638,127 @@ def buildout_manifest(
     })
 
 
+@app.command(name="csv-manifest")
+@_guard
+def csv_manifest(  # noqa: C901 - a linear map-each-row-then-report, read top-down
+    csv_file: Annotated[
+        Path | None, typer.Option("--csv", help="Spreadsheet of listings (canonical columns)")
+    ] = None,
+    pdf_dir: Annotated[
+        Path | None, typer.Option(help="Dir of the OM PDFs (named per each row's id/pdf column)")
+    ] = None,
+    out_dir: Annotated[
+        Path | None, typer.Option(help="Where payload sidecars + manifest.json are written")
+    ] = None,
+    broker: Annotated[str | None, typer.Option(help="assertedBy.broker (who is asserting)")] = None,
+    brokerage: Annotated[str | None, typer.Option(help="assertedBy.brokerage")] = None,
+    license: Annotated[str | None, typer.Option(help="assertedBy.license")] = None,  # noqa: A002
+    asserted_date: Annotated[
+        str | None, typer.Option(help="ISO 8601 assertion date")
+    ] = None,
+    noi_type: Annotated[
+        str | None, typer.Option(help="in-place | pro-forma (a required assertion)")
+    ] = None,
+    noi_as_of: Annotated[
+        str | None, typer.Option(help="deal.noiAsOfDate (default: --asserted-date)")
+    ] = None,
+    min_fields: Annotated[
+        int, typer.Option(help="Flag any mapped payload with fewer than this many tracked fields")
+    ] = 3,
+    template: Annotated[
+        Path | None,
+        typer.Option(help="Write a blank template CSV (headers + one example) here, then exit"),
+    ] = None,
+) -> None:
+    """Bridge: turn a broker's spreadsheet into an ``om embed-batch`` manifest (catalog seed).
+
+    The low-touch on-ramp: a broker exports their back catalog to a CSV (the canonical columns; run
+    ``--template`` to get a blank one) and drops the OM PDFs in a folder. Each row maps
+    deterministically (zero inference) to a schema-valid openOM payload written as ``<id>.om.json``,
+    paired with its ``<id>.pdf`` in a manifest, with a ``coverage.json`` triage report. Assertion
+    identity comes from the flags (or per-row ``broker``/``brokerage``/``license``/``noiType``
+    columns for a multi-broker catalog), stamped verbatim - never inferred. Review the payloads (the
+    assertion gate), then run ``om embed-batch --manifest <out-dir>/manifest.json``.
+    """
+    if template is not None:
+        template.write_text(template_csv(), encoding="utf-8")
+        if not _output.quiet:
+            typer.echo(f"csv-manifest: wrote a template with {len(CANONICAL_COLUMNS)} columns "
+                       f"-> {template}", err=True)
+        _emit({"template": str(template), "columns": list(CANONICAL_COLUMNS)})
+        return
+
+    missing = [
+        n for n, v in (
+            ("--csv", csv_file), ("--pdf-dir", pdf_dir), ("--out-dir", out_dir),
+            ("--broker", broker), ("--brokerage", brokerage), ("--license", license),
+            ("--asserted-date", asserted_date), ("--noi-type", noi_type),
+        ) if not v
+    ]
+    if missing:
+        typer.echo(f"error: missing required option(s): {', '.join(missing)}", err=True)
+        raise typer.Exit(2)
+    # Narrow every required option to non-None for the type checker (the guard above proved it).
+    assert csv_file and pdf_dir and out_dir and asserted_date
+    assert broker and brokerage and license and noi_type
+
+    default_by = {"broker": broker, "brokerage": brokerage, "license": license}
+    rows = list(csv.DictReader(io.StringIO(_read_bytes(csv_file).decode("utf-8-sig"))))
+    out_dir.mkdir(parents=True, exist_ok=True)
+    manifest: list[dict[str, str]] = []
+    skipped: list[dict[str, str]] = []
+    coverage: list[dict[str, Any]] = []
+    sparse: list[str] = []
+    seen: set[str] = set()
+    for i, row in enumerate(rows):
+        pdf_name = (row.get("pdf") or "").strip() or (
+            f"{(row.get('id') or '').strip()}.pdf" if (row.get("id") or "").strip() else ""
+        )
+        stem = Path(pdf_name).stem if pdf_name else f"row{i + 1}"
+        if not pdf_name:
+            skipped.append({"id": stem, "reason": "row names no pdf (needs an 'id' or 'pdf')"})
+            continue
+        if stem in seen:
+            skipped.append({"id": stem, "reason": "duplicate id/pdf in the CSV"})
+            continue
+        seen.add(stem)
+        pdf = pdf_dir / pdf_name
+        if not pdf.exists():
+            skipped.append({"id": stem, "reason": f"no OM PDF at {pdf_name}"})
+            continue
+        ov = override_identity(row)
+        asserted_by = {k: ov.get(k, default_by[k]) for k in ("broker", "brokerage", "license")}
+        payload = row_to_payload(
+            row, asserted_by=asserted_by, asserted_date=asserted_date,
+            noi_type=ov.get("noiType", noi_type), noi_as_of=ov.get("noiAsOfDate", noi_as_of),
+        )
+        sidecar = out_dir / f"{stem}.om.json"
+        sidecar.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8")
+        manifest.append(
+            {"pdf": str(pdf.resolve()), "payload": str(sidecar.resolve()),
+             "assertedDate": asserted_date}
+        )
+        cov = payload_coverage(payload)
+        coverage.append({"id": stem, **cov})
+        if cov["filled"] < min_fields:
+            sparse.append(stem)
+    (out_dir / "manifest.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8")
+    (out_dir / "coverage.json").write_text(
+        json.dumps({"listings": coverage, "sparse": sparse, "minFields": min_fields}, indent=2),
+        encoding="utf-8",
+    )
+    if not _output.quiet:
+        note = f", {len(sparse)} sparse (<{min_fields} fields)" if sparse else ""
+        typer.echo(
+            f"csv-manifest: {len(manifest)} mapped, {len(skipped)} skipped{note} -> {out_dir}",
+            err=True,
+        )
+    _emit({
+        "mapped": len(manifest), "skipped": skipped, "sparse": sparse,
+        "manifest": str(out_dir / "manifest.json"), "coverage": str(out_dir / "coverage.json"),
+    })
+
+
 @app.command(name="buildout-pull")
 @_guard
 def buildout_pull(

openom_cli-0.1.1/tests/test_csv_map.py

@@ -0,0 +1,160 @@
+"""Cover the CSV->openOM mapper (pure, deterministic - the spreadsheet on-ramp for bulk seeding)."""
+
+from __future__ import annotations
+
+import csv
+import io
+import json
+from pathlib import Path
+from typing import Any
+
+import jsonschema
+import pikepdf
+from typer.testing import CliRunner
+
+from openom_cli.csv_map import (
+    CANONICAL_COLUMNS,
+    override_identity,
+    row_to_payload,
+    template_csv,
+)
+from openom_cli.main import app
+
+_runner = CliRunner()
+
+SPEC = Path(__file__).resolve().parents[2] / "spec"
+
+
+def _schema() -> dict[str, Any]:
+    return json.loads((SPEC / "om-0.1.schema.json").read_text(encoding="utf-8"))
+
+
+_BY = {"broker": "Jane Example", "brokerage": "Example Advisors", "license": "TX 12345"}
+
+
+def _map(row: dict[str, str]) -> dict[str, Any]:
+    return row_to_payload(
+        row, asserted_by=_BY, asserted_date="2026-08-15", noi_type="in-place"
+    )
+
+
+def test_full_row_maps_to_a_schema_valid_payload() -> None:
+    row = {
+        "id": "123-main", "pdf": "123-main.pdf",
+        "streetAddress": "123 Main St", "city": "Austin", "state": "TX - Texas",
+        "postalCode": "78701", "propertyType": "Retail", "buildingSF": "9,100",
+        "yearBuilt": "2019", "askingPrice": "1,850,000", "capRatePct": "6.25", "noi": "115625",
+        "tenant": "Example Retail, LLC", "leaseType": "NNN Lease",
+        "commencement": "5/1/2019", "expiration": "4/30/2034",
+    }
+    payload = _map(row)
+    jsonschema.Draft202012Validator(_schema()).validate(payload)  # raises if invalid
+    assert payload["deal"]["capRate"] == 0.0625  # percent -> fraction
+    assert payload["deal"]["askingPrice"] == 1850000  # commas stripped
+    assert payload["property"]["address"]["addressRegion"] == "TX"  # 'TX - Texas' -> 'TX'
+    assert payload["property"]["propertyType"] == "retail"  # lowercased
+    assert payload["lease"]["leaseTypeAsserted"] == "NNN"
+    assert payload["lease"]["commencement"] == "2019-05-01"  # M/D/Y -> ISO
+    assert payload["lease"]["termMonths"] == 179  # derived, deterministic
+    assert payload["property"]["address"]["addressCountry"] == "US"  # defaulted from a US state
+    assert payload["deal"]["pricePerSF"] == round(1850000 / 9100, 2)  # derived
+
+
+def test_iso_dates_pass_through() -> None:
+    payload = _map({"commencement": "2019-05-01", "expiration": "2034-04-30"})
+    assert payload["lease"]["commencement"] == "2019-05-01"
+    assert payload["lease"]["expiration"] == "2034-04-30"
+
+
+def test_malformed_dates_are_dropped_not_crashed() -> None:
+    # A garbled ISO-looking date or an unparseable value is omitted (never a crash, never a guess).
+    payload = _map({"askingPrice": "1000000", "commencement": "2019-13-40", "expiration": "later"})
+    assert "commencement" not in payload.get("lease", {})
+    assert "expiration" not in payload.get("lease", {})
+    # a 4-digit-year, 3-part value with non-integer parts hits the ISO parse-failure branch
+    assert "lease" not in _map({"askingPrice": "1", "commencement": "20x9-05-01"})
+
+
+def test_absent_cells_are_omitted_never_guessed() -> None:
+    payload = _map({"askingPrice": "1000000"})  # nothing else
+    assert "capRate" not in payload["deal"]
+    assert "lease" not in payload  # no lease cells -> no lease object
+    assert "property" not in payload  # no property cells -> no property object
+    assert payload["assertedBy"] == _BY  # identity stamped verbatim
+
+
+def test_blank_cells_treated_as_absent() -> None:
+    payload = _map({"askingPrice": "1000000", "capRatePct": "  ", "tenant": ""})
+    assert "capRate" not in payload["deal"]
+    assert "lease" not in payload
+
+
+def test_override_identity_reads_only_present_override_columns() -> None:
+    row = {"broker": "Sam Other", "askingPrice": "1", "city": "Austin"}
+    assert override_identity(row) == {"broker": "Sam Other"}
+    assert override_identity({"askingPrice": "1"}) == {}
+
+
+def test_template_has_headers_and_one_example_row() -> None:
+    text = template_csv()
+    rows = list(csv.reader(io.StringIO(text)))
+    assert rows[0] == list(CANONICAL_COLUMNS)  # header is the canonical vocabulary
+    assert len(rows) == 2  # header + one worked example
+    # the example row itself maps to a schema-valid payload
+    example = dict(zip(CANONICAL_COLUMNS, rows[1], strict=True))
+    jsonschema.Draft202012Validator(_schema()).validate(_map(example))
+
+
+# --- end-to-end CLI: the full spreadsheet on-ramp (csv-manifest -> embed-batch -> read) ---
+def _blank_pdf(path: Path) -> None:
+    pdf = pikepdf.new()
+    pdf.add_blank_page(page_size=(612, 792))
+    buf = io.BytesIO()
+    pdf.save(buf)
+    path.write_bytes(buf.getvalue())
+
+
+def test_csv_manifest_template_then_embed_batch_roundtrip(tmp_path: Path) -> None:
+    # 1) --template writes a fillable CSV; a broker fills two rows.
+    tmpl = tmp_path / "template.csv"
+    r0 = _runner.invoke(app, ["csv-manifest", "--template", str(tmpl)])
+    assert r0.exit_code == 0, r0.output
+    assert tmpl.read_text(encoding="utf-8").splitlines()[0].startswith("id,pdf,")
+
+    pdf_dir = tmp_path / "pdfs"
+    pdf_dir.mkdir()
+    _blank_pdf(pdf_dir / "deal-1.pdf")
+    _blank_pdf(pdf_dir / "deal-2.pdf")
+    csv_file = tmp_path / "catalog.csv"
+    csv_file.write_text(
+        "id,streetAddress,city,state,askingPrice,capRatePct,noi\n"
+        "deal-1,1 A St,Austin,TX,1850000,6.25,115625\n"
+        "deal-2,2 B St,Dallas,TX,2400000,7.0,168000\n"
+        "deal-3,3 C St,Waco,TX,900000,8,72000\n",  # deal-3 has no PDF -> skipped
+        encoding="utf-8",
+    )
+    out = tmp_path / "mapped"
+    r1 = _runner.invoke(
+        app,
+        ["csv-manifest", "--csv", str(csv_file), "--pdf-dir", str(pdf_dir), "--out-dir", str(out),
+         "--broker", "Jane Example", "--brokerage", "Example Advisors", "--license", "TX 12345",
+         "--asserted-date", "2026-08-15", "--noi-type", "in-place"],
+    )
+    assert r1.exit_code == 0, r1.output
+    summary = json.loads(r1.output[r1.output.index("{"):])
+    assert summary["mapped"] == 2
+    assert any(s["id"] == "deal-3" for s in summary["skipped"])  # no PDF -> skipped with a reason
+
+    # 2) embed-batch consumes the produced manifest; both OMs round-trip hash-valid.
+    r2 = _runner.invoke(
+        app,
+        ["embed-batch", "--manifest", str(out / "manifest.json"),
+         "--out-dir", str(tmp_path / "embedded")],
+    )
+    assert r2.exit_code == 0, r2.output
+    for name in ("deal-1.pdf", "deal-2.pdf"):
+        rr = _runner.invoke(app, ["read", str(tmp_path / "embedded" / name)])
+        assert rr.exit_code == 0, rr.output
+        got = json.loads(rr.output)
+        assert got["verification"]["hashValid"] is True
+        assert got["payload"]["assertedBy"]["broker"] == "Jane Example"