gea-program 0.47.0__tar.gz → 0.48.0__tar.gz

{gea_program-0.47.0 → gea_program-0.48.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gea-program
-Version: 0.47.0
+Version: 0.48.0
 Summary: GEA-Program: downhole gauge monitoring - canonical measurement records, quality rules, drift evaluation with a re-fit change log, well-test validation, alarm management, model cards, store-and-forward, monthly SLA measurement, FAT/SAT protocols and a dashboard, generated from live objects in the client's reporting outline.
 Author: Daniel T. Murphy
 License: MPL-2.0
@@ -648,6 +648,19 @@ against it; `gea seedlink --config ... --hello` prints what a server says of
 itself; `gea help seedlink`. A TexNet station is served by EarthScope at
 `rtserve.iris.washington.edu:18000`, network `TX`.
 
+### One record, tied together
+
+The invoice check, rig time, the daily summary, the alerts, the signed
+dossier and the declarations sheet now read each other: an invoice that
+differs is an alert; the morning summary carries each well's invoice check
+and yesterday's rig time; a signed dossier carries the contract, the
+invoices and daily reports as attached, the rig-time targets and casing,
+and both checks as they read at signing. The declarations sheet declares
+the contract, the rig-time targets, block weights, casing shoes and the
+watch and summary settings, and every file the program reads as a table -
+the sheet, an invoice, a daily report - may be an Excel workbook. Casing
+shoes split rig time by hole section.
+
 ### Rig time
 
 What the rig did each second of the drilling record, read from the hook

{gea_program-0.47.0 → gea_program-0.48.0}/README.md

@@ -609,6 +609,19 @@ against it; `gea seedlink --config ... --hello` prints what a server says of
 itself; `gea help seedlink`. A TexNet station is served by EarthScope at
 `rtserve.iris.washington.edu:18000`, network `TX`.
 
+### One record, tied together
+
+The invoice check, rig time, the daily summary, the alerts, the signed
+dossier and the declarations sheet now read each other: an invoice that
+differs is an alert; the morning summary carries each well's invoice check
+and yesterday's rig time; a signed dossier carries the contract, the
+invoices and daily reports as attached, the rig-time targets and casing,
+and both checks as they read at signing. The declarations sheet declares
+the contract, the rig-time targets, block weights, casing shoes and the
+watch and summary settings, and every file the program reads as a table -
+the sheet, an invoice, a daily report - may be an Excel workbook. Casing
+shoes split rig time by hole section.
+
 ### Rig time
 
 What the rig did each second of the drilling record, read from the hook

{gea_program-0.47.0 → gea_program-0.48.0}/gea/__init__.py

@@ -123,7 +123,7 @@ from .profile_catalog import (
     read_pangaea_txt,
 )
 
-__version__ = "0.47.0"
+__version__ = "0.48.0"
 __all__ = [
     "aging_rate", "rate_psi_yr", "accuracy_psi",
     "Sensor", "SimulatorConfig", "DownholeEngine",

{gea_program-0.47.0 → gea_program-0.48.0}/gea/__main__.py

@@ -599,6 +599,9 @@ def main(argv=None) -> int:
     p_rt.add_argument("--trip-out", type=str, default=None, help="declare the tripping-out target, ft/h")
     p_rt.add_argument("--source", type=str, default="", help="where the targets came from")
     p_rt.add_argument("--block-weight", type=str, default=None, help="with --well: the block weight in klbf where the estimate is wrong ('' removes it)")
+    p_rt.add_argument("--casing", type=str, default=None, help="with --well, --shoe-md and --source: declare a casing shoe, its size in inches as written (13-3/8)")
+    p_rt.add_argument("--shoe-md", type=str, default=None, help="the shoe's measured depth, ft")
+    p_rt.add_argument("--set-time", type=str, default=None, help="when the casing was set (the recorder's clock); otherwise when the hole is 50 ft past the shoe")
     p_rt.add_argument("--actor", type=str, default="cli")
     p_rt.add_argument("--json", action="store_true")
 
@@ -2205,9 +2208,10 @@ def main(argv=None) -> int:
         if not a.file:
             raise SystemExit("declare: give --file <sheet.csv> (or --template <out.csv> to start one)")
         try:
-            with open(a.file, encoding="utf-8-sig") as f:
-                text_ = f.read()
-        except OSError as e:
+            from . import xlsx_read as _XR
+            with open(a.file, "rb") as f:
+                text_ = _XR.text_of(f.read())
+        except (OSError, ValueError, KeyError) as e:
             raise SystemExit(f"declare: {e}")
         p = _DC.apply(ws, text_, a.actor, os.path.dirname(os.path.abspath(a.file)), write=a.apply)
         print(_json.dumps(p, default=str) if a.json else _DC.text(p))
@@ -2376,6 +2380,12 @@ def main(argv=None) -> int:
         _safe_stdout()
         try:
             ws = Workspace(a.workspace)
+            if a.casing:
+                if not a.well:
+                    raise ValueError("--casing needs --well")
+                c_ = _RS.set_casing(ws, a.actor, a.well, a.casing, a.shoe_md, a.source, a.set_time)
+                print(f"casing {c_['size']} in, shoe at {c_['shoe_md_ft']:,.0f} ft MD ({c_['source']})")
+                return 0
             if a.w2w_min is not None or a.trip_in is not None or a.trip_out is not None or a.block_weight is not None:
                 if a.block_weight is not None and not a.well:
                     raise ValueError("--block-weight needs --well")

{gea_program-0.47.0 → gea_program-0.48.0}/gea/acceptance_tests.py

@@ -10913,6 +10913,206 @@ def section_cp_rig_time(tmp: str) -> None:
        + (f" [failed: {', '.join(failed3)}]" if failed3 else ""))
 
 
+def section_cq_tied_together(tmp: str) -> None:
+    """Section CQ - the recent parts tied together: an Excel workbook read without Excel (shared and inline strings,
+    numbers, booleans, dates and times from their cell formats, the 1904 epoch, a named sheet) and taken wherever a
+    CSV is - the invoice check's files and the declarations sheet; the declarations sheet's new tables (the contract,
+    the rig-time targets, block weights, casing, the watch and summary settings) planned, written, round-tripped and
+    refused when wrong; rig time by hole section from declared casing; the invoice differing raised as an alert and
+    cleared; the daily summary carrying each invoice check and yesterday's rig time; a signed dossier carrying the
+    contract, the files, the targets, the casing and both checks as they read at signing; the API, the CLI, the tabs
+    and the help pages."""
+    import http.cookiejar
+    import subprocess as _sp
+    import urllib.request
+    import urllib.error
+    import base64 as _b64
+    import zipfile as _zf
+    from datetime import datetime as _dtm, timedelta as _td, timezone as _tz
+    from . import __version__
+    from . import alerts as AL
+    from . import declarations as DC
+    from . import dossier as DZ
+    from . import helplib as H
+    from . import invoice_check as IC
+    from . import pad_summary as PS
+    from . import rig_state as RS
+    from . import seismic_watch as SW
+    from . import xlsx_read as XR
+    from .workspace import Workspace
+    from .service import Service, Users
+    pkg = Path(__file__).parent
+    d = Path(tmp, "cq"); d.mkdir()
+
+    # -- CQ1 the workbook -------------------------------------------------------------------------------------------------
+    XR.write_simple(str(d / "a.xlsx"), [["Date", "Rate Type", "Quantity", "Note"], [_dtm(2026, 7, 1), "Operating day rate", 1, 'with, a "comma"'],
+                                        [_dtm(2026, 7, 2, 6, 30), "Standby", 12.5, None]])
+    a = XR.rows((d / "a.xlsx").read_bytes())
+    ns = 'xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"'
+    with _zf.ZipFile(d / "b.xlsx", "w") as z:                 # a hand-made workbook: two sheets, inline strings, booleans, the 1904 epoch, a custom date format
+        z.writestr("xl/workbook.xml", f'<workbook {ns}><workbookPr date1904="1"/><sheets><sheet name="Notes" sheetId="1" r:id="rId1"/><sheet name="Invoice" sheetId="2" r:id="rId2"/></sheets></workbook>')
+        z.writestr("xl/_rels/workbook.xml.rels", '<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Target="worksheets/sheet1.xml" Type="x"/><Relationship Id="rId2" Target="/xl/worksheets/sheet2.xml" Type="x"/></Relationships>')
+        z.writestr("xl/styles.xml", f'<styleSheet {ns}><numFmts count="1"><numFmt numFmtId="164" formatCode="dd/mm/yyyy\\ hh:mm"/><numFmt numFmtId="165" formatCode="#,##0.00"/></numFmts>'
+                                    '<cellXfs count="3"><xf numFmtId="0"/><xf numFmtId="164"/><xf numFmtId="165"/></cellXfs></styleSheet>')
+        z.writestr("xl/worksheets/sheet1.xml", f'<worksheet {ns}><sheetData><row r="1"><c r="A1" t="inlineStr"><is><t>notes</t></is></c></row></sheetData></worksheet>')
+        z.writestr("xl/worksheets/sheet2.xml", f'<worksheet {ns}><sheetData><row r="1"><c r="A1" t="inlineStr"><is><t>When</t></is></c><c r="C1" t="inlineStr"><is><t>Paid</t></is></c></row>'
+                                               '<row r="3"><c r="A3" s="1"><v>44743.25</v></c><c r="B3" s="2"><v>1234.5</v></c><c r="C3" t="b"><v>1</v></c></row></sheetData></worksheet>')
+    bx = (d / "b.xlsx").read_bytes()
+    b2 = XR.rows(bx, "Invoice")
+    c1 = {
+        "simple": a == [["Date", "Rate Type", "Quantity", "Note"], ["2026-07-01", "Operating day rate", "1", 'with, a "comma"'], ["2026-07-02T06:30", "Standby", "12.5"]],
+        "csv": XR.to_csv((d / "a.xlsx").read_bytes()).splitlines()[1] == '2026-07-01,Operating day rate,1,"with, a ""comma"""',
+        "sheets": XR.sheets(bx) == ["Notes", "Invoice"] and XR.rows(bx) == [["notes"]],
+        "hand_made": b2 == [["When", "", "Paid"], [], ["2026-07-02T06:00", "1234.5", "TRUE"]],
+        "text_of": XR.text_of(b"a,b\n1,2\n") == "a,b\n1,2\n" and not XR.is_xlsx(b"PK not a zip"),
+    }
+    try:
+        XR.rows(bx, "Nope"); c1["named"] = False
+    except ValueError as e:
+        c1["named"] = "Notes, Invoice" in str(e)
+    failed1 = [k for k, v in c1.items() if not v]
+    ok(not failed1,
+       "CQ1 an Excel workbook read with the standard library: shared and inline strings, a comma and quotes kept, whole numbers without "
+       "'.0', dates and date-times from built-in and custom formats, the 1904 epoch, booleans, a gap row kept as a gap, the first or a "
+       "named sheet; CSV text passed through" + (f" [failed: {', '.join(failed1)}]" if failed1 else ""))
+
+    # -- CQ2 the declarations sheet, casing, and the invoice as a workbook --------------------------------------------------
+    F = RS.selftest_edr(str(d))
+    G = IC.selftest_files(str(d))
+    wsp = str(d / "ws"); ws = Workspace.create(wsp, "Pad CQ", actor="tester")
+    w1 = ws.add_well_file(F["path"], display="RIG-1", actor="tester"); w2 = ws.add_well_file(F["path"], display="RIG-2", actor="tester")
+    wi = ws.add_well_file(G["edr"], display="INV-1", actor="tester")
+    sheet = ("[contract],day_start,report_date,operating,standby,repair,repair_hours_per_month,source\n,06:00,start,30000,25000,0,2,daywork contract A\n"
+             "[rig_time],w2w_min,trip_in_ft_h,source\n,,2800,program 2026\n[block_weights],well,block_weight_klbf\n,RIG-2,42\n"
+             "[casing],well,size_in,shoe_md_ft,set,source\n,RIG-1,9-5/8,10150,,tally\n,RIG-2,9-5/8,10150,,tally\n,RIG-1,13-3/8,2000,,tally\n"
+             "[settings],setting,value\n,watch_every_min,10\n,summary_at,10:45\n,summary_send,no\n")
+    rows_ = [ln.split(",") for ln in sheet.strip().splitlines()]
+    XR.write_simple(str(d / "decl.xlsx"), rows_)
+    p_x = DC.plan(ws, XR.text_of((d / "decl.xlsx").read_bytes()))
+    p_w = DC.apply(ws, sheet, "tester", write=True)
+    p_rt = DC.plan(ws, DC.template(ws))
+    p_bad = DC.plan(ws, "[contract],operating,source\n,30000,\n[settings],setting,value\n,watch_mode,sometimes\n,colour,blue\n[casing],well,size_in,shoe_md_ft,source\n,RIG-1,9 5/8,100,x\n,NOPE,7,100,x\n")
+    pad = RS.pad(ws)
+    W1 = next(w for w in pad["wells"] if w["display"] == "RIG-1")
+    secs = [c["section"] for c in W1["connection_list"]]
+    XR.write_simple(str(d / "inv.xlsx"), [ln.split(",") if i == 0 else [(_dtm.fromisoformat(x) if j == 0 else (float(x) if re.fullmatch(r"[\d.]+", x) else x)) for j, x in enumerate(ln.split(","))]
+                                          for i, ln in enumerate(G["invoice"].strip().splitlines())])
+    IC.attach(ws, "tester", wi["id"], "ddr", "ddr.csv", G["ddr"].encode(), "reports")
+    IC.attach(ws, "tester", wi["id"], "invoice", "inv.xlsx", (d / "inv.xlsx").read_bytes(), "invoice as a workbook")
+    ic = IC.check(ws, wi["id"])
+    c2 = {
+        "xlsx_sheet": p_x["counts"]["NEW"] == 6 and p_x["counts"]["CHANGE"] == 3 and not p_x["errors"],
+        "written": p_w["applied"] and IC.terms(ws)["rates_usd_per_day"] == {"operating": 30000.0, "standby": 25000.0, "repair": 0.0}
+                   and RS.settings(ws)["targets"] == {"trip_in_ft_h": 2800.0} and RS.settings(ws)["block_weight_klbf"] == {w2["id"]: 42.0}
+                   and [x["size"] for x in RS.casing(ws)[w1["id"]]] == ["13-3/8", "9-5/8"] and SW.settings(ws)["every_min"] == 10
+                   and PS.settings(ws)["at"] == "10:45" and PS.settings(ws)["send"] is False,
+        "round_trip": p_rt["counts"]["NEW"] == 0 and p_rt["counts"]["CHANGE"] == 0 and not p_rt["errors"],
+        "refused": len(p_bad["errors"]) == 5 and any("watch_mode" in e for e in p_bad["errors"]) and any("no setting 'colour'" in e for e in p_bad["errors"])
+                   and any("13-3/8, 9-5/8 or 7" in e for e in p_bad["errors"]) and any("came from" in e for e in p_bad["errors"]),
+        "sections": secs == ["below the 13-3/8 in shoe", "below the 9-5/8 in shoe", "below the 9-5/8 in shoe"]
+                    and all(x["hole"] == "CASED" for x in W1["trips"]) and set(pad["targets"]["by_section"]) == {"below the 9-5/8 in shoe"}
+                    and next(w for w in pad["wells"] if w["display"] == "RIG-2")["block_weight_klbf"] == 42.0,
+        "invoice_xlsx": ic["status"] == "DIFFERS" and ic["totals"]["usd_in_question"] == 37666.67,
+    }
+    failed2 = [k for k, v in c2.items() if not v]
+    ok(not failed2,
+       "CQ2 the declarations sheet's new tables - the contract, the rig-time targets, a block weight, casing shoes, the watch and summary "
+       "settings - read from a workbook, written, round-tripped to nothing to do, and refused when wrong (no source, an unknown setting, a "
+       "size not in inches as written, a well not there); connections by hole section and trips inside casing; an invoice attached as a "
+       "workbook checked to the same 37,666.67" + (f" [failed: {', '.join(failed2)}]" if failed2 else ""))
+
+    # -- CQ3 the alert, the summary, the dossier ----------------------------------------------------------------------
+    now = _dtm(2026, 7, 2, 12, 0, tzinfo=_tz.utc)
+    k1 = AL.check_conditions(ws, None, now, force=True)
+    st1 = dict(AL.state(ws)["conditions"])
+    g = PS.gather(ws, _dtm(2026, 8, 2, 7, 0, tzinfo=_tz.utc), _dtm(2026, 8, 1, 0, 0, tzinfo=_tz.utc))
+    RS.write_report(ws, __version__, "tester")
+    e1 = DZ.build(ws, "tester", __version__, None)
+    with _zf.ZipFile(os.path.join(ws.reports_dir, e1["path"])) as z:
+        names = set(z.namelist())
+        rt = json.loads(z.read("results/rig_time.json"))
+        icj = json.loads(z.read(f"results/invoice_check/{wi['id']}.json"))
+    v1 = DZ.verify(os.path.join(ws.reports_dir, e1["path"]), ws)
+    for f_ in IC.files(ws, wi["id"]):                     # the corrected report and invoice replace them
+        IC.remove(ws, "tester", f_["file"])
+    ddr_ok = G["ddr"].replace("2026-07-02,14:00,16:00,2,6,Trip out", "2026-07-02,14:00,16:00,2,7,Service rig").replace("6200,7600", "6200,7400")
+    inv_ok = ("Date,Rate Type,Quantity,Unit,Unit Price,Amount\n2026-07-01,Operating day rate,1,day,30000,30000\n"
+              "2026-07-02,Operating day rate,22,hour,1250,27500\n2026-07-02,Repair rate,2,hour,0,0\n"
+              "2026-07-03,Standby rate,20,hour,1041.6667,20833.33\n2026-07-03,Operating day rate,4,hour,1250,5000\n2026-07-04,Operating day rate,1,day,30000,30000\n")
+    IC.attach(ws, "tester", wi["id"], "ddr", "ddr_ok.csv", ddr_ok.encode(), "corrected reports")
+    IC.attach(ws, "tester", wi["id"], "invoice", "inv_ok.csv", inv_ok.encode(), "corrected invoice")
+    k2 = AL.check_conditions(ws, None, now + _td(minutes=1))
+    c3 = {
+        "alert": [a_["event"] for a_ in k1 if a_["event"] == "invoice.differs"] == ["invoice.differs"] and f"invoice.differs:{wi['id']}" in st1
+                 and "37,666.67" in next(a_ for a_ in k1 if a_["event"] == "invoice.differs")["detail"],
+        "cleared": [a_["event"] for a_ in k2] == ["condition.cleared"] and k2[0]["cleared"] == "invoice.differs",
+        "summary": any(x["item"] == "the invoice check of INV-1" and x["status"] == "DIFFERS" for x in g["due"])
+                   and [x["well"] for x in g["rig_time"]] == ["RIG-1", "RIG-2"] and all(x["connections"] == 3 and x["date"] == "2026-08-01" for x in g["rig_time"])
+                   and all(x["lost_usd"] is not None for x in g["rig_time"]),
+        "dossier": v1["status"] == "VALID" and {"declarations/economics/invoices/terms.json", "declarations/economics/invoices/files.json",
+                                                 "declarations/drillstring/rig_time.json", "declarations/drillstring/casing.json"} <= names
+                   and any(n.startswith(f"declarations/economics/invoices/{wi['id']}/") and n.endswith("_inv.xlsx") for n in names)
+                   and icj["status"] == "DIFFERS" and len(rt["wells"]) == 3 and rt["targets"]["trip_in_ft_h"]["value"] == 2800,
+    }
+    failed3 = [k for k, v in c3.items() if not v]
+    ok(not failed3,
+       "CQ3 an invoice that differs raised as an alert with its dollars, and cleared when a corrected invoice replaces it; the daily summary "
+       "carrying the invoice check and yesterday's rig time well by well with its cost at the contract's rate; a signed pad dossier carrying "
+       "the contract, the invoice files as attached, the targets and the casing, and both checks as they read at signing - VALID"
+       + (f" [failed: {', '.join(failed3)}]" if failed3 else ""))
+
+    # -- CQ4 the API, the CLI, the tabs, the help ------------------------------------------------------------------------
+    Users(str(Path(wsp, "users.json"))).add("v", "viewer-pass-1", "viewer"); Users(str(Path(wsp, "users.json"))).add("op", "op-pass-123", "operator")
+    svc = Service(wsp, port=0, scheduler=False).start()
+    base_url = svc.url.rstrip("/")
+
+    def opener(name, pw_):
+        o = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(http.cookiejar.CookieJar()))
+        rq = urllib.request.Request(base_url + "/api/login", method="POST", data=json.dumps({"name": name, "password": pw_}).encode())
+        rq.add_header("Content-Type", "application/json"); rq.add_header("X-GEA-Action", "1"); o.open(rq, timeout=30)
+        return o
+
+    def call(o, method, path, body=None):
+        rq = urllib.request.Request(base_url + path, method=method, data=(json.dumps(body).encode() if body is not None else None))
+        if method == "POST":
+            rq.add_header("Content-Type", "application/json"); rq.add_header("X-GEA-Action", "1")
+        try:
+            with o.open(rq, timeout=120) as rr_:
+                return rr_.status, json.loads(rr_.read())
+        except urllib.error.HTTPError as e:
+            return e.code, json.loads(e.read())
+    try:
+        V, O = opener("v", "viewer-pass-1"), opener("op", "op-pass-123")
+        s_vc, _ = call(V, "POST", "/api/rig-time/casing", {"well_id": w2["id"], "size": "7", "shoe_md_ft": 9000, "source": "x"})
+        s_oc, ocj = call(O, "POST", "/api/rig-time/casing", {"well_id": w2["id"], "size": "7", "shoe_md_ft": 9000, "source": "tally"})
+        s_ob, _ = call(O, "POST", "/api/rig-time/casing", {"well_id": w2["id"], "size": "seven", "shoe_md_ft": 9000, "source": "tally"})
+        s_or, _ = call(O, "POST", "/api/rig-time/casing/remove", {"well_id": w2["id"], "size": "7"})
+        s_dx, dxj = call(O, "POST", "/api/declare", {"content_b64": _b64.b64encode((d / "decl.xlsx").read_bytes()).decode()})
+        s_ia, iaj = call(O, "POST", "/api/invoice-check/attach", {"well_id": wi["id"], "kind": "invoice", "filename": "inv2.xlsx", "source": "workbook",
+                                                                   "content_b64": _b64.b64encode((d / "inv.xlsx").read_bytes()).decode()})
+        s_al, alj = call(V, "GET", "/api/alerts")
+    finally:
+        svc.stop()
+    cli_c = _sp.run([sys.executable, "-m", "gea", "rig-time", "--workspace", wsp, "--well", w2["id"], "--casing", "13-3/8", "--shoe-md", "2000", "--source", "tally"],
+                    capture_output=True, text=True, cwd=str(pkg.parent))
+    cli_d = _sp.run([sys.executable, "-m", "gea", "declare", "--workspace", wsp, "--file", str(d / "decl.xlsx")], capture_output=True, text=True, cwd=str(pkg.parent))
+    page_src = (pkg / "web" / "app.html").read_text(encoding="utf-8")
+    c4 = {
+        "api": s_vc == 403 and s_oc == 200 and ocj["shoe_md_ft"] == 9000 and s_ob == 400 and s_or == 200 and s_dx == 200 and dxj["counts"]["SAME"] == 9
+               and s_ia == 200 and iaj["lines"] == 7 and s_al == 200 and "invoice.differs" in alj["events"],
+        "cli": cli_c.returncode == 0 and "13-3/8 in, shoe at 2,000 ft" in cli_c.stdout and cli_d.returncode == 0 and "already declared" in cli_d.stdout,
+        "page": all(x in page_src for x in ("rtC", "/api/rig-time/casing", ".xlsx", "content_b64: DC.b64", "'invoice.differs': 'Invoice differs'")),
+        "help": all(not H.check(t_) for t_ in ("declare", "rig-time", "invoice-check", "dossier", "alerts", "pad-summary"))
+                and "[casing]" in (pkg / "help" / "declare.md").read_text(encoding="utf-8") and "Hole sections" in (pkg / "help" / "rig-time.md").read_text(encoding="utf-8")
+                and "invoice.differs" in (pkg / "help" / "alerts.md").read_text(encoding="utf-8") and "results/" in (pkg / "help" / "dossier.md").read_text(encoding="utf-8"),
+    }
+    failed4 = [k for k, v in c4.items() if not v]
+    ok(not failed4,
+       "CQ4 on the API (only an operator declares or removes a casing shoe, a size that is not one refused; a declarations workbook and an "
+       "invoice workbook sent as they are; the new alert listed), the CLI (a casing shoe, a declarations workbook), the tabs and the help pages"
+       + (f" [failed: {', '.join(failed4)}]" if failed4 else ""))
+
+
 def main() -> int:
     print("GEA-Program - ACCEPTANCE SUITE (the product gate)")
     # a handle a job subprocess still holds at the end must not turn a finished gate into a traceback on Windows:
@@ -11010,6 +11210,7 @@ def main() -> int:
         section_cn_four(tmp)
         section_co_invoice_check(tmp)
         section_cp_rig_time(tmp)
+        section_cq_tied_together(tmp)
     if _FAILS:
         print(f"[ACCEPTANCE] {len(_FAILS)} FAILURES ({_PASS} passed):")
         for f in _FAILS:

{gea_program-0.47.0 → gea_program-0.48.0}/gea/alerts.py

@@ -13,7 +13,9 @@ worth an event, for the conditions the operator does not watch by hand:
 - `penetrator.outside` - a line of an ESP well's penetrator check is OUTSIDE its rating (the Completion tab);
 - `dossier.changed` - the newest signed dossier of a well or the pad no longer matches the workspace: a source
   record or an earlier audit line has changed since it was signed (checked once a day);
-- `condition.cleared` - one of the last three is no longer so.
+- `invoice.differs` - a well's invoice cross-check finds a day that DIFFERS (checked once a day, and when a file is
+  attached);
+- `condition.cleared` - one of the last four is no longer so.
 
 Every alert is written to `records/alerts.jsonl` whether or not any notification channel is set up - the Alarms
 tab lists them - and is handed to the notification rules, which send it where the site said (a rule per event
@@ -28,7 +30,7 @@ from datetime import datetime, timezone
 from typing import Dict, List, Optional
 
 EVENTS = ('seismic.protocol', 'seismic.event', 'seismic.watch_down', 'seismic.watch_up', 'motor.due', 'penetrator.outside',
-          'dossier.changed', 'condition.cleared')
+          'dossier.changed', 'invoice.differs', 'condition.cleared')
 CONDITIONS_EVERY_S = 900          # motors and penetrators: every 15 minutes
 DOSSIERS_EVERY_S = 86400          # dossiers: once a day
 _LOCK = threading.Lock()
@@ -173,5 +175,21 @@ def check_conditions(ws, notifier, now: Optional[datetime] = None, force: bool =
                              'dossier_id': e['dossier_id'], 'detail': '; '.join(f"{c['check']}: {c['detail']}" for c in bad)}, now, raised)
         except Exception as ex:
             last['dossiers_error'] = f'{type(ex).__name__}: {ex}'
+    if due('invoices', DOSSIERS_EVERY_S):
+        last['invoices'] = _utc(now)
+        try:
+            from . import invoice_check as IC
+            if IC.terms(ws):
+                for wid in sorted({f['well_id'] for f in IC.files(ws)}):
+                    if wid not in wells:
+                        continue
+                    r = IC.check(ws, wid)
+                    T = r.get('totals') or {}
+                    _transition(ws, notifier, st, f'invoice.differs:{wid}', r.get('status') == 'DIFFERS',
+                                {'event': 'invoice.differs', 'well_id': wid, 'well': wells[wid], 'what': f'the invoice cross-check of {wells[wid]}',
+                                 'detail': f"{T.get('differs', 0)} day(s) differ, {T.get('usd_in_question', 0):,.2f} USD in question of {T.get('billed_checked_usd', 0):,.2f} billed at day rates",
+                                 'usd_in_question': T.get('usd_in_question')}, now, raised)
+        except Exception as ex:
+            last['invoices_error'] = f'{type(ex).__name__}: {ex}'
     save_state(ws, st)
     return raised

{gea_program-0.47.0 → gea_program-0.48.0}/gea/declarations.py

@@ -67,9 +67,24 @@ TABLES: Dict[str, dict] = {
                             'gas_group', 'temp_class', 'ambient_min_f', 'ambient_max_f', 'source'),
                    'need': ('well', 'source'),
                    'about': "the ESP drive's output limits and the motor nameplate, the wellhead's hazardous-area classification (e.g. Class I, Division 1), gas group, temperature class and the site's ambient range"},
+    'contract': {'cols': ('day_start', 'report_date', 'operating', 'standby', 'repair', 'moving', 'force_majeure', 'repair_hours_per_job',
+                          'repair_hours_per_month', 'source'),
+                 'need': ('operating', 'source'),
+                 'about': "the drilling contract's terms for the invoice check: the hour the report day starts on the rig's clock (06:00), whether a report is dated by the day it starts or ends (start/end), the day rates in USD (operating, and standby, repair, moving and force majeure where the contract has them), the repair allowance in hours a job and a month, and the contract it came from"},
+    'rig_time': {'cols': ('w2w_min', 'trip_in_ft_h', 'trip_out_ft_h', 'source'), 'need': ('source',),
+                 'about': "the rig-time targets from the drilling program: weight-to-weight connection minutes and tripping speeds in and out (ft/h); blank keeps the pad's own best quarter"},
+    'block_weights': {'cols': ('well', 'block_weight_klbf'), 'need': ('well', 'block_weight_klbf'),
+                      'about': "a well's travelling block weight in klbf, where the rig-time estimate from the hook load is wrong"},
+    'casing': {'cols': ('well', 'size_in', 'shoe_md_ft', 'set', 'source'), 'need': ('well', 'size_in', 'shoe_md_ft', 'source'),
+               'about': "casing shoes: the size in inches as written (13-3/8, 9-5/8, 7), the shoe's measured depth in ft, when it was set (the recorder's clock; blank: when the hole is 50 ft past it) and the tally or report it came from"},
+    'settings': {'cols': ('setting', 'value'), 'need': ('setting', 'value'),
+                 'about': "the program's settings: watch_enabled (yes/no), watch_every_min, watch_mode (operations/always), summary_enabled (yes/no), summary_at (HH:MM UTC), summary_send (yes/no)"},
 }
-ORDER = ('wells', 'fleet', 'rates', 'items', 'afe', 'surveys', 'runs', 'components', 'penetrators', 'esp_drives', 'operations', 'catalogs')
-APPLY_ORDER = ('wells', 'fleet', 'rates', 'items', 'afe', 'surveys', 'runs', 'penetrators', 'esp_drives', 'operations', 'catalogs')
+ORDER = ('wells', 'fleet', 'rates', 'items', 'afe', 'surveys', 'runs', 'components', 'penetrators', 'esp_drives', 'operations', 'catalogs',
+         'contract', 'rig_time', 'block_weights', 'casing', 'settings')
+APPLY_ORDER = ('wells', 'fleet', 'rates', 'items', 'afe', 'surveys', 'runs', 'penetrators', 'esp_drives', 'operations', 'catalogs',
+               'contract', 'rig_time', 'block_weights', 'casing', 'settings')
+SETTINGS = ('watch_enabled', 'watch_every_min', 'watch_mode', 'summary_enabled', 'summary_at', 'summary_send')
 
 
 class SheetError(ValueError):
@@ -525,6 +540,98 @@ def plan(ws, text: str, base_dir: str = '') -> dict:
                                                                                rec['lat'], rec['lon'], rec['radius_km'], rec['source']))
         guard('catalogs', rec, b)
 
+    # the contract, rig time, casing and settings
+    from . import invoice_check as IC
+    from . import rig_state as RS
+    from . import seismic_watch as SW
+    from . import pad_summary as PS
+    for rec in tables.get('contract', [])[:1]:
+        def b(rec=rec):
+            rates = {k: rec.get(k) for k in IC.KINDS if (rec.get(k) or '').strip()}
+            new = IC.terms_record(rec.get('source', ''), (rec.get('day_start') or '06:00').strip(), (rec.get('report_date') or 'start').strip(), rates,
+                                  rec.get('repair_hours_per_job'), rec.get('repair_hours_per_month'))
+            old = IC.terms(ws)
+            keys = ('source', 'day_start', 'report_date', 'rates_usd_per_day', 'repair_hours_per_job', 'repair_hours_per_month')
+            if old and all(old.get(k) == new[k] for k in keys):
+                add('contract', rec['_row'], None, 'SAME', 'the contract terms already declared')
+                return
+            add('contract', rec['_row'], None, 'CHANGE' if old else 'NEW',
+                f"operating {new['rates_usd_per_day']['operating']:,.2f} USD/day, report day from {new['day_start']} ({new['source']})",
+                lambda actor, rec=rec, rates=rates: IC.set_terms(ws, actor, rec.get('source', ''), (rec.get('day_start') or '06:00').strip(),
+                                                                (rec.get('report_date') or 'start').strip(), rates, rec.get('repair_hours_per_job'),
+                                                                rec.get('repair_hours_per_month')))
+        guard('contract', rec, b)
+    if len(tables.get('contract', [])) > 1:
+        errs.append(f"row {tables['contract'][1]['_row']}: [contract] the contract is one row")
+    for rec in tables.get('rig_time', [])[:1]:
+        def b(rec=rec):
+            vals = {k: _f(rec.get(k), k) for k in ('w2w_min', 'trip_in_ft_h', 'trip_out_ft_h')}
+            if not any(v is not None for v in vals.values()):
+                raise SheetError('give at least one target')
+            cur = RS.settings(ws).get('targets', {})
+            want = {k: v for k, v in vals.items() if v is not None}
+            if all(cur.get(k) == v for k, v in want.items()) and RS.settings(ws).get('targets_source') == rec['source'].strip():
+                add('rig_time', rec['_row'], None, 'SAME', 'the targets already declared')
+                return
+            add('rig_time', rec['_row'], None, 'CHANGE' if cur else 'NEW', ', '.join(f"{k.replace('_', ' ')} {v:g}" for k, v in want.items()) + f" ({rec['source'].strip()})",
+                lambda actor, vals=vals, rec=rec: RS.set_settings(ws, actor, rec['source'], vals['w2w_min'], vals['trip_in_ft_h'], vals['trip_out_ft_h']))
+        guard('rig_time', rec, b)
+    bw_have = RS.settings(ws).get('block_weight_klbf', {})
+    for rec in tables.get('block_weights', []):
+        def b(rec=rec):
+            wid = _well_id(ws, rec['well'], wells)
+            v = _f(rec.get('block_weight_klbf'), 'the block weight')
+            if bw_have.get(wid) == v:
+                add('block_weights', rec['_row'], wid, 'SAME', f'{v:g} klbf already declared')
+                return
+            add('block_weights', rec['_row'], wid, 'CHANGE' if wid in bw_have else 'NEW', f'block weight {v:g} klbf',
+                lambda actor, wid=wid, v=v: RS.set_settings(ws, actor, '', None, None, None, (wid, v)))
+        guard('block_weights', rec, b)
+    cs_have = RS.casing(ws)
+    for rec in tables.get('casing', []):
+        def b(rec=rec):
+            wid = _well_id(ws, rec['well'], wells)
+            new = RS.casing_record(ws, wid, rec.get('size_in', ''), rec.get('shoe_md_ft'), rec.get('source', ''), rec.get('set') or None)
+            old = next((x for x in cs_have.get(wid, []) if x['size'] == new['size']), None)
+            if old == new:
+                add('casing', rec['_row'], wid, 'SAME', f"{new['size']} in shoe already declared")
+                return
+            add('casing', rec['_row'], wid, 'CHANGE' if old else 'NEW', f"{new['size']} in shoe at {new['shoe_md_ft']:,.0f} ft ({new['source']})",
+                lambda actor, wid=wid, rec=rec: RS.set_casing(ws, actor, wid, rec.get('size_in', ''), rec.get('shoe_md_ft'), rec.get('source', ''), rec.get('set') or None))
+        guard('casing', rec, b)
+    sw_now, ps_now = SW.settings(ws), PS.settings(ws)
+    yes = lambda v: str(v).strip().lower() in ('yes', 'y', 'true', '1', 'on')
+    for rec in tables.get('settings', []):
+        def b(rec=rec):
+            k, v = rec['setting'].strip(), rec['value'].strip()
+            if k not in SETTINGS:
+                raise SheetError(f"no setting {k!r}: one of {', '.join(SETTINGS)}")
+            if k.endswith(('_enabled', '_send')) and v.lower() not in ('yes', 'no', 'y', 'n', 'true', 'false', '1', '0', 'on', 'off'):
+                raise SheetError(f'{k} is yes or no, not {v!r}')
+            if k == 'watch_every_min':
+                x = _f(v, k)
+                cur, want, fn = sw_now['every_min'], x, (lambda actor, x=x: SW.set_settings(ws, actor, every_min=x))
+            elif k == 'watch_mode':
+                if v not in ('operations', 'always'):
+                    raise SheetError("watch_mode is 'operations' or 'always'")
+                cur, want, fn = sw_now['mode'], v, (lambda actor, v=v: SW.set_settings(ws, actor, mode=v))
+            elif k == 'watch_enabled':
+                cur, want, fn = sw_now['enabled'], yes(v), (lambda actor, v=v: SW.set_settings(ws, actor, enabled=yes(v)))
+            elif k == 'summary_enabled':
+                cur, want, fn = ps_now['enabled'], yes(v), (lambda actor, v=v: PS.set_settings(ws, actor, enabled=yes(v)))
+            elif k == 'summary_send':
+                cur, want, fn = ps_now['send'], yes(v), (lambda actor, v=v: PS.set_settings(ws, actor, send=yes(v)))
+            else:
+                if not re.fullmatch(r'([01]?\d|2[0-3]):[0-5]\d', v):
+                    raise SheetError('summary_at is HH:MM, 24-hour, UTC')
+                want = f"{int(v.split(':')[0]):02d}:{v.split(':')[1]}"
+                cur, fn = ps_now['at'], (lambda actor, w_=want: PS.set_settings(ws, actor, at=w_))
+            if cur == want:
+                add('settings', rec['_row'], None, 'SAME', f'{k} already {_g(want)}')
+                return
+            add('settings', rec['_row'], None, 'CHANGE', f'{k} {_g(cur)} -> {_g(want)}', fn)
+        guard('settings', rec, b)
+
     steps.sort(key=lambda s: (APPLY_ORDER.index(s['table']), s['row']))
     counts = {k: sum(1 for s in steps if s['action'] == k) for k in ('NEW', 'CHANGE', 'SAME')}
     return {'steps': steps, 'errors': sorted(errs, key=lambda e: int(re.match(r'row (\d+)', e).group(1)) if re.match(r'row (\d+)', e) else 0),
@@ -669,4 +776,31 @@ def template(ws) -> str:
     head('catalogs')
     for c in OS.catalogs(ws):
         w.writerow(['# ' + c['file'] + ' is attached (' + str(c['n_events']) + ' events); list it here only to replace it'])
+    from . import invoice_check as IC
+    from . import rig_state as RS
+    from . import seismic_watch as SW
+    from . import pad_summary as PS
+    head('contract')
+    t = IC.terms(ws)
+    if t:
+        r = t['rates_usd_per_day']
+        w.writerow(['', t['day_start'], t.get('report_date', 'start')] + [_g(r.get(k)) for k in IC.KINDS]
+                   + [_g(t.get('repair_hours_per_job')), _g(t.get('repair_hours_per_month')), t['source']])
+    head('rig_time')
+    rs_ = RS.settings(ws)
+    if rs_.get('targets'):
+        tg = rs_['targets']
+        w.writerow(['', _g(tg.get('w2w_min')), _g(tg.get('trip_in_ft_h')), _g(tg.get('trip_out_ft_h')), rs_.get('targets_source', '')])
+    head('block_weights')
+    for wid, v in rs_.get('block_weight_klbf', {}).items():
+        w.writerow(['', wid, _g(v)])
+    head('casing')
+    for wid, lst in RS.casing(ws).items():
+        for c in lst:
+            w.writerow(['', wid, c['size'], _g(c['shoe_md_ft']), c.get('set') or '', c['source']])
+    head('settings')
+    sw, ps = SW.settings(ws), PS.settings(ws)
+    for k, v in (('watch_enabled', _g(bool(sw['enabled']))), ('watch_every_min', _g(float(sw['every_min']))), ('watch_mode', sw['mode']),
+                 ('summary_enabled', _g(bool(ps['enabled']))), ('summary_at', ps['at']), ('summary_send', _g(bool(ps['send'])))):
+        w.writerow(['', k, v])
     return buf.getvalue()

{gea_program-0.47.0 → gea_program-0.48.0}/gea/dossier.py

@@ -5,7 +5,8 @@ A dossier is a zip:
 - `manifest.json` - the well or pad, the time, the program and its version, and every file the dossier carries
   with its size and SHA-256: the reports as they stand (with the history version each one is), the declarations
   behind them (the well's identity and position, its survey, rates and AFE, runs, penetrator, operations,
-  catalogues, the fleet), the audit log, and - by hash, or embedded when asked - the source records the reports
+  catalogues, the fleet, the contract's terms with the invoices and daily reports attached, the rig-time targets
+  and casing), the invoice cross-check and the rig time as they read at signing (`results/`), the audit log, and - by hash, or embedded when asked - the source records the reports
   were made from; the audit log's hash chain (every line chained to the one before, across its rotated segments)
   and its length; the previous dossier's id and manifest hash, so the dossiers of a workspace form one chain;
 - `signature.json` - an Ed25519 signature (RFC 8032) over the manifest's exact bytes, with the public key;
@@ -272,11 +273,15 @@ def _gather(ws, well_id: Optional[str]) -> Dict[str, List[tuple]]:
             out_list.append((os.path.join(wd, 'well.json'), f"declarations/wells/{w['id']}/well.json"))
         files_in(os.path.join(wd, 'trajectory'), f"declarations/wells/{w['id']}/trajectory")
     for rel in (('economics', 'costs.json'), ('drillstring', 'runs.json'), ('completion', 'esp.json'), ('seismicity', 'operations.json'),
-                ('seismicity', 'catalogs.json'), ('seismicity', 'settings.json'), ('config', 'fleet.json'), ('workspace.json',)):
+                ('seismicity', 'catalogs.json'), ('seismicity', 'settings.json'), ('config', 'fleet.json'), ('workspace.json',),
+                ('economics', 'invoices', 'terms.json'), ('economics', 'invoices', 'files.json'), ('drillstring', 'rig_time.json'),
+                ('drillstring', 'casing.json')):
         p = os.path.join(ws.path, *rel)
         if os.path.isfile(p):
             out_list.append((p, 'declarations/' + '/'.join(rel)))
     files_in(os.path.join(ws.path, 'seismicity', 'catalogs'), 'declarations/seismicity/catalogs')
+    for w in wells:                                        # the contractor's invoices and daily reports, as attached
+        files_in(os.path.join(ws.path, 'economics', 'invoices', w['id']), f"declarations/economics/invoices/{w['id']}")
     for p in audit_segments(ws):
         out['audit'].append((p, 'audit/' + os.path.basename(p)))
     for w in wells:
@@ -289,6 +294,32 @@ def _gather(ws, well_id: Optional[str]) -> Dict[str, List[tuple]]:
     return out
 
 
+def _results(ws, well_id: Optional[str]) -> List[tuple]:
+    """The invoice cross-check of each well with invoices or reports attached, and the rig time of each well with a
+    drilling record, as JSON computed when the dossier is signed (a check that cannot run is left out, and says why)."""
+    out = []
+    wells = [well_id] if well_id else [w['id'] for w in ws.wells()]
+    try:
+        from . import invoice_check as IC
+        if IC.terms(ws):
+            for wid in wells:
+                if IC.files(ws, wid):
+                    r = IC.check(ws, wid)
+                    out.append((f'results/invoice_check/{wid}.json', json.dumps(r, indent=1, sort_keys=True, default=str).encode('utf-8')))
+    except Exception as e:
+        out.append(('results/invoice_check/not_run.txt', f'the invoice check did not run: {type(e).__name__}: {e}\n'.encode('utf-8')))
+    try:
+        from . import rig_state as RS
+        p = RS.pad(ws)
+        ws_ = [w for w in p['wells'] if w['well_id'] in wells]
+        if ws_:
+            p = {**p, 'wells': ws_}
+            out.append(('results/rig_time.json', json.dumps(p, indent=1, sort_keys=True, default=str).encode('utf-8')))
+    except Exception as e:
+        out.append(('results/rig_time_not_run.txt', f'rig time did not run: {type(e).__name__}: {e}\n'.encode('utf-8')))
+    return out
+
+
 def _history_versions(ws) -> Dict[str, dict]:
     """The latest history entry of each report file, by its path under reports/, with that version's folder."""
     from . import report_history as RH
@@ -354,6 +385,9 @@ def build(ws, actor: str, version: str, well_id: Optional[str] = None, include_s
                 if hv and hv['sha256'] == row['sha256']:
                     row['history'] = {'dir': hv['dir'], 'written_utc': hv['utc'], 'report_id': hv['report_id']}
             files.append(row); payload.append((arc, b))
+    for arc, b in _results(ws, well_id):                   # the checks as they read at signing
+        files.append({'path': arc, 'role': 'result', 'bytes': len(b), 'sha256': hashlib.sha256(b).hexdigest(), 'workspace_path': None})
+        payload.append((arc, b))
     kh = key_history(ws)
     if kh['rotations']:                                  # the key's lineage, so a client who knows an earlier fingerprint can follow it
         b = json.dumps({'schema': 1, 'rotations': kh['rotations'],

{gea_program-0.47.0 → gea_program-0.48.0}/gea/help/alerts.md

@@ -6,7 +6,7 @@
 
 **How the watch reads the catalogue**: it runs in the service's background poller, so only while `gea serve` (the control panel) is running. An operation is under way when a declared operation (Operations & Seismicity, or the declarations sheet's `[operations]`) has started and has not ended, at a well with a surface position. A poll reads the USGS real-time summary feed of the past day, all magnitudes, updated every minute - the feed USGS asks automated programs to use - and keeps the earthquakes inside the circle around the operating wells (out to the screening distance, 20 km by default, beyond the farthest). On the first poll, or after a gap of a day or more, it asks the FDSN event service for the whole gap instead (from the operation's start, at most 30 days back). USGS revises magnitudes and locations; the newest version wins, and a revision that reaches a higher protocol level is a new alert.
 
-**The alerts**: `seismic.protocol` (a protocol level reached, once per level), `seismic.event` (any earthquake within the association radius during or after an operation, once), `seismic.watch_down` (no answer from the catalogue for three intervals while an operation runs) and `seismic.watch_up`; `motor.due` (checked every 15 minutes), `penetrator.outside` (every 15 minutes), `dossier.changed` (the newest dossier of each well and of the pad, re-checked daily), and `condition.cleared` when one of the last three ends.
+**The alerts**: `seismic.protocol` (a protocol level reached, once per level), `seismic.event` (any earthquake within the association radius during or after an operation, once), `seismic.watch_down` (no answer from the catalogue for three intervals while an operation runs) and `seismic.watch_up`; `motor.due` (checked every 15 minutes), `penetrator.outside` (every 15 minutes), `dossier.changed` (the newest dossier of each well and of the pad, re-checked daily), `invoice.differs` (a well's invoice cross-check finds a day that differs; checked daily and when a file is attached), and `condition.cleared` when one of the last four ends.
 
 **Getting them to a person**: every alert is written to `records/alerts.jsonl` and shown on the Alarms tab. To have it sent, set up notifications (Administration, Notifications: load the example, edit, validate and commit) with an email (SMTP) or webhook channel and a rule naming the alert, for example `{"event": "seismic.protocol", "channels": ["mail"]}`; the password stays in an environment variable named by `password_env`. A text message: most carriers deliver email sent to the phone's email-to-text address, or a webhook to a messaging service. The `quiet_s` window keeps one alert from repeating.