gea-program 0.43.0__tar.gz → 0.44.0__tar.gz

{gea_program-0.43.0 → gea_program-0.44.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gea-program
-Version: 0.43.0
+Version: 0.44.0
 Summary: GEA-Program: downhole gauge monitoring - canonical measurement records, quality rules, drift evaluation with a re-fit change log, well-test validation, alarm management, model cards, store-and-forward, monthly SLA measurement, FAT/SAT protocols and a dashboard, generated from live objects in the client's reporting outline.
 Author: Daniel T. Murphy
 License: MPL-2.0
@@ -648,6 +648,20 @@ against it; `gea seedlink --config ... --hello` prints what a server says of
 itself; `gea help seedlink`. A TexNet station is served by EarthScope at
 `rtserve.iris.washington.edu:18000`, network `TX`.
 
+### The earthquake watch and the alerts
+
+While a well operates, the program reads the USGS earthquake catalogue
+every few minutes from the site's machine - the real-time feed USGS asks
+automated programs to use, and the catalogue service for any gap - and
+raises an alert when an earthquake near the well reaches the chosen
+regulator's protocol level; a revision to a higher level is a new alert,
+and a watch that cannot reach the catalogue says so. The same alerts cover
+what nobody watches by hand: a component's service interval used up, a
+penetrator check outside its rating, a signed dossier the workspace no
+longer matches. Every alert is listed on the Alarms tab and sent through
+the notification rules (email or a webhook). `gea seismicity --watch-once`,
+`gea alerts`, `gea help alerts`.
+
 ### Report history and the signed dossier
 
 Every client report a workspace writes is kept, version by version, under

{gea_program-0.43.0 → gea_program-0.44.0}/README.md

@@ -609,6 +609,20 @@ against it; `gea seedlink --config ... --hello` prints what a server says of
 itself; `gea help seedlink`. A TexNet station is served by EarthScope at
 `rtserve.iris.washington.edu:18000`, network `TX`.
 
+### The earthquake watch and the alerts
+
+While a well operates, the program reads the USGS earthquake catalogue
+every few minutes from the site's machine - the real-time feed USGS asks
+automated programs to use, and the catalogue service for any gap - and
+raises an alert when an earthquake near the well reaches the chosen
+regulator's protocol level; a revision to a higher level is a new alert,
+and a watch that cannot reach the catalogue says so. The same alerts cover
+what nobody watches by hand: a component's service interval used up, a
+penetrator check outside its rating, a signed dossier the workspace no
+longer matches. Every alert is listed on the Alarms tab and sent through
+the notification rules (email or a webhook). `gea seismicity --watch-once`,
+`gea alerts`, `gea help alerts`.
+
 ### Report history and the signed dossier
 
 Every client report a workspace writes is kept, version by version, under

{gea_program-0.43.0 → gea_program-0.44.0}/gea/__init__.py

@@ -123,7 +123,7 @@ from .profile_catalog import (
     read_pangaea_txt,
 )
 
-__version__ = "0.43.0"
+__version__ = "0.44.0"
 __all__ = [
     "aging_rate", "rate_psi_yr", "accuracy_psi",
     "Sensor", "SimulatorConfig", "DownholeEngine",

{gea_program-0.43.0 → gea_program-0.44.0}/gea/__main__.py

@@ -563,9 +563,19 @@ def main(argv=None) -> int:
     p_os.add_argument("--kind", type=str, default=None, choices=("drilling", "completion", "disposal", "injection", "other"))
     p_os.add_argument("--remove-op", type=str, default=None)
     p_os.add_argument("--remove-catalog", type=str, default=None)
+    p_os.add_argument("--watch", type=str, default=None, choices=("on", "off"), help="turn the live earthquake watch on or off")
+    p_os.add_argument("--watch-every", type=float, default=None, help="the watch's interval in minutes (default 5)")
+    p_os.add_argument("--watch-mode", type=str, default=None, choices=("operations", "always"), help="poll only while an operation is under way (default), or always")
+    p_os.add_argument("--watch-once", action="store_true", help="poll the USGS catalogue now, from this machine, and say what came back")
     p_os.add_argument("--actor", type=str, default="cli")
     p_os.add_argument("--json", action="store_true")
 
+    p_al = sub.add_parser("alerts", help="the program's alerts: earthquakes near operating wells and the protocol levels they reach, the earthquake watch going quiet, components due for service, penetrator checks outside their ratings, dossiers the workspace no longer matches")
+    p_al.add_argument("--workspace", type=str, required=True)
+    p_al.add_argument("--check", action="store_true", help="check the motors, penetrators and dossiers now, sending what began or cleared through the notification rules")
+    p_al.add_argument("-n", type=int, default=30)
+    p_al.add_argument("--json", action="store_true")
+
     p_dz = sub.add_parser("dossier", help="the signed dossier: a well's (or the pad's) reports, declarations, audit log and source hashes in one zip, signed with the site's Ed25519 key, with a checker anyone can run without this program")
     p_dz.add_argument("--workspace", type=str, default=None)
     p_dz.add_argument("--well", type=str, default=None, help="a well; without it, the pad (every well and the site reports)")
@@ -2225,6 +2235,26 @@ def main(argv=None) -> int:
                 return 0
             if a.remove_op:
                 _OS.remove_operation(ws, a.actor, a.remove_op); print(f"removed {a.remove_op}"); return 0
+            if a.watch or a.watch_every is not None or a.watch_mode or a.watch_once:
+                from . import seismic_watch as _SW
+                if ws is None:
+                    raise SystemExit("seismicity: the watch needs --workspace")
+                if a.watch or a.watch_every is not None or a.watch_mode:
+                    _SW.set_settings(ws, a.actor, None if a.watch is None else a.watch == "on", a.watch_mode, a.watch_every)
+                if a.watch_once:
+                    from .notify import Notifier
+                    n_ = Notifier(ws.path); n_.reload()
+                    r_ = _SW.poll(ws, n_, force=True)
+                    if r_.get("ok"):
+                        print(f"reached the catalogue ({r_['how']}): {r_['in_circle']} earthquake(s) in the watch circle, {r_['new']} new; {len(r_['alerts'])} alert(s)")
+                        for al_ in r_["alerts"]:
+                            print(f"  {al_['event']}: {al_.get('detail', '')}")
+                    else:
+                        print(f"did not reach the catalogue: {r_.get('error') or r_.get('why')}")
+                st_ = _SW.status(ws)
+                print(f"watch: {st_['state']} - every {st_['settings']['every_min']:g} min, {st_['settings']['mode']}; last good poll {st_['last_ok_utc'] or 'never'}"
+                      + (f"; last error {st_['last_error']}" if st_.get('last_error') else ""))
+                return 0 if (not a.watch_once or r_.get("ok")) else 1
             if a.remove_catalog:
                 _OS.remove_catalog(ws, a.actor, a.remove_catalog); print(f"removed {a.remove_catalog}"); return 0
             over = {k: v for k, v in (("protocol", a.protocol), ("radius_km", a.radius_km), ("screen_km", a.screen_km), ("post_days", a.post_days), ("mc", a.mc)) if v is not None}
@@ -2306,6 +2336,30 @@ def main(argv=None) -> int:
             for v in f_["versions"][: (50 if a.family else 3)]:
                 print(f"      {v['utc']}  {v.get('report_id') or ''}  reports/{v['dir']}")
         return 0
+    elif a.cmd == "alerts":
+        import json as _json
+        from . import alerts as _AL
+        from .cli import _safe_stdout
+        from .notify import Notifier
+        from .workspace import Workspace, WorkspaceError
+        _safe_stdout()
+        try:
+            ws = Workspace(a.workspace)
+        except WorkspaceError as e:
+            raise SystemExit(f"alerts: {e}")
+        if a.check:
+            n_ = Notifier(ws.path); n_.reload()
+            r_ = _AL.check_conditions(ws, n_, force=True)
+            print(f"checked: {len(r_)} alert(s) raised or cleared")
+        rows = _AL.recent(ws, a.n)
+        if a.json:
+            print(_json.dumps(rows, default=str)); return 0
+        for r in rows:
+            sent = ", ".join(f"{d['channel']}{'' if d['ok'] else ' (failed)'}" for d in r.get("delivered") or []) or "no channel set up"
+            print(f"{r['utc']}  {r['event']:<20} {r.get('well') or '':<16} {r.get('detail', '')}  [{sent}]")
+        if not rows:
+            print("no alerts yet")
+        return 0
     elif a.cmd == "simulation":
         import json as _json
         from . import simulation as _SIM

{gea_program-0.43.0 → gea_program-0.44.0}/gea/acceptance_tests.py

@@ -10116,6 +10116,221 @@ def section_cl_history_and_dossier(tmp: str) -> None:
        + (f" [failed: {', '.join(failed3)}]" if failed3 else ""))
 
 
+def section_cm_watch_and_alerts(tmp: str) -> None:
+    """Section CM - the live earthquake watch and the program's alerts: the watch idle without an operation, its first poll
+    a catalogue query over the gap and the next ones the real-time feed, the circle kept, a revision raising a higher
+    level once, the watch going quiet and coming back - all through a stand-in catalogue served over HTTP; an alert
+    delivered through a webhook rule; a service interval used up, a penetrator outside its rating and a dossier the
+    workspace no longer matches each raised once and cleared; the API, the CLI, the tabs and the help page."""
+    import http.cookiejar
+    import http.server
+    import subprocess as _sp
+    import threading as _th
+    import urllib.request
+    import urllib.error
+    from datetime import datetime, timedelta, timezone
+    from . import __version__
+    from . import alerts as AL
+    from . import dossier as DZ
+    from . import drill_string as DS
+    from . import helplib as H
+    from . import op_seismicity as OS
+    from . import penetrator as PN
+    from . import seismic_watch as SW
+    from .notify import Notifier
+    from .workspace import Workspace
+    from .service import Service, Users
+    pkg = Path(__file__).parent
+    d = Path(tmp, "cm"); d.mkdir()
+    now = datetime.now(timezone.utc).replace(microsecond=0)
+
+    def feat(eid, t, lat, lon, m, upd=1, typ="earthquake"):
+        return {"type": "Feature", "id": eid, "properties": {"mag": m, "magType": "ml", "time": int(t.timestamp() * 1000), "updated": upd, "place": "SYNTHETIC",
+                                                              "status": "automatic", "net": "tx", "type": typ},
+                "geometry": {"type": "Point", "coordinates": [lon, lat, 5.0]}}
+    served = {"query": [], "feed": [], "hook": [], "fail": False}
+    FEEDS = {"feed": [feat("tx1", now - timedelta(hours=20), 32.01, -102.0, 1.7), feat("tx2", now - timedelta(minutes=3), 32.005, -102.01, 2.4),
+                      feat("qb", now - timedelta(hours=2), 32.0, -102.02, 1.9, typ="quarry blast")],
+             "query": [feat("tx1", now - timedelta(hours=20), 32.01, -102.0, 1.7), feat("far", now - timedelta(hours=5), 33.5, -102.0, 3.5)]}
+
+    class Stand(http.server.BaseHTTPRequestHandler):
+        def log_message(self, *a):
+            pass
+
+        def do_GET(self):
+            if served["fail"]:
+                self.send_response(503); self.end_headers(); return
+            kind = "query" if "/fdsnws/" in self.path else "feed"
+            served[kind].append(self.path)
+            b = json.dumps({"type": "FeatureCollection", "features": FEEDS[kind]}).encode()
+            self.send_response(200); self.send_header("Content-Type", "application/json"); self.send_header("Content-Length", str(len(b))); self.end_headers()
+            self.wfile.write(b)
+
+        def do_POST(self):
+            n = int(self.headers.get("Content-Length") or 0)
+            served["hook"].append(json.loads(self.rfile.read(n) or b"{}"))
+            self.send_response(200); self.end_headers()
+    srv = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Stand)
+    _th.Thread(target=srv.serve_forever, daemon=True).start()
+    base = f"http://127.0.0.1:{srv.server_address[1]}"
+
+    wsp = str(d / "ws"); ws = Workspace.create(wsp, "Pad CM", actor="tester")
+    w = ws.add_well_file(str(pkg / "sample_well_profile.csv"), display="FRAC-1", actor="tester")
+    Path(wsp, "seismicity").mkdir(exist_ok=True)
+    Path(wsp, "seismicity", "watch.json").write_text(json.dumps({"enabled": True, "mode": "operations", "every_min": 5,
+                                                                  "feed_url": base + "/earthquakes/feed/v1.0/summary/all_day.geojson",
+                                                                  "query_url": base + "/fdsnws/event/1/query"}), encoding="utf-8")
+    Path(wsp, "config", "notifications.json").write_text(json.dumps({"channels": [{"name": "hook", "kind": "webhook", "url": base + "/hook"}],
+                                                                      "rules": [{"event": "seismic.protocol", "channels": ["hook"]}, {"event": "motor.due", "channels": ["hook"]}],
+                                                                      "quiet_s": 0}), encoding="utf-8")
+    nt = Notifier(wsp); nt.reload()
+
+    # -- CM1 the watch -----------------------------------------------------------------------------------------------
+    r_nopos = SW.poll(ws, nt, now)
+    ws.set_well(w["id"], surface={"lat": 32.0, "lon": -102.0, "datum": "NAD83"}, actor="tester")
+    r_idle = SW.poll(ws, nt, now)
+    st_idle = SW.status(ws, now)["state"]
+    OS.save_operation(ws, "tester", w["id"], "completion", (now - timedelta(days=2)).strftime("%Y-%m-%d"), None, "stage log")
+    OS.set_settings(ws, "tester", protocol_id="bc-ksmma-2021")
+    r1 = SW.poll(ws, nt, now)
+    r_soon = SW.poll(ws, nt, now + timedelta(minutes=2))
+    r2 = SW.poll(ws, nt, now + timedelta(minutes=5))
+    n_feed_r2 = len(served["feed"])
+    FEEDS["feed"][1] = feat("tx2", now - timedelta(minutes=3), 32.005, -102.01, 3.1, upd=2)
+    r3 = SW.poll(ws, nt, now + timedelta(minutes=10))
+    r3b = SW.poll(ws, nt, now + timedelta(minutes=15))
+    served["fail"] = True
+    fails = [SW.poll(ws, nt, now + timedelta(minutes=m)) for m in (20, 25, 30, 35)]
+    st_down = SW.status(ws, now + timedelta(minutes=36))["state"]
+    served["fail"] = False
+    r_back = SW.poll(ws, nt, now + timedelta(minutes=40))
+    q = served["query"][0] if served["query"] else ""
+    pg = OS.page(ws)
+    evs = {e["event_id"]: e for e in pg["wells"][0]["events"]}
+    al = AL.recent(ws, 100)
+    c1 = {
+        "idle": r_nopos["polled"] is False and r_idle["polled"] is False and "no operation under way" in r_idle["why"] and st_idle == "IDLE",
+        "query_first": r1["ok"] and r1["how"] == "query" and len(served["query"]) == 1 and "format=geojson" in q and "maxradiuskm=20.0" in q
+                       and (now - timedelta(days=2)).strftime("%Y-%m-%d") in q and r1["in_circle"] == 1,
+        "feed_next": r_soon["polled"] is False and r_soon["why"] == "not due" and r2["how"] == "feed" and r2["new"] == 2 and n_feed_r2 == 1,
+        "alerts": [a["event"] for a in r1["alerts"]] == ["seismic.event", "seismic.protocol"] and [(a["event"], a.get("level_m")) for a in r2["alerts"]] == [("seismic.event", None), ("seismic.protocol", 2.0)]
+                  and [(a["event"], a.get("level_m")) for a in r3["alerts"]] == [("seismic.protocol", 3.0)] and r3b["alerts"] == [],
+        "down_up": [len(f["alerts"]) for f in fails] == [0, 0, 1, 0] and fails[2]["alerts"][0]["event"] == "seismic.watch_down" and st_down == "NOT REACHING THE CATALOGUE"
+                   and [a["event"] for a in r_back["alerts"]] == ["seismic.watch_up"],
+        "catalogue": "live_usgs.txt" in [c["file"] for c in OS.catalogs(ws)] and set(evs) == {"tx1", "tx2"} and evs["tx2"]["magnitude"] == 3.1
+                     and evs["tx2"]["protocol"]["highest"]["m"] == 3.0 and "qb" not in evs and pg["watch"]["state"] == "WATCHING",
+        "delivered": len(served["hook"]) == 3 and all(h_.get("event") == "seismic.protocol" for h_ in served["hook"]) and "subject" in served["hook"][0]
+                     and any(a["event"] == "seismic.protocol" and a["delivered"] and a["delivered"][0]["ok"] for a in al)
+                     and any(a["event"] == "seismic.event" and not a["delivered"] for a in al),
+    }
+    SW.set_settings(ws, "tester", enabled=False)
+    c1["off"] = SW.poll(ws, nt, now + timedelta(hours=1), force=True)["why"] == "the watch is off" and SW.status(ws)["state"] == "OFF"
+    SW.set_settings(ws, "tester", enabled=True, every_min=2)
+    try:
+        SW.set_settings(ws, "tester", every_min=0.1); c1["bad_interval"] = False
+    except ValueError:
+        c1["bad_interval"] = True
+    failed1 = [k for k, v in c1.items() if not v]
+    ok(not failed1,
+       "CM1 the live earthquake watch, against a stand-in catalogue served over HTTP: idle with no well placed or no operation under way; "
+       "its first poll a catalogue query over the circle from the operation's start, the next ones the real-time feed, not before the "
+       "interval; an earthquake 170 km away and a quarry blast left out; every earthquake near the operation an alert once, each protocol "
+       "level once, a revision to M 3.1 the higher level once; three intervals without an answer an alert, and the answer again another; the "
+       "live catalogue read by the tab; the protocol alerts delivered through a webhook rule; off when turned off"
+       + (f" [failed: {', '.join(failed1)}]" if failed1 else ""))
+
+    # -- CM2 the conditions ------------------------------------------------------------------------------------------
+    syn = DS.synthetic_motor_record(str(d))
+    we = ws.add_well_file(syn["path"], display="SYNTHETIC EDR well", actor="tester")
+    comps = [{"class_id": "pdc_bit", "maker": "Acme", "model": "X616", "serial": "B1"},
+             {"class_id": "pdm_motor", "motor_id": "msd-6-3-4-78-6.4", "serial": "M100", "service_limit": 10, "service_basis": "circulating_hours", "service_source": "rental contract"}]
+    run = DS.save_run(ws, "tester", we["id"], "Run 1", comps)
+    k1 = AL.check_conditions(ws, nt, now, force=True)
+    k2 = AL.check_conditions(ws, nt, now, force=True)
+    comps[1]["service_limit"] = 100
+    DS.save_run(ws, "tester", we["id"], "Run 1", comps, run_id=run["id"])
+    k3 = AL.check_conditions(ws, nt, now, force=True)
+    PN.save_install(ws, "tester", w["id"], "genco-gps-5", void_test_psi=6000, void_held="yes")
+    k4 = AL.check_conditions(ws, nt, now, force=True)
+    PN.save_install(ws, "tester", w["id"], "genco-gps-5", void_test_psi=4000, void_held="yes")
+    k5 = AL.check_conditions(ws, nt, now, force=True)
+    ws.refresh_dashboard(actor="tester")
+    DZ.build(ws, "tester", __version__, we["id"])
+    k6 = AL.check_conditions(ws, nt, now + timedelta(minutes=1))           # the daily dossier check ran with k1: not due again yet
+    src = next(Path(wsp, "wells", we["id"], "source").iterdir())
+    raw = src.read_bytes(); src.write_bytes(raw + b"\n")
+    k7 = AL.check_conditions(ws, nt, now + timedelta(days=1, minutes=1))
+    src.write_bytes(raw)
+    k8 = AL.check_conditions(ws, nt, now + timedelta(days=2, minutes=2))
+    ev = lambda ks: [a["event"] for a in ks]
+    c2 = {
+        "motor": ev(k1) == ["motor.due"] and "M100" in k1[0]["what"] and "rental contract" in k1[0]["detail"] and ev(k2) == []
+                 and ev(k3) == ["condition.cleared"] and k3[0]["cleared"] == "motor.due" and any(h_.get("event") == "motor.due" for h_ in served["hook"]),
+        "penetrator": ev(k4) == ["penetrator.outside"] and "void test" in k4[0]["detail"] and ev(k5) == ["condition.cleared"],
+        "dossier": ev(k6) == [] and ev(k7) == ["dossier.changed"] and "source records" in k7[0]["detail"] and ev(k8) == ["condition.cleared"],
+        "state": AL.state(ws)["conditions"] == {},
+    }
+    failed2 = [k for k, v in c2.items() if not v]
+    ok(not failed2,
+       "CM2 the conditions: a motor past its declared service interval an alert once, delivered by its rule, and cleared when the interval "
+       "is raised; a penetrator's void test above its rating an alert and cleared when corrected; a signed dossier whose source record "
+       "changes an alert at the next daily check, and cleared when the record is restored"
+       + (f" [failed: {', '.join(failed2)}]" if failed2 else ""))
+
+    # -- CM3 the API, the CLI, the tabs ------------------------------------------------------------------------------
+    Users(str(Path(wsp, "users.json"))).add("v", "viewer-pass-1", "viewer"); Users(str(Path(wsp, "users.json"))).add("op", "op-pass-123", "operator")
+    svc = Service(wsp, port=0, scheduler=False).start()
+    base_url = svc.url.rstrip("/")
+
+    def opener(name, pw_):
+        o = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(http.cookiejar.CookieJar()))
+        rq = urllib.request.Request(base_url + "/api/login", method="POST", data=json.dumps({"name": name, "password": pw_}).encode())
+        rq.add_header("Content-Type", "application/json"); rq.add_header("X-GEA-Action", "1"); o.open(rq, timeout=30)
+        return o
+
+    def call(o, method, path, body=None):
+        rq = urllib.request.Request(base_url + path, method=method, data=(json.dumps(body).encode() if body is not None else None))
+        if method == "POST":
+            rq.add_header("Content-Type", "application/json"); rq.add_header("X-GEA-Action", "1")
+        try:
+            with o.open(rq, timeout=120) as rr_:
+                return rr_.status, json.loads(rr_.read())
+        except urllib.error.HTTPError as e:
+            return e.code, json.loads(e.read())
+    try:
+        V, O = opener("v", "viewer-pass-1"), opener("op", "op-pass-123")
+        s_a, aj = call(V, "GET", "/api/alerts")
+        s_p, pj = call(V, "GET", "/api/opseis")
+        s_vw, _ = call(V, "POST", "/api/opseis/watch", {"every_min": 3})
+        s_ow, oj = call(O, "POST", "/api/opseis/watch", {"every_min": 3, "mode": "always", "enabled": True})
+        s_bw, _ = call(O, "POST", "/api/opseis/watch", {"mode": "sometimes"})
+        s_pp, ppj = call(O, "POST", "/api/opseis/watch/poll", {})
+    finally:
+        svc.stop()
+    cli_once = _sp.run([sys.executable, "-m", "gea", "seismicity", "--workspace", wsp, "--watch-once"], capture_output=True, text=True, cwd=str(pkg.parent))
+    cli_off = _sp.run([sys.executable, "-m", "gea", "seismicity", "--workspace", wsp, "--watch", "off"], capture_output=True, text=True, cwd=str(pkg.parent))
+    cli_al = _sp.run([sys.executable, "-m", "gea", "alerts", "--workspace", wsp, "-n", "50"], capture_output=True, text=True, cwd=str(pkg.parent))
+    srv.shutdown()
+    page_src = (pkg / "web" / "app.html").read_text(encoding="utf-8")
+    service_src = (pkg / "service.py").read_text(encoding="utf-8")
+    c3 = {
+        "api": s_a == 200 and aj["notifications_configured"] and len(aj["alerts"]) >= 10 and "seismic.protocol" in aj["events"] and s_p == 200 and pj["watch"]["settings"]["every_min"] == 2
+               and s_vw == 403 and s_ow == 200 and oj["settings"]["mode"] == "always" and oj["settings"]["every_min"] == 3 and s_bw == 400
+               and s_pp == 200 and ppj["result"]["ok"] and ppj["status"]["state"] == "WATCHING",
+        "cli": cli_once.returncode == 0 and "reached the catalogue" in cli_once.stdout and "watch: WATCHING" in cli_once.stdout
+               and cli_off.returncode == 0 and "watch: OFF" in cli_off.stdout and cli_al.returncode == 0 and "seismic.protocol" in cli_al.stdout and "hook" in cli_al.stdout,
+        "poller": "_SW.tick(self)" in service_src,
+        "page": all(x in page_src for x in ("swCard", "Live earthquake watch", "/api/opseis/watch/poll", "alCard", "Program alerts", "/api/alerts")),
+        "help": any(t[0] == "alerts" for t in H.INDEX) and not H.check("alerts") and "seismic.protocol" in (pkg / "help" / "notifications.md").read_text(encoding="utf-8"),
+    }
+    failed3 = [k for k, v in c3.items() if not v]
+    ok(not failed3,
+       "CM3 the alerts on the API (anyone signed in reads them and the watch; only an operator sets the watch or polls it; a mode that is "
+       "not one refused), the CLI (a poll now through the catalogue, the watch off, the alerts with where each was sent), the service's "
+       "poller running the watch, the watch card and the Program alerts card, and the help page"
+       + (f" [failed: {', '.join(failed3)}]" if failed3 else ""))
+
+
 def main() -> int:
     print("GEA-Program - ACCEPTANCE SUITE (the product gate)")
     # a handle a job subprocess still holds at the end must not turn a finished gate into a traceback on Windows:
@@ -10209,6 +10424,7 @@ def main() -> int:
         section_cj_declarations(tmp)
         section_ck_operations_seismicity(tmp)
         section_cl_history_and_dossier(tmp)
+        section_cm_watch_and_alerts(tmp)
     if _FAILS:
         print(f"[ACCEPTANCE] {len(_FAILS)} FAILURES ({_PASS} passed):")
         for f in _FAILS:

gea_program-0.44.0/gea/alerts.py

@@ -0,0 +1,177 @@
+"""Program alerts: the conditions this program watches for, raised once when they begin and once when they clear.
+
+The notification rules (`notify`) deliver an event to email or a webhook. This module decides when a condition is
+worth an event, for the conditions the operator does not watch by hand:
+
+- `seismic.protocol` - an earthquake near a well during (or in the association window after) one of its
+  operations reaches a level of the chosen regulator's protocol; raised again only if a later revision of its
+  magnitude reaches a higher level (from the live earthquake watch, `seismic_watch`);
+- `seismic.event` - any new earthquake within the association radius during or after an operation;
+- `seismic.watch_down` / `seismic.watch_up` - the earthquake watch has not reached the catalogue for three of its
+  intervals while an operation runs, and when it does again;
+- `motor.due` - a drill-string component's declared service interval is used up (the Drill String tab's DUE);
+- `penetrator.outside` - a line of an ESP well's penetrator check is OUTSIDE its rating (the Completion tab);
+- `dossier.changed` - the newest signed dossier of a well or the pad no longer matches the workspace: a source
+  record or an earlier audit line has changed since it was signed (checked once a day);
+- `condition.cleared` - one of the last three is no longer so.
+
+Every alert is written to `records/alerts.jsonl` whether or not any notification channel is set up - the Alarms
+tab lists them - and is handed to the notification rules, which send it where the site said (a rule per event
+name in `notifications.json`). The state that keeps an alert from repeating is `records/alerts_state.json`.
+"""
+from __future__ import annotations
+
+import json
+import os
+import threading
+from datetime import datetime, timezone
+from typing import Dict, List, Optional
+
+EVENTS = ('seismic.protocol', 'seismic.event', 'seismic.watch_down', 'seismic.watch_up', 'motor.due', 'penetrator.outside',
+          'dossier.changed', 'condition.cleared')
+CONDITIONS_EVERY_S = 900          # motors and penetrators: every 15 minutes
+DOSSIERS_EVERY_S = 86400          # dossiers: once a day
+_LOCK = threading.Lock()
+
+
+def _utc(now: Optional[datetime] = None) -> str:
+    return (now or datetime.now(timezone.utc)).strftime('%Y-%m-%dT%H:%M:%SZ')
+
+
+def _path(ws, name: str) -> str:
+    return os.path.join(ws.path, 'records', name)
+
+
+def state(ws) -> dict:
+    p = _path(ws, 'alerts_state.json')
+    if not os.path.isfile(p):
+        return {'conditions': {}, 'last': {}}
+    try:
+        with open(p, encoding='utf-8') as f:
+            return json.load(f)
+    except (OSError, ValueError):
+        return {'conditions': {}, 'last': {}}
+
+
+def save_state(ws, s: dict) -> None:
+    p = _path(ws, 'alerts_state.json')
+    os.makedirs(os.path.dirname(p), exist_ok=True)
+    with open(p + '.tmp', 'w', encoding='utf-8') as f:
+        json.dump(s, f, indent=1)
+    os.replace(p + '.tmp', p)
+
+
+def raise_alert(ws, notifier, ev: dict, now: Optional[datetime] = None) -> dict:
+    """Record an alert and hand it to the notification rules. Returns the record with the deliveries."""
+    rec = {'utc': _utc(now), **ev}
+    deliveries: List[dict] = []
+    if notifier is not None:
+        try:
+            deliveries = notifier.emit(dict(ev))
+        except Exception as e:                       # a channel never stops the record
+            deliveries = [{'ok': False, 'error': f'{type(e).__name__}: {e}'}]
+    rec['delivered'] = [{'channel': d.get('channel'), 'ok': bool(d.get('ok'))} for d in deliveries]
+    p = _path(ws, 'alerts.jsonl')
+    os.makedirs(os.path.dirname(p), exist_ok=True)
+    with _LOCK:
+        with open(p, 'a', encoding='utf-8') as f:
+            f.write(json.dumps(rec, default=str, sort_keys=True) + '\n')
+    return rec
+
+
+def recent(ws, n: int = 100) -> List[dict]:
+    p = _path(ws, 'alerts.jsonl')
+    if not os.path.isfile(p):
+        return []
+    rows = []
+    with open(p, encoding='utf-8') as f:
+        for line in f:
+            try:
+                rows.append(json.loads(line))
+            except ValueError:
+                pass
+    return list(reversed(rows[-n:]))
+
+
+def _transition(ws, notifier, st: dict, key: str, bad: bool, ev: dict, now: datetime, raised: List[dict]) -> None:
+    was = st['conditions'].get(key)
+    if bad and not was:
+        st['conditions'][key] = {'since': _utc(now), 'event': ev['event']}
+        raised.append(raise_alert(ws, notifier, {**ev, 'key': key}, now))
+    elif not bad and was:
+        st['conditions'].pop(key, None)
+        raised.append(raise_alert(ws, notifier, {'event': 'condition.cleared', 'key': key + ':cleared', 'cleared': was['event'],
+                                                 'since': was['since'], 'well_id': ev.get('well_id'), 'well': ev.get('well'),
+                                                 'detail': f"no longer so: {ev.get('what', key)}"}, now))
+
+
+def check_conditions(ws, notifier, now: Optional[datetime] = None, force: bool = False) -> List[dict]:
+    """Motors, penetrators (every 15 minutes) and dossiers (daily): raise what began, clear what ended."""
+    from . import drill_string as DS
+    from . import penetrator as PN
+    now = now or datetime.now(timezone.utc)
+    st = state(ws)
+    raised: List[dict] = []
+    last = st.setdefault('last', {})
+
+    def due(name, every):
+        t = last.get(name)
+        if force or not t:
+            return True
+        return (now - datetime.strptime(t, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone.utc)).total_seconds() >= every
+    wells = {w['id']: (w.get('display') or w['id']) for w in ws.wells()}
+    if due('conditions', CONDITIONS_EVERY_S):
+        last['conditions'] = _utc(now)
+        seen = set()
+        if DS.runs(ws):
+            try:
+                for e in DS.tracking(ws, build=True):
+                    life = e.get('life')
+                    if not life:
+                        continue
+                    key = f"motor.due:{e['key']}"
+                    seen.add(key)
+                    wid = e['runs'][-1]['well_id'] if e['runs'] else None
+                    what = f"{e['class_name']} {e['maker']} {e['model']} {e['serial']}".strip()
+                    _transition(ws, notifier, st, key, life['status'] == 'DUE',
+                                {'event': 'motor.due', 'well_id': wid, 'well': wells.get(wid, wid), 'what': what,
+                                 'detail': f"{life['used']:g} of {life['limit']:g} {life['basis'].replace('_', ' ')} used ({life['used_pct']:g} %); interval: {life['basis']}, {(e['service'] or {}).get('source', '')}"},
+                                now, raised)
+            except Exception as ex:
+                last['conditions_error'] = f'{type(ex).__name__}: {ex}'
+        for wid in list(PN.records(ws)):
+            if wid not in wells:
+                continue
+            try:
+                c = PN.check(ws, wid)
+            except Exception:
+                continue
+            bad = [r for r in c['rows'] if r['status'] == 'OUTSIDE']
+            key = f'penetrator.outside:{wid}'
+            seen.add(key)
+            _transition(ws, notifier, st, key, bool(bad),
+                        {'event': 'penetrator.outside', 'well_id': wid, 'well': wells[wid], 'what': f"the penetrator check of {wells[wid]}",
+                         'detail': '; '.join(f"{r['item']}: {r['why']}" for r in bad)}, now, raised)
+        for key in [k for k in st['conditions'] if k.split(':')[0] in ('motor.due', 'penetrator.outside') and k not in seen]:
+            st['conditions'].pop(key, None)                  # the run or the record was removed: nothing left to clear
+    if due('dossiers', DOSSIERS_EVERY_S):
+        last['dossiers'] = _utc(now)
+        try:
+            from . import dossier as DZ
+            newest: Dict[str, dict] = {}
+            for e in DZ.ledger(ws):
+                newest[e.get('well_id') or 'pad'] = e
+            for scope, e in newest.items():
+                p = os.path.join(ws.reports_dir, e['path'])
+                key = f'dossier.changed:{scope}'
+                if not os.path.isfile(p):
+                    continue
+                r = DZ.verify(p, ws)
+                bad = [c for c in (r.get('workspace_checks') or []) + r['checks'] if not c['ok']]
+                _transition(ws, notifier, st, key, bool(bad),
+                            {'event': 'dossier.changed', 'well_id': e.get('well_id'), 'well': e.get('display'), 'what': f"the dossier {e['dossier_id']}",
+                             'dossier_id': e['dossier_id'], 'detail': '; '.join(f"{c['check']}: {c['detail']}" for c in bad)}, now, raised)
+        except Exception as ex:
+            last['dossiers_error'] = f'{type(ex).__name__}: {ex}'
+    save_state(ws, st)
+    return raised

gea_program-0.44.0/gea/help/alerts.md

@@ -0,0 +1,17 @@
+# The earthquake watch and the program's alerts
+
+> While a well operates, the program reads the USGS earthquake catalogue every few minutes from this machine and raises an alert when an earthquake near the well reaches the chosen regulator's protocol level. It also raises alerts for the conditions nobody watches by hand: the watch itself going quiet, a component's service interval used up, a penetrator check outside its rating, and a signed dossier the workspace no longer matches.
+
+**The command**: the Operations & Seismicity tab's **Live earthquake watch** card - on or off, the interval (5 minutes by default), whether it polls only while an operation is under way (the default) or always, and **Poll now** to try the connection. Every alert is listed on the Alarms tab under **Program alerts**. From a terminal: `gea seismicity --workspace C:\site --watch-once` polls now and says what came back; `--watch on|off`, `--watch-every 5`, `--watch-mode always`; `gea alerts --workspace C:\site` lists the alerts, `--check` checks the motors, penetrators and dossiers now.
+
+**How the watch reads the catalogue**: it runs in the service's background poller, so only while `gea serve` (the control panel) is running. An operation is under way when a declared operation (Operations & Seismicity, or the declarations sheet's `[operations]`) has started and has not ended, at a well with a surface position. A poll reads the USGS real-time summary feed of the past day, all magnitudes, updated every minute - the feed USGS asks automated programs to use - and keeps the earthquakes inside the circle around the operating wells (out to the screening distance, 20 km by default, beyond the farthest). On the first poll, or after a gap of a day or more, it asks the FDSN event service for the whole gap instead (from the operation's start, at most 30 days back). USGS revises magnitudes and locations; the newest version wins, and a revision that reaches a higher protocol level is a new alert.
+
+**The alerts**: `seismic.protocol` (a protocol level reached, once per level), `seismic.event` (any earthquake within the association radius during or after an operation, once), `seismic.watch_down` (no answer from the catalogue for three intervals while an operation runs) and `seismic.watch_up`; `motor.due` (checked every 15 minutes), `penetrator.outside` (every 15 minutes), `dossier.changed` (the newest dossier of each well and of the pad, re-checked daily), and `condition.cleared` when one of the last three ends.
+
+**Getting them to a person**: every alert is written to `records/alerts.jsonl` and shown on the Alarms tab. To have it sent, set up notifications (Administration, Notifications: load the example, edit, validate and commit) with an email (SMTP) or webhook channel and a rule naming the alert, for example `{"event": "seismic.protocol", "channels": ["mail"]}`; the password stays in an environment variable named by `password_env`. A text message: most carriers deliver email sent to the phone's email-to-text address, or a webhook to a messaging service. The `quiet_s` window keeps one alert from repeating.
+
+**What it writes**: `seismicity/watch.json` (the settings), `seismicity/live/events.json` (each earthquake's latest version) and `state.json`, the catalogue the tab reads (`seismicity/catalogs/live_usgs.txt`, listed with the window and circle the watch covered), `records/alerts.jsonl`, `records/alerts_state.json`; the audit log records each change of settings and each manual poll.
+
+**The number to check**: an alert's magnitude, time and distance against the earthquake's USGS event page (its id is in the alert); the watch's last good poll on the tab against the clock.
+
+**What this page will not call a measurement**: the earthquake - its magnitude and position are the catalogue's, automatic at first and revised later; a cause - an alert says an earthquake reached a level near an operation, not that the operation caused it; the regulator's decision - the protocol's magnitude of record may be another network's. A watch that cannot reach the catalogue says so; it never reports quiet it did not see.

{gea_program-0.43.0 → gea_program-0.44.0}/gea/help/notifications.md

@@ -2,7 +2,7 @@
 
 > Rules route events to a webhook or a mailbox; every attempt is logged; secrets stay out of the file.
 
-**The command**: Administration -> Notifications: "Load the example", edit, "Validate and commit" (it is versioned as `notifications`), "Send a test" per channel. `gea notify --example`, `gea notify --workspace C:\site --test mail`, `gea notify --workspace C:\site --log 20`. Events: alarm.activated (by priority), alarm.shelved, job.failed, patch.down, patch.up, approval.pending, file.imported.
+**The command**: Administration -> Notifications: "Load the example", edit, "Validate and commit" (it is versioned as `notifications`), "Send a test" per channel. `gea notify --example`, `gea notify --workspace C:\site --test mail`, `gea notify --workspace C:\site --log 20`. Events: alarm.activated (by priority), alarm.shelved, job.failed, patch.down, patch.up, approval.pending, file.imported; and the program's alerts - seismic.protocol, seismic.event, seismic.watch_down, seismic.watch_up, motor.due, penetrator.outside, dossier.changed, condition.cleared (see the help page on the earthquake watch and the alerts).
 
 **What it writes**: `records/notifications.jsonl` - one line per delivery attempt and per suppression (the quiet window stops repeats of the same key); the service's poller reads the alarm logs, the jobs, the patch states and the approval queue every 10 seconds and starts from a baseline at start-up, so history is never re-announced.
 

{gea_program-0.43.0 → gea_program-0.44.0}/gea/help/opseis.md

@@ -4,6 +4,8 @@
 
 **The command**: the Operations & Seismicity tab. Choose the scene - your wells, or the worked example - and the protocol (British Columbia's Kiskatinaw order, Alberta's Subsurface Order No. 2, Oklahoma's completion notice, Ohio's 2014 permit conditions, Texas's response plan), the association radius (5 km by default), the days after an operation that still count (90) and the completeness magnitude. Each well shows its operations as bands on a time line, every event within the screening distance (20 km) as a circle by its distance from the well and sized by magnitude, the events table with the rig's state at an event during a drilling record, the protocol level each reaches, and the rate comparison. The operator declares operation windows (or the declarations sheet's `[operations]`) and attaches catalogue exports with the window, minimum magnitude and circle they were queried for (or `[catalogs]`); the program cannot reach a catalogue service itself, so the tab gives the ComCat query to open in a browser. From a terminal: `gea seismicity --workspace C:\site`; `--scene mahoning` for the example; `--protocols`; `--query`; `--attach FILE --start --end --min-magnitude --lat --lon --catalog-radius-km --source`; `--declare-op --well W --kind completion --start --end --source`; `--set` keeps the settings.
 
+**The live watch**: while an operation is under way the program can read the USGS catalogue itself every few minutes and raise an alert when an earthquake reaches the protocol's level - see the help page on the earthquake watch and the alerts.
+
 **What it writes**: `seismicity/operations.json`, `seismicity/catalogs.json` with each catalogue file kept verbatim beside it (its SHA-256 recorded), and `seismicity/settings.json` in the workspace; each declaration, attachment, removal and change of settings a line in the audit log. The protocols ship with the program (`reference/seismicity_protocols.json`), each with its document; the worked example's catalogue (`fixtures/comcat_mahoning_1990-2026.txt`) and its wells and dates (`reference/mahoning_example.json`), each with its source.
 
 **The arithmetic**: an event's distance is the great-circle distance from the well's declared surface position (converted to WGS84). It is DURING an operation inside its window, AFTER one within the association days that follow, and OUTSIDE otherwise; an event near two wells' operations is named on both, each with the other beside it. The rate comparison counts the events at or above the completeness within the radius over the catalogue's window: n inside the operation windows (with their association days) and m outside. If one constant rate held, n would be binomial in n + m with the share of time inside; the probability of n or more is the exact conditional test for two Poisson rates. HIGHER DURING OPERATIONS means that probability is under 1 % and the rate inside is the higher; otherwise NOT DISTINGUISHED. With no event outside, the background is given as its 95 % upper bound, 3 events over its time.

{gea_program-0.43.0 → gea_program-0.44.0}/gea/helplib.py

@@ -50,6 +50,7 @@ INDEX: List[tuple] = [
     ('declare', 'The declarations sheet', 'everything the operator declares about a pad in one spreadsheet - identity, position, elevation, surveys, rates, AFE, fleet, runs, penetrators and drives - checked row by row before anything is written'),
     ('penetrator', 'Completion - the ESP penetrator', "the wellhead penetrator's ratings from its maker's documents, each well's installation record, and the ESP drive, motor and site checked against them; the permanent magnet motor safety note"),
     ('opseis', 'Operations and seismicity', "each well's drilling, completion and disposal windows against the earthquake catalogue around it: the events during and after each, the levels a regulator's protocol would reach, the rate during operations against the rest of the record; the Mahoning Valley worked example"),
+    ('alerts', 'The earthquake watch and the alerts', "the USGS catalogue read every few minutes while a well operates, an alert when a protocol level is reached; and alerts for the watch going quiet, service intervals used up, penetrators outside their ratings and dossiers the workspace no longer matches"),
     ('report-history', 'Report history', 'every version of every client report kept in a dated folder with its SHA-256, when the next one replaces it'),
     ('dossier', 'The signed dossier', "a well's or the pad's reports, declarations, audit log and source hashes in one zip signed with the site's Ed25519 key, with a checker anyone can run without GEA"),
     ('economics', 'Economics', 'each well\'s days, footage and cost by the operator\'s own declared rates, against the EIA oil and gas spot prices while it was drilled, and any series the site imports'),