gea-program 0.22.0__tar.gz → 0.24.0__tar.gz

{gea_program-0.22.0 → gea_program-0.24.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gea-program
-Version: 0.22.0
+Version: 0.24.0
 Summary: GEA-Program: downhole gauge monitoring - canonical measurement records, quality rules, drift evaluation with a re-fit change log, well-test validation, alarm management, model cards, store-and-forward, monthly SLA measurement, FAT/SAT protocols and a dashboard, generated from live objects in the client's reporting outline.
 Author: Daniel T. Murphy
 License: MPL-2.0
@@ -125,7 +125,7 @@ gea dashboard --catalog-well volve_f12_f14_production_excerpt:15/9-F-12:10000 --
 gea client-report --report accuracy --out client_report
 gea model-cards --out model_cards
 gea sbom --out sbom
-gea accept                     # the product gate (353 checks)
+gea accept                     # the product gate (359 checks)
 gea help drift                 # the help library, by the job (16 pages; the same text is on every dashboard page)
 gea guide                      # the click-by-click tester guide (docs/TESTER_GUIDE.md)
 gea gui                        # the desktop window (pip install "gea-program[desktop]")
@@ -648,6 +648,39 @@ against it; `gea seedlink --config ... --hello` prints what a server says of
 itself; `gea help seedlink`. A TexNet station is served by EarthScope at
 `rtserve.iris.washington.edu:18000`, network `TX`.
 
+### The log as delivered, and the historian
+
+A well log arrives as DLIS (API RP66 V1); `gea/dlis.py` reads the container
+as the standard lays it out, with no dependency - the storage unit label,
+visible records, logical record segments and trailers, the sets of objects
+with every representation code, the frames and channels, the frame data -
+and hands the strata survey the same stream the LAS port does: `gea dlis
+--file log.dlis --info` describes it, `--to-las` exports a frame, `gea survey
+--file log.dlis` reads it directly, and the Files page detects it by its
+label. Encrypted records are skipped and named, never decrypted or guessed.
+Proven on a real Schlumberger station log (104 channels, two frames, 3,252
+records in a tenth of a second). The operator's gauge history in a PI System
+comes out through PI Web API (`gea piwebapi`, protocol `piwebapi` on the
+Patch panel): GET only, the archive between two times with its paging
+followed, every value with the archive's timestamp and flags, credentials
+from environment variables the map names. `gea help dlis`, `gea help
+historian`.
+
+### The field ports
+
+The three links a gauge panel on a pad actually offers, each read-only in
+the code (no write, control, time set or device-changing command exists),
+each refusing to run without the site's map and its source, each with a
+stand-in on the loopback to rehearse against, each one more protocol on the
+Patch panel: **Modbus RTU** (`gea modbus-rtu`; the RS-485 drop through
+pyserial, the `serial` extra, or RTU over TCP through a serial-to-Ethernet
+converter; CRC-16 framed and checked by this program; functions 03 and 04),
+**DNP3** (`gea dnp3`; IEEE 1815 data link, transport and application layers;
+integrity and event polls; analog, binary and counter objects with their
+flags and 48-bit event times; confirms and unsolicited responses handled),
+**HART-IP** (`gea hartip`; the session, command 0, 3 and 9; HART unit codes
+converted; device and variable status to quality). `gea help field-ports`.
+
 ### The interval against the mapped tops
 
 Where does the injection sit in the geology? The operator declares the

{gea_program-0.22.0 → gea_program-0.24.0}/README.md

@@ -86,7 +86,7 @@ gea dashboard --catalog-well volve_f12_f14_production_excerpt:15/9-F-12:10000 --
 gea client-report --report accuracy --out client_report
 gea model-cards --out model_cards
 gea sbom --out sbom
-gea accept                     # the product gate (353 checks)
+gea accept                     # the product gate (359 checks)
 gea help drift                 # the help library, by the job (16 pages; the same text is on every dashboard page)
 gea guide                      # the click-by-click tester guide (docs/TESTER_GUIDE.md)
 gea gui                        # the desktop window (pip install "gea-program[desktop]")
@@ -609,6 +609,39 @@ against it; `gea seedlink --config ... --hello` prints what a server says of
 itself; `gea help seedlink`. A TexNet station is served by EarthScope at
 `rtserve.iris.washington.edu:18000`, network `TX`.
 
+### The log as delivered, and the historian
+
+A well log arrives as DLIS (API RP66 V1); `gea/dlis.py` reads the container
+as the standard lays it out, with no dependency - the storage unit label,
+visible records, logical record segments and trailers, the sets of objects
+with every representation code, the frames and channels, the frame data -
+and hands the strata survey the same stream the LAS port does: `gea dlis
+--file log.dlis --info` describes it, `--to-las` exports a frame, `gea survey
+--file log.dlis` reads it directly, and the Files page detects it by its
+label. Encrypted records are skipped and named, never decrypted or guessed.
+Proven on a real Schlumberger station log (104 channels, two frames, 3,252
+records in a tenth of a second). The operator's gauge history in a PI System
+comes out through PI Web API (`gea piwebapi`, protocol `piwebapi` on the
+Patch panel): GET only, the archive between two times with its paging
+followed, every value with the archive's timestamp and flags, credentials
+from environment variables the map names. `gea help dlis`, `gea help
+historian`.
+
+### The field ports
+
+The three links a gauge panel on a pad actually offers, each read-only in
+the code (no write, control, time set or device-changing command exists),
+each refusing to run without the site's map and its source, each with a
+stand-in on the loopback to rehearse against, each one more protocol on the
+Patch panel: **Modbus RTU** (`gea modbus-rtu`; the RS-485 drop through
+pyserial, the `serial` extra, or RTU over TCP through a serial-to-Ethernet
+converter; CRC-16 framed and checked by this program; functions 03 and 04),
+**DNP3** (`gea dnp3`; IEEE 1815 data link, transport and application layers;
+integrity and event polls; analog, binary and counter objects with their
+flags and 48-bit event times; confirms and unsolicited responses handled),
+**HART-IP** (`gea hartip`; the session, command 0, 3 and 9; HART unit codes
+converted; device and variable status to quality). `gea help field-ports`.
+
 ### The interval against the mapped tops
 
 Where does the injection sit in the geology? The operator declares the

{gea_program-0.22.0 → gea_program-0.24.0}/gea/__init__.py

@@ -123,7 +123,7 @@ from .profile_catalog import (
     read_pangaea_txt,
 )
 
-__version__ = "0.22.0"
+__version__ = "0.24.0"
 __all__ = [
     "aging_rate", "rate_psi_yr", "accuracy_psi",
     "Sensor", "SimulatorConfig", "DownholeEngine",

{gea_program-0.22.0 → gea_program-0.24.0}/gea/__main__.py

@@ -278,6 +278,35 @@ def main(argv=None) -> int:
         p_lp.add_argument("--stream-csv", type=str, default=None, help="also write a historian-style CSV the other commands ingest")
         p_lp.add_argument("--selftest", action="store_true", help="etp: the codec round-trip and a session against the simulated store on the loopback")
 
+    p_dl = sub.add_parser("dlis", help="a DLIS (RP66 V1) well log: --info describes it (origin, channels, frames, what was skipped); --frame --out writes the frame as a historian-style CSV; --to-las writes LAS 2.0")
+    p_dl.add_argument("--file", type=str, required=True)
+    p_dl.add_argument("--info", action="store_true")
+    p_dl.add_argument("--frame", type=str, default=None, help="the frame to read when the file holds several")
+    p_dl.add_argument("--out", type=str, default=None, help="write the frame's channels as CSV (index first)")
+    p_dl.add_argument("--to-las", type=str, default=None, help="write the frame as a LAS 2.0 file")
+    p_dl.add_argument("--json", action="store_true")
+    p_pi = sub.add_parser("piwebapi", help="the PI Web API historian pull, read-only: --config (base_url, auth by environment variable names, tags with paths); --selftest against the stand-in")
+    p_pi.add_argument("--config", type=str, default=None)
+    p_pi.add_argument("--write-example-config", type=str, default=None)
+    p_pi.add_argument("--out", type=str, default=None, help="write the records CSV here")
+    p_pi.add_argument("--stream-csv", type=str, default=None)
+    p_pi.add_argument("--seconds", type=float, default=None, help="ignored: a pull is one request per tag (kept so the live tap's call shape is the same)")
+    p_pi.add_argument("--record", type=str, default=None, help="write the pull's summary as one JSON line here")
+    p_pi.add_argument("--selftest", action="store_true")
+    for name, helptxt in (("modbus-rtu", "Modbus RTU port, read-only: the serial line (pyserial) or RTU over TCP; functions 03/04 only; the stand-in slave with --selftest"),
+                          ("dnp3", "DNP3 (IEEE 1815) master, read-only: integrity and event polls over TCP; analog inputs, binaries, counters with flags and event times; the stand-in outstation with --selftest"),
+                          ("hartip", "HART-IP client, read-only: session, command 0, 3 and 9; the variables with their unit codes and status; the stand-in device with --selftest")):
+        p_fp = sub.add_parser(name, help=helptxt)
+        p_fp.add_argument("--config", type=str, default=None, help="site map JSON (client-owned): the transport, the mapping and its source")
+        p_fp.add_argument("--write-example-config", type=str, default=None, help="write the example map (the stand-in) here and exit")
+        p_fp.add_argument("--seconds", type=float, default=None, help="run for this long (polls = seconds / interval_s); default the map's polls")
+        p_fp.add_argument("--polls", type=int, default=None, help="poll this many times instead")
+        p_fp.add_argument("--record", type=str, default=None, help="write one JSON line per poll (the values and faults) here")
+        p_fp.add_argument("--replay", type=str, default=None, help="not offered by these ports (a poll is a request; there is nothing to replay)")
+        p_fp.add_argument("--out", type=str, default=None, help="write the records CSV here")
+        p_fp.add_argument("--stream-csv", type=str, default=None, help="also write a historian-style CSV the other commands ingest")
+        p_fp.add_argument("--selftest", action="store_true", help="the codec vectors and a session against the stand-in on the loopback")
+
     p_ws = sub.add_parser("workspace", help="the client site folder: init, add wells (file | catalogue | live), list, migrate a --out folder, refresh the dashboard, audit log")
     p_ws.add_argument("--path", type=str, required=True, help="the workspace folder")
     p_ws.add_argument("--action", type=str, default="list",
@@ -378,7 +407,7 @@ def main(argv=None) -> int:
     p_ws.add_argument("--entry", type=str, default=None, help="add-catalog: catalogue entry")
     p_ws.add_argument("--well", type=str, default=None, help="add-catalog: well tag inside the entry; remove: well id")
     p_ws.add_argument("--station-md", type=float, default=None, help="station MD in ft (catalogue wells need it)")
-    p_ws.add_argument("--port", type=str, default=None, choices=["opcua", "mqtt", "modbus_g6"], help="add-live: the port")
+    p_ws.add_argument("--port", type=str, default=None, choices=["opcua", "mqtt", "modbus_g6", "wits0", "witsml", "etp", "modbus_rtu", "dnp3", "hartip"], help="add-live: the port")
     p_ws.add_argument("--from", dest="from_dir", type=str, default=None, help="migrate: a folder written by `gea dashboard --out`")
     p_ws.add_argument("--actor", type=str, default="cli", help="who is acting (recorded in the audit log)")
     p_ws.add_argument("--month", type=str, default=None, help="refresh: also measure the SLA for YYYY-MM")
@@ -657,8 +686,9 @@ def main(argv=None) -> int:
     p_fl.add_argument("--station-md", type=float, default=None)
     p_fl.add_argument("--actor", type=str, default="cli")
 
-    p_sy = sub.add_parser("survey", help="a LAS file in, one strata report out (--demo: the bundled public KTB excerpt); --out writes report.txt + survey.json")
-    p_sy.add_argument("--file", type=str, default=None, help="LAS 2.0 file")
+    p_sy = sub.add_parser("survey", help="a LAS or DLIS file in, one strata report out (--demo: the bundled public KTB excerpt); --out writes report.txt + survey.json")
+    p_sy.add_argument("--file", type=str, default=None, help="LAS 2.0 or DLIS (RP66 V1) file")
+    p_sy.add_argument("--frame", type=str, default=None, help="DLIS: the frame to read when the file holds several (gea dlis --file X --info lists them)")
     p_sy.add_argument("--demo", action="store_true")
     p_sy.add_argument("--family", type=str, default="continental_crystalline", help="prior family for the estimator")
     p_sy.add_argument("--lat", type=float, default=None)
@@ -794,8 +824,8 @@ def main(argv=None) -> int:
         import json as _json
         from .survey_cmd import run_survey
         if not (a.file or a.demo):
-            raise SystemExit("survey needs --file <LAS> or --demo")
-        txt, machine = run_survey(path=a.file, demo=a.demo, family=a.family, lat=a.lat, elev=a.elev)
+            raise SystemExit("survey needs --file <LAS or DLIS> or --demo")
+        txt, machine = run_survey(path=a.file, demo=a.demo, family=a.family, lat=a.lat, elev=a.elev, frame=a.frame)
         print(txt)
         if a.out:
             os.makedirs(a.out, exist_ok=True)
@@ -2268,6 +2298,175 @@ def main(argv=None) -> int:
             stop.set()
             print("etp-sim: stopped")
         return 0
+    elif a.cmd == "dlis":
+        import json as _json
+        from .dlis import DlisError, DlisFile, read as _read_dlis
+        try:
+            if a.info or not (a.out or a.to_las):
+                with open(a.file, "rb") as f:
+                    info = DlisFile(f.read(), os.path.basename(a.file)).describe()
+                if a.json:
+                    print(_json.dumps(info, indent=1, default=str)); return 0
+                o = info["origin"]
+                print(f"dlis: {a.file}")
+                print(f"  storage set: {info['storage_unit']['storage_set_id']}; file {info['file_header'].get('ID', '').strip()} seq {info['file_header'].get('SEQUENCE-NUMBER', '').strip()}")
+                print(f"  well: {o.get('WELL-NAME', '').strip() or '?'}  field: {o.get('FIELD-NAME', '').strip() or '?'}  company: {o.get('COMPANY', '').strip() or '?'}  producer: {o.get('PRODUCER-NAME', '').strip() or '?'}  created: {o.get('CREATION-TIME', '?')}")
+                print(f"  {info['n_records']} logical records; sets: " + ", ".join(f"{t}{'(' + n + ')' if n else ''} x{k}" for t, n, k in info["eflr_sets"]))
+                if info.get("encrypted_records"):
+                    print("  encrypted records skipped: " + ", ".join(f"{k} x{n}" for k, n in info["encrypted_records"].items()))
+                for u in info["unparsed"]:
+                    print("  not parsed: " + u)
+                print(f"  {len(info['channels'])} channels:")
+                for c in info["channels"]:
+                    print(f"    {c['name']:<12s} {c['units']:<10s} {c['code']:<7s} dim {c['dimension']}  {c['long_name']}")
+                for fr in info["frames"]:
+                    print(f"  frame {fr['name']}: index {fr['index_type']} {fr['direction']}, spacing {fr['spacing']} {fr['spacing_units']}, {fr['index_min']}..{fr['index_max']}; {len(fr['channels'])} channels: {', '.join(fr['channels'][:12])}{' ...' if len(fr['channels']) > 12 else ''}")
+                return 0
+            st = _read_dlis(a.file, frame=a.frame)
+        except (DlisError, OSError) as e:
+            raise SystemExit(f"dlis: {e}")
+        print(f"dlis: frame {st.meta['frame']}: {len(st.index)} rows, index {st.meta['index_channel']} ({st.meta['index_units']}) {float(st.index[0]):g}..{float(st.index[-1]):g}, {len(st.channels)} channels"
+              + (f"; {st.meta['notes']}" if st.meta.get("notes") else ""))
+        if a.out:
+            import csv as _csv
+            with open(a.out, "w", newline="", encoding="utf-8") as f:
+                w = _csv.writer(f)
+                w.writerow([st.meta["index_channel"]] + list(st.channels))
+                w.writerow([st.meta["index_units"]] + [st.channels[c].unit for c in st.channels])
+                for i in range(len(st.index)):
+                    w.writerow([float(st.index[i])] + ["" if st.channels[c].values[i] != st.channels[c].values[i] else float(st.channels[c].values[i]) for c in st.channels])
+            print("  csv:", a.out)
+        if a.to_las:
+            with open(a.to_las, "w", encoding="utf-8") as f:
+                f.write("~VERSION INFORMATION\n VERS.   2.0 : CWLS LOG ASCII STANDARD - VERSION 2.0\n WRAP.   NO  : ONE LINE PER DEPTH STEP\n")
+                f.write("~WELL INFORMATION\n")
+                f.write(f" STRT.{st.meta['index_units']:<6s} {float(st.index[0]):.4f} : START\n STOP.{st.meta['index_units']:<6s} {float(st.index[-1]):.4f} : STOP\n")
+                f.write(f" STEP.{st.meta['index_units']:<6s} {(float(st.index[1]) - float(st.index[0])) if len(st.index) > 1 else 0.0:.4f} : STEP\n NULL.       -999.25 : NULL VALUE\n")
+                f.write(f" WELL.       {st.meta.get('WELL-NAME', '').strip()} : WELL\n FLD .       {st.meta.get('FIELD-NAME', '').strip()} : FIELD\n COMP.       {st.meta.get('COMPANY', '').strip()} : COMPANY\n SRVC.       {st.meta.get('PRODUCER-NAME', '').strip()} : SERVICE COMPANY\n")
+                f.write(f" SRC .       DLIS {os.path.basename(a.file)} frame {st.meta['frame']} : SOURCE (gea dlis --to-las)\n")
+                f.write("~CURVE INFORMATION\n")
+                f.write(f" {st.meta['index_channel']:<8s}.{st.meta['index_units']:<8s} : INDEX\n")
+                for c in st.channels:
+                    f.write(f" {c:<8s}.{st.channels[c].unit:<8s} : {c}\n")
+                f.write("~A\n")
+                for i in range(len(st.index)):
+                    f.write(f"{float(st.index[i]):.4f}" + "".join(f" {(-999.25 if st.channels[c].values[i] != st.channels[c].values[i] else float(st.channels[c].values[i])):.4f}" for c in st.channels) + "\n")
+            print("  las:", a.to_las)
+        return 0
+    elif a.cmd == "piwebapi":
+        import json as _json
+        from .live_ports import records_to_stream, summarize, write_records_csv
+        from . import piwebapi as _PI
+        if a.selftest:
+            r = _PI.selftest()
+            print(f"piwebapi selftest: {'OK' if r['ok'] else 'FAILED'}")
+            for k, v in r.items():
+                if k != "ok":
+                    print(f"  {'ok ' if v else 'BAD'} {k}")
+            return 0 if r["ok"] else 1
+        if a.write_example_config:
+            with open(a.write_example_config, "w", encoding="utf-8") as f:
+                _json.dump(_PI.EXAMPLE_CONFIG, f, indent=1)
+            print("config:", a.write_example_config); return 0
+        if not a.config:
+            raise SystemExit("piwebapi needs --config (or --write-example-config to start one)")
+        cfg = _PI.load_config(a.config)
+        sim_h = None
+        try:
+            if cfg.get("base_url") == "sim":
+                sim = _PI.SimServer(); sim_h, port_ = sim.serve(); cfg = {**cfg, "base_url": f"http://127.0.0.1:{port_}/piwebapi", "auth": "none"}
+            try:
+                port = _PI.PiPort(cfg)
+                r = port.pull()
+            except (_PI.PiError, ValueError) as e:
+                raise SystemExit(f"piwebapi: {e}")
+        finally:
+            if sim_h is not None:
+                sim_h.close()
+        recs = port.records
+        print(f"piwebapi: {_json.dumps(port.summary(), default=str)}")
+        print(_json.dumps(summarize(recs), indent=1))
+        if a.record:
+            with open(a.record, "w", encoding="utf-8") as f:
+                f.write(_json.dumps({"pull": r, "summary": port.summary()}, default=str) + "\n")
+        if a.out:
+            print("  records:", write_records_csv(recs, a.out))
+        if a.stream_csv and recs:
+            st = records_to_stream(recs, name="piwebapi")
+            import csv as _csv
+            from datetime import datetime as _dt, timedelta as _td
+            t0 = _dt.fromisoformat(st.meta["start_time"])
+            with open(a.stream_csv, "w", newline="", encoding="utf-8") as f:
+                w = _csv.writer(f)
+                w.writerow(["timestamp"] + list(st.channels))
+                for i, t in enumerate(st.index):
+                    w.writerow([(t0 + _td(seconds=float(t))).isoformat()] + ["" if st.channels[c].values[i] != st.channels[c].values[i] else round(float(st.channels[c].values[i]), 4) for c in st.channels])
+            print("  stream:", a.stream_csv)
+        return 0 if not port.faults else 1
+    elif a.cmd in ("modbus-rtu", "dnp3", "hartip"):
+        import json as _json
+        from .live_ports import records_to_stream, summarize, write_records_csv
+        mod = __import__({"modbus-rtu": "gea.modbus_rtu", "dnp3": "gea.dnp3", "hartip": "gea.hartip"}[a.cmd], fromlist=["x"])
+        if a.selftest:
+            r = mod.selftest()
+            print(f"{a.cmd} selftest: {'OK' if r['ok'] else 'FAILED'}")
+            for k, v in r.items():
+                if k != "ok":
+                    print(f"  {'ok ' if v else 'BAD'} {k}")
+            return 0 if r["ok"] else 1
+        if a.write_example_config:
+            with open(a.write_example_config, "w", encoding="utf-8") as f:
+                _json.dump(mod.EXAMPLE_CONFIG, f, indent=1)
+            print("config:", a.write_example_config); return 0
+        if a.replay:
+            raise SystemExit(f"{a.cmd}: a poll is a request to the device; there is no recording to replay - run against the device or the stand-in")
+        if not a.config:
+            raise SystemExit(f"{a.cmd} needs --config (or --write-example-config to start one)")
+        cfg = mod.load_config(a.config)
+        polls = a.polls
+        if polls is None and a.seconds is not None:
+            polls = max(1, int(round(a.seconds / max(float(cfg.get("interval_s", 1.0)), 0.05))))
+        from .patches import close_field_port, open_field_port
+        proto = {"modbus-rtu": "modbus_rtu", "dnp3": "dnp3", "hartip": "hartip"}[a.cmd]
+        try:
+            tap = open_field_port(proto, mod, cfg)
+        except (NotImplementedError, ConnectionError, ValueError, OSError) as e:
+            raise SystemExit(f"{a.cmd}: {e}")
+        rec_f = open(a.record, "w", encoding="utf-8") if a.record else None
+        n = int(cfg.get("polls", 3) if polls is None else polls)
+        dt = float(cfg.get("interval_s", 1.0))
+        try:
+            for k in range(n):
+                before = len(tap.records)
+                r = tap.poll_once(integrity=(k == 0)) if a.cmd == "dnp3" else tap.poll_once()
+                if rec_f:
+                    rec_f.write(_json.dumps({"poll": k + 1, "utc": tap.records[-1].ingest_timestamp_utc if tap.records else None,
+                                             "values": {x.tag_id: x.value for x in tap.records[before:]}, "quality": {x.tag_id: x.quality_flag for x in tap.records[before:]},
+                                             "detail": (r if isinstance(r, dict) and a.cmd != "modbus-rtu" else None)}, default=str) + "\n")
+                if dt > 0 and k < n - 1:
+                    import time as _time
+                    _time.sleep(dt)
+        finally:
+            close_field_port(proto, tap)
+            if rec_f:
+                rec_f.close()
+        recs = tap.records
+        print(f"{a.cmd}: {_json.dumps(tap.summary(), default=str)}")
+        print(_json.dumps(summarize(recs), indent=1))
+        if a.out:
+            print("  records:", write_records_csv(recs, a.out))
+        if a.stream_csv and recs:
+            st = records_to_stream(recs, name=proto)
+            import csv as _csv
+            from datetime import datetime as _dt, timedelta as _td
+            t0 = _dt.fromisoformat(st.meta["start_time"])
+            with open(a.stream_csv, "w", newline="", encoding="utf-8") as f:
+                w = _csv.writer(f)
+                w.writerow(["timestamp"] + list(st.channels))
+                for i, t in enumerate(st.index):
+                    w.writerow([(t0 + _td(seconds=float(t))).isoformat()] + ["" if st.channels[c].values[i] != st.channels[c].values[i] else round(float(st.channels[c].values[i]), 4) for c in st.channels])
+            print("  stream:", a.stream_csv)
+        return 0
     elif a.cmd in ("opcua", "mqtt", "wits0", "witsml", "etp"):
         import json as _json
         from .live_ports import records_to_stream, summarize, write_records_csv