gea-program 0.18.0__tar.gz → 0.19.0__tar.gz

{gea_program-0.18.0 → gea_program-0.19.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gea-program
-Version: 0.18.0
+Version: 0.19.0
 Summary: GEA-Program: downhole gauge monitoring - canonical measurement records, quality rules, drift evaluation with a re-fit change log, well-test validation, alarm management, model cards, store-and-forward, monthly SLA measurement, FAT/SAT protocols and a dashboard, generated from live objects in the client's reporting outline.
 Author: Daniel T. Murphy
 License: MPL-2.0
@@ -125,7 +125,7 @@ gea dashboard --catalog-well volve_f12_f14_production_excerpt:15/9-F-12:10000 --
 gea client-report --report accuracy --out client_report
 gea model-cards --out model_cards
 gea sbom --out sbom
-gea accept                     # the product gate (342 checks)
+gea accept                     # the product gate (344 checks)
 gea help drift                 # the help library, by the job (16 pages; the same text is on every dashboard page)
 gea guide                      # the click-by-click tester guide (docs/TESTER_GUIDE.md)
 gea gui                        # the desktop window (pip install "gea-program[desktop]")
@@ -710,6 +710,24 @@ after the report was written - and "Update every report from its source"
 dashboard for every well, then every seismic station, errors collected and
 listed, never hidden.
 
+### The month-end packet
+
+A client's month was five or six commands and a walk through the reports
+folder. `gea workspace --path C:\site --action month-end --period 2026-09`
+(or `--period previous`) runs the refresh for the period and collects every
+report the program writes - each well's drift report, alarm month, well-test
+validation, shut-in report, data resilience, machine vibration and the
+certificate register's ISO/IEC 17025 7.8 conformity; each site's SRA packet
+for the period and site report; every seismic station and track; the
+accuracy statement, the SLA month, the FAT/SAT protocol, the model cards and
+the SBOM - into `reports/month_end/<period>/` with `INDEX` (html, md, json:
+every item INCLUDED with its files or NOT AVAILABLE with the reason that
+names what the site lacks; every failed step; OPEN for a month that has not
+ended) and `MANIFEST.json` (every file's SHA-256), zipped beside it. Nothing
+is recomputed in the index. On the Reports page: the packets on record,
+Assemble as a job, and Schedule monthly (day and hour, the previous month).
+`gea help month-end`.
+
 ### Before a site goes live
 
 `deploy/README.md` is the checklist: keep the service on the loopback

{gea_program-0.18.0 → gea_program-0.19.0}/README.md

@@ -86,7 +86,7 @@ gea dashboard --catalog-well volve_f12_f14_production_excerpt:15/9-F-12:10000 --
 gea client-report --report accuracy --out client_report
 gea model-cards --out model_cards
 gea sbom --out sbom
-gea accept                     # the product gate (342 checks)
+gea accept                     # the product gate (344 checks)
 gea help drift                 # the help library, by the job (16 pages; the same text is on every dashboard page)
 gea guide                      # the click-by-click tester guide (docs/TESTER_GUIDE.md)
 gea gui                        # the desktop window (pip install "gea-program[desktop]")
@@ -671,6 +671,24 @@ after the report was written - and "Update every report from its source"
 dashboard for every well, then every seismic station, errors collected and
 listed, never hidden.
 
+### The month-end packet
+
+A client's month was five or six commands and a walk through the reports
+folder. `gea workspace --path C:\site --action month-end --period 2026-09`
+(or `--period previous`) runs the refresh for the period and collects every
+report the program writes - each well's drift report, alarm month, well-test
+validation, shut-in report, data resilience, machine vibration and the
+certificate register's ISO/IEC 17025 7.8 conformity; each site's SRA packet
+for the period and site report; every seismic station and track; the
+accuracy statement, the SLA month, the FAT/SAT protocol, the model cards and
+the SBOM - into `reports/month_end/<period>/` with `INDEX` (html, md, json:
+every item INCLUDED with its files or NOT AVAILABLE with the reason that
+names what the site lacks; every failed step; OPEN for a month that has not
+ended) and `MANIFEST.json` (every file's SHA-256), zipped beside it. Nothing
+is recomputed in the index. On the Reports page: the packets on record,
+Assemble as a job, and Schedule monthly (day and hour, the previous month).
+`gea help month-end`.
+
 ### Before a site goes live
 
 `deploy/README.md` is the checklist: keep the service on the loopback

{gea_program-0.18.0 → gea_program-0.19.0}/gea/__init__.py

@@ -123,7 +123,7 @@ from .profile_catalog import (
     read_pangaea_txt,
 )
 
-__version__ = "0.18.0"
+__version__ = "0.19.0"
 __all__ = [
     "aging_rate", "rate_psi_yr", "accuracy_psi",
     "Sensor", "SimulatorConfig", "DownholeEngine",

{gea_program-0.18.0 → gea_program-0.19.0}/gea/__main__.py

@@ -285,7 +285,7 @@ def main(argv=None) -> int:
                                "add-seismic", "refresh-seismic", "remove-seismic", "sar-film", "refresh-all", "add-track", "refresh-track", "remove-track",
                                "seismic-dataset", "seismic-field", "add-site", "sites", "site-report", "remove-site", "rename",
                                "set-well", "sra-define", "sra-report", "associate", "osdu-export", "quakeml-export", "well-identity", "vibration-report",
-                               "add-live-station", "live-fold", "live-run"])
+                               "add-live-station", "live-fold", "live-run", "month-end"])
     p_ws.add_argument("--stations", type=str, nargs="+", default=None, help="add-track: two or more array station ids of this workspace")
     p_ws.add_argument("--site", type=str, default=None, help="site-report/remove-site: the site id")
     p_ws.add_argument("--api", type=str, default=None, help="set-well: the API number")
@@ -307,7 +307,7 @@ def main(argv=None) -> int:
     p_ws.add_argument("--checkpoint-months", type=int, default=3, help="sra-define: months between checkpoints with Commission staff (3)")
     p_ws.add_argument("--formation-boundary", type=str, default="base of the Wolfcamp", help="sra-define: the stratigraphic boundary between the two tiers")
     p_ws.add_argument("--triggering-event", type=str, default="", help="sra-define: the catalogue id of the event the plan answers")
-    p_ws.add_argument("--catalog", type=str, default=None, help="sra-report: a TexNet earthquake-catalogue export (CSV)")
+    p_ws.add_argument("--catalog", type=str, default=None, help="sra-report / month-end: a TexNet earthquake-catalogue export (CSV)")
     p_ws.add_argument("--aftershocks", type=str, nargs="*", default=None, help="sra-report: catalogue ids the operator declares as aftershocks (exempt)")
     p_ws.add_argument("--start", type=str, default=None, help="sra-report: first day of the record, YYYY-MM-DD")
     p_ws.add_argument("--end", type=str, default=None, help="sra-report: last day of the record, YYYY-MM-DD")
@@ -359,6 +359,8 @@ def main(argv=None) -> int:
     p_ws.add_argument("--seedlink-config", type=str, default=None, help="add-live-station: the SeedLink config JSON (gea seedlink --write-example-config)")
     p_ws.add_argument("--include-open", action="store_true", help="live-fold: fold the current day's files too (they are still being written)")
     p_ws.add_argument("--seconds", type=float, default=None, help="live-run: how long to run (default: until Ctrl+C)")
+    p_ws.add_argument("--period", type=str, default="previous", help="month-end: the month, YYYY-MM, or 'previous' (the last closed month)")
+    p_ws.add_argument("--no-refresh", action="store_true", help="month-end: collect the reports as they stand without refreshing first")
     p_ws.add_argument("--name", type=str, default=None, help="init: site name; add-*: display name")
     p_ws.add_argument("--file", type=str, default=None, help="add-file: the client's data file; add-live: the port configuration JSON")
     p_ws.add_argument("--entry", type=str, default=None, help="add-catalog: catalogue entry")
@@ -1944,6 +1946,10 @@ def main(argv=None) -> int:
                     raise SystemExit("well-identity needs --well <id>")
                 r = ws.well_identity(a.well)
                 print(_ppdm_text(r)); return 0
+            elif a.action == "month-end":
+                from .month_end import report_text as _me_text
+                r = ws.month_end(a.period, actor=a.actor, refresh=not a.no_refresh, catalog_csv=a.catalog)
+                print(_me_text(r)); return 0 if not r["errors"] else 1
             elif a.action == "add-live-station":
                 if not a.seedlink_config or not a.name:
                     raise SystemExit("add-live-station needs --name <display> and --seedlink-config <json>")

{gea_program-0.18.0 → gea_program-0.19.0}/gea/acceptance_tests.py

@@ -5872,6 +5872,130 @@ def section_bl_backup(tmp: str) -> None:
        "`--verify` says OK or names the tampered file, `--restore` brings the accounts back; the help page says a backup never restored is a hope"
        + (f" [failed: {', '.join(failed2)}; r_st0={r_st0.stdout[-120:]!r} {r_st0.stderr[-120:]!r}]" if failed2 else ""))
 
+
+def section_bm_month_end(tmp: str) -> None:
+    """Section BM - the month-end packet: the refresh for the period, every report collected with INCLUDED or NOT
+    AVAILABLE and the reason, the index and the manifest, the archive, the records; the page, the job and the
+    monthly schedule."""
+    import http.cookiejar
+    import subprocess as _sp
+    import urllib.request
+    import urllib.error
+    import zipfile
+    from . import month_end as ME
+    from . import helplib as H
+    from .telemetry import TelemetryRecorder, TelemetryConfig
+    from .downhole_engine import DownholeEngine, SimulatorConfig
+    from .workspace import Workspace
+    from .service import Service, Users
+    pkg = Path(__file__).parent
+    d = Path(tmp, "bm"); d.mkdir()
+    hist = str(d / "hist.csv")
+    TelemetryRecorder(engine=DownholeEngine(SimulatorConfig()), config=TelemetryConfig(duration_hours=2.0, seed=5)).run().export_csv(hist)
+    wsp = str(d / "ws"); ws = Workspace.create(wsp, "Pad 3", actor="tester")
+    w = ws.add_well_file(hist, display="SWD 1", actor="tester")
+    site = ws.add_site("Pad 3 site", wells=[w["id"]], actor="tester")
+    bad_period = None
+    try:
+        ME.period_bounds("2026-9")
+    except ValueError as e:
+        bad_period = str(e)
+    pb = ME.period_bounds("2026-02")
+    rec = ME.assemble(ws, "2026-09", actor="tester")
+    idx = json.loads((Path(rec["folder"]) / "INDEX.json").read_text(encoding="utf-8"))
+    man = json.loads((Path(rec["folder"]) / "MANIFEST.json").read_text(encoding="utf-8"))
+    by_name = {i["name"]: i for i in idx["items"]}
+    v_folder = ME.verify(rec["folder"]); v_zip = ME.verify(rec["archive"])
+    with zipfile.ZipFile(rec["archive"]) as z:
+        znames = set(z.namelist())
+    # a second assembly of the same period rebuilds whole; a future month is OPEN
+    rec2 = ME.assemble(ws, "2026-09", actor="tester", refresh=False)
+    from datetime import datetime as _dt, timezone as _tz
+    now = _dt.now(_tz.utc)
+    cur = f"{now.year:04d}-{now.month:02d}"
+    rec3 = ME.assemble(ws, cur, actor="tester", refresh=False)
+    pk = ME.packets(ws)
+    audit = [a["action"] for a in ws.audit_log()]
+    c1 = {
+        "period_checked": bool(bad_period) and "YYYY-MM" in bad_period and pb["days"] == 28 and pb["start"] == "2026-02-01T00:00:00Z" and pb["end"] == "2026-02-28T23:59:59Z" and pb["closed"] is True,
+        "included": by_name["SWD 1: gauge drift report"]["status"] == "INCLUDED" and by_name["SWD 1: alarm event report"]["status"] == "INCLUDED"
+                    and by_name["SLA month 2026-09"]["status"] == "INCLUDED" and by_name["Accuracy statement"]["status"] == "INCLUDED"
+                    and by_name["Pad 3 site: site report"]["status"] == "INCLUDED" and by_name["Model cards and SBOM"]["status"] == "INCLUDED",
+        "not_available_with_reason": by_name["SWD 1: machine vibration"]["status"] == "NOT AVAILABLE" and "no machine record" in by_name["SWD 1: machine vibration"]["reason"]
+                    and by_name["Pad 3 site: SRA packet for 2026-09"]["status"] == "NOT AVAILABLE" and "no Seismicity Response Area" in by_name["Pad 3 site: SRA packet for 2026-09"]["reason"]
+                    and "no certificate register" in by_name["SWD 1: certificate conformity (ISO/IEC 17025 7.8)"]["reason"],
+        "files_hashed": all(f["sha256"] and f["bytes"] > 0 for i in idx["items"] for f in i["files"]) and man["n_files"] == len(man["files"]) and man["n_files"] >= 20
+                    and "wells/SWD-1/gauge_drift_report.html" in man["files"] and "site/sla_report_2026-09.html" in man["files"] and "INDEX.html" in man["files"],
+        "record": rec["included"] == len([i for i in idx["items"] if i["status"] == "INCLUDED"]) and rec["closed"] is True and rec["errors"] == [] and os.path.isfile(rec["archive"])
+                    and os.path.isfile(rec["archive"] + "") and rec["archive"].endswith("month_end_Pad-3_2026-09.zip"),
+        "verify": v_folder["status"] == "OK" and v_zip["status"] == "OK" and v_zip["n_files"] == man["n_files"] and "MANIFEST.json" in znames and "INDEX.html" in znames,
+        "rebuilt_whole_and_open": rec2["generated_utc"] >= rec["generated_utc"] and rec2["refresh"] is False and rec3["closed"] is False and "OPEN" in (Path(rec3["folder"]) / "INDEX.md").read_text(encoding="utf-8")
+                    and len(pk) == 3 and all(p["present"] for p in pk) and pk[-1]["period"] == cur,
+        "audited": audit.count("month_end.assemble") == 3 and "dashboard.refresh" in audit and "site.report" in audit,
+        "not_a_measurement": "does not call a measurement" in (Path(rec["folder"]) / "INDEX.md").read_text(encoding="utf-8"),
+    }
+    failed1 = [k for k, v in c1.items() if not v]
+    ok(not failed1,
+       "BM1 the month-end packet: the period checked (YYYY-MM; February's 28 days; a past month closed), the refresh run for it, every report "
+       "collected - the well's drift and alarm reports, the SLA month, the accuracy statement, the site report, the model cards - INCLUDED with "
+       "hashed files, and the vibration report, the SRA packet and the certificate conformity NOT AVAILABLE with the reason that names what the "
+       "site lacks; INDEX and MANIFEST written, the archive verifies against the manifest, a second assembly rebuilds the period whole, the "
+       "current month is marked OPEN, every packet on record, every assembly in the audit"
+       + (f" [failed: {', '.join(failed1)}]" if failed1 else ""))
+
+    # -- the page, the job, the schedule, the CLI, the help --------------------------------------------------------
+    users = Users(str(Path(wsp, "users.json"))); users.add("op", "operator-pass-1", "operator"); users.add("adm", "admin-pass-1", "admin")
+    svc = Service(wsp, port=0, scheduler=False).start()
+    base = svc.url.rstrip("/")
+    opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(http.cookiejar.CookieJar()))
+
+    def call(method, path, body=None):
+        req = urllib.request.Request(base + path, method=method, data=(json.dumps(body).encode() if body is not None else None))
+        req.add_header("Content-Type", "application/json"); req.add_header("X-GEA-Action", "1")
+        try:
+            with opener.open(req, timeout=300) as rr_:
+                return rr_.status, json.loads(rr_.read())
+        except urllib.error.HTTPError as e:
+            return e.code, json.loads(e.read())
+    try:
+        call("POST", "/api/login", {"name": "op", "password": "operator-pass-1"})
+        st_l, lst = call("GET", "/api/month-end")
+        st_bad = call("POST", "/api/month-end/run", {"period": "Sept 2026"})[0]
+        st_sched_op = call("POST", "/api/month-end/schedule", {"day": 1, "at": "03:00"})[0]
+        st_run, job = call("POST", "/api/month-end/run", {"period": "2026-08", "no_refresh": True})
+        done = svc.app.runner.wait(job["id"], 300)
+        st_l2, lst2 = call("GET", "/api/month-end")
+        req = urllib.request.Request(base + "/reports/month_end/2026-08/INDEX.html"); req.add_header("X-GEA-Action", "1")
+        with opener.open(req, timeout=60) as rr_:
+            html_ok = rr_.status == 200 and b"Month-End Packet" in rr_.read()
+        call("POST", "/api/login", {"name": "adm", "password": "admin-pass-1"})
+        st_sched_bad = call("POST", "/api/month-end/schedule", {"day": 31, "at": "03:00"})[0]
+        st_sched, sch = call("POST", "/api/month-end/schedule", {"day": 2, "at": "03:15"})
+        st_l3, lst3 = call("GET", "/api/month-end")
+    finally:
+        svc.stop()
+    page_src = (pkg / "web" / "app.html").read_text(encoding="utf-8")
+    help_md = (pkg / "help" / "month-end.md").read_text(encoding="utf-8")
+    r_cli = _sp.run([sys.executable, "-m", "gea", "workspace", "--path", wsp, "--action", "month-end", "--period", "2026-07", "--no-refresh"], capture_output=True, text=True)
+    r_prev = _sp.run([sys.executable, "-m", "gea", "workspace", "--path", wsp, "--action", "month-end", "--no-refresh"], capture_output=True, text=True)
+    c2 = {
+        "list": st_l == 200 and len(lst["packets"]) == 3 and lst["previous"] == ME.previous_period() and lst["schedule"] is None,
+        "refusals": st_bad == 400 and st_sched_op == 403 and st_sched_bad == 400,
+        "job": st_run == 200 and done["status"] == "DONE" and st_l2 == 200 and any(p["period"] == "2026-08" and p["present"] for p in lst2["packets"]) and html_ok,
+        "schedule": st_sched == 200 and sch["name"] == "month-end-monthly" and sch["monthly_day"] == 2 and sch["daily_at"] == "03:15" and "previous" in sch["args"]
+                    and st_l3 == 200 and lst3["schedule"]["name"] == "month-end-monthly",
+        "page": "Month-end packet" in page_src and "/api/month-end/run" in page_src and "/api/month-end/schedule" in page_src,
+        "cli": r_cli.returncode == 0 and "month-end 2026-07 (closed)" in r_cli.stdout and r_prev.returncode == 0 and f"month-end {ME.previous_period()} (closed)" in r_prev.stdout,
+        "help": any(t["topic"] == "month-end" for t in H.topics()) and not H.check("month-end") and "does not recompute" in help_md.split("will not call a measurement")[1],
+    }
+    failed2 = [k for k, v in c2.items() if not v]
+    ok(not failed2,
+       "BM2 the Reports page carries the packet card: the packets on record with their index and archive, the period defaulting to the last closed "
+       "month, Assemble as a job (a malformed period refused) whose INDEX the page then serves, Schedule monthly for an administrator only (day 1..28) "
+       "as the `month-end-monthly` entry for the previous month; `gea workspace --action month-end` with a period or with 'previous'; the help page "
+       "says the index collects and does not recompute"
+       + (f" [failed: {', '.join(failed2)}; cli={r_cli.stdout[-200:]!r} {r_cli.stderr[-200:]!r}]" if failed2 else ""))
+
 def main() -> int:
     print("GEA-Program - ACCEPTANCE SUITE (the product gate)")
     # a handle a job subprocess still holds at the end must not turn a finished gate into a traceback on Windows:
@@ -5939,6 +6063,7 @@ def main() -> int:
         section_bj_etp(tmp)
         section_bk_seedlink(tmp)
         section_bl_backup(tmp)
+        section_bm_month_end(tmp)
     if _FAILS:
         print(f"[ACCEPTANCE] {len(_FAILS)} FAILURES ({_PASS} passed):")
         for f in _FAILS:

gea_program-0.19.0/gea/help/month-end.md

@@ -0,0 +1,11 @@
+# The month-end packet
+
+> A client's month was five or six commands and a walk through the reports folder. The packet is that month assembled once: the refresh for the period, then every report the program writes for every well, site, station and track, collected into one dated folder and one archive, with an index that says item by item what is there and what is not, and a manifest of every file's hash.
+
+**The command**: `gea workspace --path C:\site --action month-end --period 2026-09` (or `--period previous`, the last closed month; `--no-refresh` collects the reports as they stand; `--catalog` hands the regulator's catalogue CSV to the SRA packet). On the dashboard: Reports -> Month-end packet -> the period and Assemble (a job), and for an administrator Schedule monthly (a schedule entry, `month-end-monthly`, on the day and hour given, for the previous month). The result is `reports/month_end/<period>/` with `INDEX.html` / `.md` / `.json` and `MANIFEST.json`, and `reports/month_end/month_end_<site>_<period>.zip`; each packet is logged in `records/month_end.jsonl` and in the audit (`month_end.assemble`).
+
+**What it writes**: for the site - the dashboard index, the accuracy statement, the SLA month for that period, the FAT/SAT protocol, the model cards and the SBOM; for every well - the gauge drift report, the alarm event report, the well-test validation, the shut-in / build-up report, the data resilience report, the machine vibration report, and the certificate register's conformity against ISO/IEC 17025 7.8 as it stands; for every site - the SRA packet for the period (when a Seismicity Response Area is defined) and the site report; for every seismic station and track - their reports. Each item is INCLUDED with its files or NOT AVAILABLE with the reason (no machine record, no SRA defined, no certificate register, no well test detected, never refreshed); every step that failed is listed; a month that has not ended is marked OPEN.
+
+**The number to check**: `N INCLUDED, M NOT AVAILABLE, 0 FAILED` on the index's front page, and `closed: yes` for a month that has ended.
+
+**What this page will not call a measurement**: an item that is NOT AVAILABLE - the reason names what the site lacks, and the packet never stands in for it. A number in any of the reports - each carries its own method and declarations; the index collects and does not recompute. A month that has not ended - the packet says OPEN and the one assembled after the month closes supersedes it. A packet whose files no longer match its manifest - `MANIFEST.json` is the check, and a packet is rebuilt whole, never merged.

{gea_program-0.18.0 → gea_program-0.19.0}/gea/helplib.py

@@ -46,6 +46,7 @@ INDEX: List[tuple] = [
     ('conformance', 'Conformity, in the standards\' words', 'certificates against ISO/IEC 17025 7.8 and the bias against its class under a named ILAC-G8 decision rule'),
     ('etp', 'WITSML 2.x over ETP', 'the live port on Energistics ETP v1.2: subscribe to the store\'s channels and it pushes'),
     ('seedlink', 'Live stations (SeedLink)', 'the seismic leg\'s live port: a station\'s records as it writes them, resumed by sequence number, folded into its record list'),
+    ('month-end', 'The month-end packet', 'the month\'s deliverable assembled once: every report collected, INCLUDED or NOT AVAILABLE with the reason, hashed'),
     ('backup', 'The backup', 'the site copied off the machine, verified, and restored once before it is trusted'),
     ('doctor', 'Which code is running', 'gea doctor and the serve start-up gate'),
     ('audit-update', 'Audit / Update', 'the whole audit log with filters, the program against PyPI, every report against its source'),

gea_program-0.19.0/gea/month_end.py

@@ -0,0 +1,290 @@
+"""month_end - the month's deliverable, assembled once, from the reports the program already writes.
+
+A client's month has, until now, been five or six commands and a walk through reports/: the drift report of
+each well, the alarm month, the well-test validations, the vibration report where a machine record exists,
+the certificate conformity, the SRA packet of each site that has one, the site report, the seismic station
+and track reports, the accuracy statement and the SLA month. This module runs the refresh for the period,
+then collects every one of those into one dated folder and one archive - with an index that says, item by
+item, INCLUDED (with the file's hash) or NOT AVAILABLE (with the reason: no machine record, no SRA defined,
+no certificate register, never refreshed) - and a manifest of every file. Nothing is recomputed here beyond
+the refresh the program would run anyway; the packet is what the reports said, collected, so it can be
+handed over and later verified against the files it came from.
+
+What it will not do: make up an item that has no source (it is listed as NOT AVAILABLE with the reason);
+include a report older than the refresh that produced the packet unless told to skip the refresh (and then
+the index says every report's own generated time); name a month that has not ended as closed (it is marked
+OPEN).
+"""
+from __future__ import annotations
+
+import calendar
+import hashlib
+import json
+import os
+import shutil
+import zipfile
+from datetime import datetime, timezone
+from typing import Dict, List, Optional
+
+PROTOCOL = 'gea.month_end/1'
+
+
+def _utc() -> str:
+    return datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
+
+
+def _sha256(path: str) -> str:
+    h = hashlib.sha256()
+    with open(path, 'rb') as f:
+        for chunk in iter(lambda: f.read(1 << 20), b''):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def period_bounds(period: str) -> Dict[str, str]:
+    """'2026-09' -> the first and last instants of that month (UTC) and whether the month has ended."""
+    try:
+        y, m = int(period[:4]), int(period[5:7])
+        if period[4] != '-' or not (1 <= m <= 12) or len(period) != 7:
+            raise ValueError
+    except (ValueError, IndexError):
+        raise ValueError(f'period must be YYYY-MM, got {period!r}')
+    last = calendar.monthrange(y, m)[1]
+    start = f'{y:04d}-{m:02d}-01T00:00:00Z'
+    end = f'{y:04d}-{m:02d}-{last:02d}T23:59:59Z'
+    now = datetime.now(timezone.utc)
+    closed = (y, m) < (now.year, now.month)
+    return {'period': period, 'start': start, 'end': end, 'closed': closed, 'days': last}
+
+
+def previous_period(now: Optional[datetime] = None) -> str:
+    """The last closed month as YYYY-MM - what a monthly schedule entry asks for."""
+    now = now or datetime.now(timezone.utc)
+    y, m = (now.year, now.month - 1) if now.month > 1 else (now.year - 1, 12)
+    return f'{y:04d}-{m:02d}'
+
+
+def _copy_family(src_dir: str, names: List[str], dst_dir: str) -> List[dict]:
+    """Copy the files of one report family (any extension of each basename) and return their entries."""
+    out = []
+    if not os.path.isdir(src_dir):
+        return out
+    for fn in sorted(os.listdir(src_dir)):
+        stem = fn.split('.')[0]
+        if any(fn == n or fn.startswith(n + '.') or fn.startswith(n + '_') for n in names) or stem in names:
+            s = os.path.join(src_dir, fn)
+            if os.path.isfile(s):
+                os.makedirs(dst_dir, exist_ok=True)
+                d = os.path.join(dst_dir, fn)
+                shutil.copyfile(s, d)
+                out.append({'file': fn, 'sha256': _sha256(d), 'bytes': os.path.getsize(d),
+                            'generated_utc': datetime.fromtimestamp(os.path.getmtime(s), timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')})
+    return out
+
+
+def assemble(ws, period: str, actor: str = 'system', refresh: bool = True, catalog_csv: Optional[str] = None) -> dict:
+    """Run the refresh for the period (unless refresh=False), collect every report into reports/month_end/<period>/,
+    write INDEX (markdown, html, json) and MANIFEST.json, zip it, audit it. Returns the packet record."""
+    from . import __version__
+    from .client_reports import Document, Section, Table, write as _write, PROGRAM_NAME
+    pb = period_bounds(period)
+    items: List[dict] = []
+    errors: List[str] = []
+    if refresh:
+        try:
+            ws.refresh_dashboard(actor=actor, month=period)
+        except Exception as e:                                   # the refresh's own failure is an item, never a silent gap
+            errors.append(f'dashboard refresh: {e}')
+        for st in ws.seismic_stations():
+            try:
+                ws.refresh_seismic(st['id'], actor=actor)
+            except Exception as e:
+                errors.append(f"seismic {st['id']}: {e}")
+        for tk in ws.tracks():
+            try:
+                ws.refresh_track(tk['id'], actor=actor)
+            except Exception as e:
+                errors.append(f"track {tk['id']}: {e}")
+    out = ws.dir('reports', 'month_end', period)
+    for old in os.listdir(out):                                  # a packet for a period is rebuilt whole, never merged
+        p = os.path.join(out, old)
+        shutil.rmtree(p) if os.path.isdir(p) else os.remove(p)
+    rep = ws.reports_dir
+
+    def item(kind: str, ident: str, name: str, files: List[dict], reason: str = '') -> None:
+        items.append({'kind': kind, 'id': ident, 'name': name, 'status': 'INCLUDED' if files else 'NOT AVAILABLE',
+                      'reason': '' if files else (reason or 'no report written'), 'files': files})
+
+    # -- the site as a whole ---------------------------------------------------------------------------------------
+    item('site', '-', 'Dashboard index', _copy_family(rep, ['index', 'dashboard'], os.path.join(out, 'site')), 'never refreshed')
+    item('site', '-', 'Accuracy statement', _copy_family(rep, ['accuracy_statement'], os.path.join(out, 'site')), 'no accuracy statement written')
+    item('site', '-', f'SLA month {period}', _copy_family(rep, [f'sla_report_{period}'], os.path.join(out, 'site')),
+         'the SLA month is written by the refresh with --month; none for this period')
+    item('site', '-', 'FAT/SAT protocol', _copy_family(rep, ['fat_protocol', 'sat_protocol'], os.path.join(out, 'site')), 'no protocol written (gea fat-sat, or the refresh with --sat)')
+    item('site', '-', 'Model cards and SBOM', _copy_family(rep, ['model_cards_index', 'sbom'], os.path.join(out, 'site')), 'never refreshed')
+    # -- every well -------------------------------------------------------------------------------------------------
+    for w in ws.wells():
+        wd = os.path.join(rep, 'wells', w['id'])
+        dst = os.path.join(out, 'wells', w['id'])
+        item('well', w['id'], f"{w['display']}: gauge drift report", _copy_family(wd, ['gauge_drift_report'], dst), 'never refreshed')
+        item('well', w['id'], f"{w['display']}: alarm event report", _copy_family(wd, ['alarm_event_report'], dst), 'no alarm report (no alarm config, or never refreshed)')
+        item('well', w['id'], f"{w['display']}: well-test validation", _copy_family(wd, ['well_test_validation'], dst), 'no well test detected in the record, or never refreshed')
+        item('well', w['id'], f"{w['display']}: shut-in / build-up", _copy_family(wd, ['pressure_transient_report'], dst), 'no shut-in found in the record, or never refreshed')
+        item('well', w['id'], f"{w['display']}: data resilience", _copy_family(wd, ['data_resilience_report'], dst), 'no store-and-forward outage was given to the refresh')
+        item('well', w['id'], f"{w['display']}: machine vibration", _copy_family(wd, ['vibration_report'], dst),
+             'no machine record for this well (gea workspace --action vibration-report)')
+        # certificate conformity: the register's standing, written here from the register as it stands
+        reg = os.path.join(ws.path, 'wells', w['id'], 'records', 'certificates.jsonl')
+        if os.path.isfile(reg):
+            try:
+                from .certificates import CertificateRegister
+                conf = CertificateRegister(reg).conformance()
+                os.makedirs(dst, exist_ok=True)
+                p = os.path.join(dst, 'certificate_conformance.json')
+                with open(p, 'w', encoding='utf-8') as f:
+                    json.dump(conf, f, indent=1, default=str)
+                item('well', w['id'], f"{w['display']}: certificate conformity (ISO/IEC 17025 7.8)",
+                     [{'file': 'certificate_conformance.json', 'sha256': _sha256(p), 'bytes': os.path.getsize(p), 'generated_utc': _utc()}])
+            except Exception as e:
+                item('well', w['id'], f"{w['display']}: certificate conformity (ISO/IEC 17025 7.8)", [], f'register could not be read: {e}')
+        else:
+            item('well', w['id'], f"{w['display']}: certificate conformity (ISO/IEC 17025 7.8)", [], 'no certificate register for this well')
+    # -- every site: the SRA packet and the site report ---------------------------------------------------------------
+    for st in ws.sites():
+        sd = os.path.join(rep, 'sites', st['id'])
+        dst = os.path.join(out, 'sites', st['id'])
+        if st.get('sra'):
+            try:
+                ws.write_sra_packet(st['id'], catalog_csv=catalog_csv, start=pb['start'], end=pb['end'], actor=actor)
+            except Exception as e:
+                errors.append(f"sra {st['id']}: {e}")
+            item('site-sra', st['id'], f"{st['display']}: SRA packet for {period}", _copy_family(sd, ['sra_packet'], dst), 'the SRA packet failed to write (see errors)')
+        else:
+            item('site-sra', st['id'], f"{st['display']}: SRA packet for {period}", [], 'no Seismicity Response Area defined for this site (gea workspace --action sra-define)')
+        try:
+            ws.write_site_report(st['id'], actor=actor)
+        except Exception as e:
+            errors.append(f"site report {st['id']}: {e}")
+        item('site-report', st['id'], f"{st['display']}: site report", _copy_family(sd, ['site_report', 'association', 'quakeml'], dst), 'the site report failed to write')
+    # -- seismic stations and tracks --------------------------------------------------------------------------------
+    for s in ws.seismic_stations():
+        item('seismic', s['id'], f"{s['display']}: seismic station report",
+             _copy_family(os.path.join(rep, 'seismic', s['id']), ['seismic_station_report', 'summary', 'detect', 'lines'], os.path.join(out, 'seismic', s['id'])), 'never refreshed')
+    for t in ws.tracks():
+        item('track', t['id'], f"{t['display']}: seismic track report",
+             _copy_family(os.path.join(rep, 'tracks', t['id']), ['seismic_track_report', 'summary'], os.path.join(out, 'tracks', t['id'])), 'never refreshed')
+    # -- the index and the manifest ----------------------------------------------------------------------------------
+    inc = [i for i in items if i['status'] == 'INCLUDED']
+    na = [i for i in items if i['status'] != 'INCLUDED']
+    now = _utc()
+    report_id = f"MONTH-{period}-{now.replace('-', '').replace(':', '')}"
+    rows = [[i['kind'], i['id'], i['name'], i['status'], i['reason'] or ', '.join(f['file'] for f in i['files'])] for i in items]
+    secs = [Section('1', 'The month', [
+        f"{ws.manifest['name']}: the deliverable for {period} ({pb['start'][:10]} to {pb['end'][:10]}, {pb['days']} days), "
+        f"{'a closed month' if pb['closed'] else 'an OPEN month - the period has not ended, and this packet will be superseded'}. "
+        f"{len(inc)} item(s) included, {len(na)} not available. "
+        + ('The refresh ran before collection, so every report is from the sources as they stand now. ' if refresh else
+           'The refresh was skipped: each report carries its own generated time below, which may predate this packet. ')
+        + (f"{len(errors)} step(s) failed and are listed in section 3." if errors else 'No step failed.')]),
+        Section('2', 'What is in the packet', ['Every report the program writes for a well, a site, a station or a track, collected as written - '
+                                               'INCLUDED with the file names, or NOT AVAILABLE with the reason. Hashes are in MANIFEST.json.'],
+                [Table(['Kind', 'Id', 'Item', 'Status', 'Files / reason'], rows)]),
+        Section('3', 'What failed', errors or ['Nothing.']),
+        Section('4', 'What this packet does not call a measurement',
+                ['An item that is NOT AVAILABLE - the reason names what the site lacks, and the packet does not stand in for it.',
+                 'A number in any report - each report carries its own method and its own declarations; this index collects, it does not recompute.',
+                 'A month that has not ended - the packet says OPEN and is superseded by the one made after the month closes.'])]
+    front = [['Report ID', report_id], ['Site', ws.manifest['name']], ['Period', period], ['Closed', 'yes' if pb['closed'] else 'NO - open month'],
+             ['Generated (UTC)', now], ['Program', f'{PROGRAM_NAME} build {__version__}'], ['Result', f'{len(inc)} INCLUDED, {len(na)} NOT AVAILABLE, {len(errors)} FAILED']]
+    doc = Document(title=f'{PROGRAM_NAME} - Month-End Packet', report_id=report_id, front=front, sections=secs,
+                   footer='Collected from the reports as written; nothing is recomputed in this index.',
+                   data={'protocol': PROTOCOL, 'report_id': report_id, 'site': ws.manifest['name'], 'period': pb, 'generated_utc': now,
+                         'program_version': __version__, 'refresh': refresh, 'items': items, 'errors': errors, '_records': [], '_catalogue_obj': None})
+    paths = _write(doc, out, 'INDEX')
+    # the manifest: every file in the packet folder
+    files = {}
+    for root, _, fns in os.walk(out):
+        for fn in sorted(fns):
+            if fn == 'MANIFEST.json':
+                continue
+            p = os.path.join(root, fn)
+            rel = os.path.relpath(p, out).replace(os.sep, '/')
+            files[rel] = {'sha256': _sha256(p), 'bytes': os.path.getsize(p)}
+    manifest = {'protocol': PROTOCOL, 'site': ws.manifest['name'], 'period': period, 'closed': pb['closed'], 'generated_utc': now, 'program_version': __version__,
+                'report_id': report_id, 'n_files': len(files), 'files': files, 'included': len(inc), 'not_available': len(na), 'errors': errors}
+    with open(os.path.join(out, 'MANIFEST.json'), 'w', encoding='utf-8') as f:
+        json.dump(manifest, f, indent=1)
+    slug = ''.join(c if c.isalnum() or c in '-_' else '-' for c in ws.manifest['name']).strip('-') or 'site'
+    zpath = os.path.join(ws.dir('reports', 'month_end'), f'month_end_{slug}_{period}.zip')
+    with zipfile.ZipFile(zpath + '.part', 'w', compression=zipfile.ZIP_DEFLATED) as z:
+        for root, _, fns in os.walk(out):
+            for fn in sorted(fns):
+                p = os.path.join(root, fn)
+                z.write(p, os.path.relpath(p, out).replace(os.sep, '/'))
+    os.replace(zpath + '.part', zpath)
+    rec = {'period': period, 'closed': pb['closed'], 'folder': out, 'archive': zpath, 'archive_sha256': _sha256(zpath), 'index': paths['html'], 'report_id': report_id,
+           'included': len(inc), 'not_available': len(na), 'errors': errors, 'n_files': len(files), 'generated_utc': now, 'refresh': refresh}
+    with open(os.path.join(ws.dir('records'), 'month_end.jsonl'), 'a', encoding='utf-8') as f:
+        f.write(json.dumps(rec) + '\n')
+    ws.audit(actor, 'month_end.assemble', {k: rec[k] for k in ('period', 'closed', 'archive', 'archive_sha256', 'included', 'not_available', 'n_files')}
+             | ({'errors': errors} if errors else {}))
+    return rec
+
+
+def packets(ws) -> List[dict]:
+    """Every packet assembled from this workspace, newest last, with whether its archive is still there."""
+    p = os.path.join(ws.path, 'records', 'month_end.jsonl')
+    out = []
+    if os.path.isfile(p):
+        with open(p, encoding='utf-8') as f:
+            for line in f:
+                if line.strip():
+                    try:
+                        r = json.loads(line)
+                    except ValueError:
+                        continue
+                    r['present'] = os.path.isfile(r.get('archive', ''))
+                    out.append(r)
+    return out
+
+
+def verify(folder_or_zip: str) -> dict:
+    """A packet folder or its archive against its MANIFEST.json."""
+    out = {'path': folder_or_zip, 'status': 'OK', 'n_files': 0, 'mismatched': [], 'missing': []}
+    if zipfile.is_zipfile(folder_or_zip):
+        with zipfile.ZipFile(folder_or_zip) as z:
+            if 'MANIFEST.json' not in z.namelist():
+                return {**out, 'status': 'FAIL', 'error': 'no MANIFEST.json'}
+            man = json.loads(z.read('MANIFEST.json'))
+            names = set(z.namelist())
+            for rel, meta in man['files'].items():
+                if rel not in names:
+                    out['missing'].append(rel); continue
+                if hashlib.sha256(z.read(rel)).hexdigest() != meta['sha256']:
+                    out['mismatched'].append(rel)
+                out['n_files'] += 1
+    else:
+        mp = os.path.join(folder_or_zip, 'MANIFEST.json')
+        if not os.path.isfile(mp):
+            return {**out, 'status': 'FAIL', 'error': 'no MANIFEST.json'}
+        man = json.load(open(mp, encoding='utf-8'))
+        for rel, meta in man['files'].items():
+            p = os.path.join(folder_or_zip, *rel.split('/'))
+            if not os.path.isfile(p):
+                out['missing'].append(rel); continue
+            if _sha256(p) != meta['sha256']:
+                out['mismatched'].append(rel)
+            out['n_files'] += 1
+    out['period'] = man.get('period'); out['site'] = man.get('site'); out['generated_utc'] = man.get('generated_utc')
+    if out['mismatched'] or out['missing']:
+        out['status'] = 'FAIL'
+    return out
+
+
+def report_text(rec: dict) -> str:
+    lines = [f"month-end {rec['period']} ({'closed' if rec['closed'] else 'OPEN month'}): {rec['included']} included, {rec['not_available']} not available, "
+             f"{len(rec['errors'])} failed; {rec['n_files']} files",
+             f"  folder:  {rec['folder']}", f"  archive: {rec['archive']}  sha256 {rec['archive_sha256'][:16]}...", f"  index:   {rec['index']}"]
+    for e in rec['errors']:
+        lines.append(f"  FAILED: {e}")
+    return '\n'.join(lines)

{gea_program-0.18.0 → gea_program-0.19.0}/gea/service.py

@@ -1602,6 +1602,10 @@ def make_handler(app: App):
                     return self._json(200, {'entries': app.catalog()})
                 if path == '/api/reports':
                     return self._json(200, app.reports_index())
+                if path == '/api/month-end':
+                    from . import month_end as ME
+                    sched = next((e for e in app.scheduler.entries() if e['name'] == 'month-end-monthly'), None)
+                    return self._json(200, {'packets': ME.packets(app.ws), 'previous': ME.previous_period(), 'schedule': sched})
                 if path == '/api/jobs':
                     return self._json(200, {'jobs': app.runner.list(int(qs.get('limit', ['50'])[0]))})
                 if path.startswith('/api/jobs/') and path.endswith('/log'):
@@ -1799,6 +1803,29 @@ def make_handler(app: App):
                                             'detail': ('the launcher will start the control panel again by itself if it stops unexpectedly'
                                                        if on else
                                                        'the launcher will hand the window to a PowerShell prompt when the control panel stops')})
+                if path == '/api/month-end/run':
+                    App.require(user, 'operator')
+                    from . import month_end as ME
+                    period = str(body.get('period') or 'previous')
+                    if period != 'previous':
+                        try:
+                            ME.period_bounds(period)
+                        except ValueError as e:
+                            raise ApiError(400, str(e))
+                    args = ['workspace', '--path', app.ws.path, '--action', 'month-end', '--period', period, '--actor', user['name']]
+                    if body.get('no_refresh'):
+                        args.append('--no-refresh')
+                    return self._json(200, app._job(args, user, f'month-end packet {period}'))
+                if path == '/api/month-end/schedule':
+                    App.require(user, 'admin')
+                    try:
+                        day = int(body.get('day', 1)); at = str(body.get('at') or '03:00')
+                    except (TypeError, ValueError):
+                        raise ApiError(400, 'day is a whole number 1..28')
+                    if not (1 <= day <= 28) or len(at) != 5 or at[2] != ':' or not (at[:2] + at[3:]).isdigit():
+                        raise ApiError(400, 'day is 1..28 and at is HH:MM')
+                    args = ['workspace', '--path', app.ws.path, '--action', 'month-end', '--period', 'previous', '--actor', 'schedule']
+                    return self._json(200, app.scheduler.add('month-end-monthly', args, user['name'], daily_at=at, monthly_day=day))
                 if path == '/api/backup/config':
                     App.require(user, 'admin')
                     return self._json(200, app.backup_set_config(user, body))