chellow 1727274025.0.0__tar.gz → 1728658886.0.0__tar.gz

{chellow-1727274025.0.0 → chellow-1728658886.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: chellow
-Version: 1727274025.0.0
+Version: 1728658886.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)

chellow-1728658886.0.0/VERSION

@@ -0,0 +1 @@
+__version__ = "1728658886.0.0"

{chellow-1727274025.0.0 → chellow-1728658886.0.0}/chellow/__init__.py

@@ -25,6 +25,7 @@ import chellow.e.views
 import chellow.gas.cv
 import chellow.gas.views
 import chellow.national_grid
+import chellow.rrun
 import chellow.testing
 from chellow.models import (
     Contract,
@@ -60,6 +61,7 @@ def get_importer_modules():
         chellow.e.bmarketidx,
         chellow.national_grid,
         chellow.rate_server,
+        chellow.rrun,
     )
 
 

{chellow-1727274025.0.0 → chellow-1728658886.0.0}/chellow/models.py

@@ -6401,6 +6401,14 @@ class ReportRun(Base, PersistentClass):
         sess.flush()
         return report_run
 
+    @staticmethod
+    def w_insert(name, user_id, title, data):
+        with Session() as wsess:
+            user = User.get_by_id(wsess, user_id)
+            report_run = ReportRun.insert(wsess, name, user, title, data)
+            wsess.commit()
+            return report_run.id
+
     @staticmethod
     def w_update(report_run_id, state):
         with Session() as wsess:

{chellow-1727274025.0.0 → chellow-1728658886.0.0}/chellow/reports/report_247.py

@@ -24,6 +24,7 @@ from chellow.models import (
     Era,
     MeasurementRequirement,
     RSession,
+    ReportRun,
     Scenario,
     Site,
     SiteEra,
@@ -122,8 +123,10 @@ def _process_site(
     now,
     summary_titles,
     title_dict,
-    era_rows,
+    org_rows,
     site_rows,
+    era_rows,
+    normal_reads,
     data_source_bill,
 ):
     scenario_hh = scenario_props.get("hh_data", {})
@@ -147,7 +150,6 @@ def _process_site(
 
     site_category = None
     site_sources = set()
-    normal_reads = set()
     site_month_data = defaultdict(int)
     for i, (order, imp_mpan_core, exp_mpan_core, imp_ss, exp_ss) in enumerate(
         sorted(calcs, key=str)
@@ -155,8 +157,6 @@ def _process_site(
         if imp_mpan_core == "displaced":
             month_data = {}
             for sname in (
-                "import-grid",
-                "export-grid",
                 "import-gen",
                 "export-gen",
                 "import-3rd-party",
@@ -169,7 +169,11 @@ def _process_site(
                 for xname in ("kwh", "net-gbp"):
                     month_data[f"{sname}-{xname}"] = 0
             month_data["billed-import-kwh"] = 0
+            month_data["import-grid-kwh"] = 0
+            month_data["export-grid-kwh"] = 0
             for suf in ("net-gbp", "vat-gbp", "gross-gbp"):
+                month_data[f"import-grid-{suf}"] = 0
+                month_data[f"export-grid-{suf}"] = 0
                 month_data[f"billed-import-{suf}"] = 0
                 month_data[f"billed-supplier-import-{suf}"] = None
                 month_data[f"billed-dc-import-{suf}"] = None
@@ -240,8 +244,6 @@ def _process_site(
             site_sources.add(source_code)
             month_data = {}
             for name in (
-                "import-grid",
-                "export-grid",
                 "import-gen",
                 "export-gen",
                 "import-3rd-party",
@@ -255,7 +257,9 @@ def _process_site(
                     month_data[f"{name}-{sname}"] = 0
             for polarity in ("import", "export"):
                 month_data[f"billed-{polarity}-kwh"] = 0
+                month_data[f"{polarity}-grid-kwh"] = 0
                 for suf in ("net-gbp", "vat-gbp", "gross-gbp"):
+                    month_data[f"{polarity}-grid-{suf}"] = 0
                     month_data[f"billed-{polarity}-{suf}"] = 0
                     month_data[f"billed-supplier-{polarity}-{suf}"] = 0
                     month_data[f"billed-dc-{polarity}-{suf}"] = 0
@@ -267,28 +271,32 @@ def _process_site(
                 kwh = sum(hh["msp-kwh"] for hh in imp_ss.hh_data)
                 imp_supplier_bill = imp_ss.supplier_bill
 
-                gbp = imp_supplier_bill.get("net-gbp", 0)
+                net_gbp = imp_supplier_bill.get("net-gbp", 0)
+                vat_gbp = imp_supplier_bill.get("vat-gbp", 0)
+                gross_gbp = imp_supplier_bill.get("gross-gbp", 0)
 
                 if source_code in ("grid", "gen-grid"):
-                    month_data["import-grid-net-gbp"] += gbp
+                    month_data["import-grid-net-gbp"] += net_gbp
+                    month_data["import-grid-vat-gbp"] += vat_gbp
+                    month_data["import-grid-gross-gbp"] += gross_gbp
                     month_data["import-grid-kwh"] += kwh
-                    month_data["used-net-gbp"] += gbp
+                    month_data["used-net-gbp"] += net_gbp
                     month_data["used-kwh"] += kwh
                     if source_code == "gen-grid":
                         month_data["export-gen-kwh"] += kwh
                 elif source_code == "3rd-party":
-                    month_data["import-3rd-party-net-gbp"] += gbp
+                    month_data["import-3rd-party-net-gbp"] += net_gbp
                     month_data["import-3rd-party-kwh"] += kwh
-                    month_data["used-3rd-party-net-gbp"] += gbp
+                    month_data["used-3rd-party-net-gbp"] += net_gbp
                     month_data["used-3rd-party-kwh"] += kwh
-                    month_data["used-net-gbp"] += gbp
+                    month_data["used-net-gbp"] += net_gbp
                     month_data["used-kwh"] += kwh
                 elif source_code == "3rd-party-reverse":
-                    month_data["export-3rd-party-net-gbp"] += gbp
+                    month_data["export-3rd-party-net-gbp"] += net_gbp
                     month_data["export-3rd-party-kwh"] += kwh
-                    month_data["used-3rd-party-net-gbp"] -= gbp
+                    month_data["used-3rd-party-net-gbp"] -= net_gbp
                     month_data["used-3rd-party-kwh"] -= kwh
-                    month_data["used-net-gbp"] -= gbp
+                    month_data["used-net-gbp"] -= net_gbp
                     month_data["used-kwh"] -= kwh
                 elif source_code == "gen":
                     month_data["import-gen-kwh"] += kwh
@@ -299,27 +307,31 @@ def _process_site(
                 kwh = sum(hh["msp-kwh"] for hh in exp_ss.hh_data)
                 exp_supplier_bill = exp_ss.supplier_bill
 
-                gbp = exp_supplier_bill.get("net-gbp", 0)
+                net_gbp = exp_supplier_bill.get("net-gbp", 0)
+                vat_gbp = exp_supplier_bill.get("vat-gbp", 0)
+                gross_gbp = exp_supplier_bill.get("gross-gbp", 0)
 
                 if source_code in ("grid", "gen-grid"):
-                    month_data["export-grid-net-gbp"] += gbp
+                    month_data["export-grid-net-gbp"] += net_gbp
+                    month_data["export-grid-vat-gbp"] += vat_gbp
+                    month_data["export-grid-gross-gbp"] += gross_gbp
                     month_data["export-grid-kwh"] += kwh
                     if source_code == "gen-grid":
                         month_data["import-gen-kwh"] += kwh
 
                 elif source_code == "3rd-party":
-                    month_data["export-3rd-party-net-gbp"] += gbp
+                    month_data["export-3rd-party-net-gbp"] += net_gbp
                     month_data["export-3rd-party-kwh"] += kwh
-                    month_data["used-3rd-party-net-gbp"] -= gbp
+                    month_data["used-3rd-party-net-gbp"] -= net_gbp
                     month_data["used-3rd-party-kwh"] -= kwh
-                    month_data["used-net-gbp"] -= gbp
+                    month_data["used-net-gbp"] -= net_gbp
                     month_data["used-kwh"] -= kwh
                 elif source_code == "3rd-party-reverse":
-                    month_data["import-3rd-party-net-gbp"] += gbp
+                    month_data["import-3rd-party-net-gbp"] += net_gbp
                     month_data["import-3rd-party-kwh"] += kwh
-                    month_data["used-3rd-party-net-gbp"] += gbp
+                    month_data["used-3rd-party-net-gbp"] += net_gbp
                     month_data["used-3rd-party-kwh"] += kwh
-                    month_data["used-net-gbp"] += gbp
+                    month_data["used-net-gbp"] += net_gbp
                     month_data["used-kwh"] += kwh
                 elif source_code == "gen":
                     month_data["export-gen-kwh"] += kwh
@@ -493,8 +505,6 @@ def _process_site(
                 if len(bills) > 0:
                     month_data = {}
                     for name in (
-                        "import-grid",
-                        "export-grid",
                         "import-gen",
                         "export-gen",
                         "import-3rd-party",
@@ -507,7 +517,11 @@ def _process_site(
                         for sname in ("kwh", "net-gbp"):
                             month_data[f"{name}-{sname}"] = 0
                     month_data["billed-import-kwh"] = 0
+                    month_data["import-grid-kwh"] = 0
+                    month_data["export-grid-kwh"] = 0
                     for suf in ("net-gbp", "vat-gbp", "gross-gbp"):
+                        month_data[f"import-grid-{suf}"] = 0
+                        month_data[f"export-grid-{suf}"] = 0
                         month_data[f"billed-import-{suf}"] = 0
                         month_data[f"billed-supplier-import-{suf}"] = 0
                         month_data[f"billed-dc-import-{suf}"] = 0
@@ -610,8 +624,6 @@ def _process_site(
                 if len(bills) > 0:
                     month_data = {}
                     for name in (
-                        "import-grid",
-                        "export-grid",
                         "import-gen",
                         "export-gen",
                         "import-3rd-party",
@@ -624,8 +636,12 @@ def _process_site(
                         for sname in ("kwh", "net-gbp"):
                             month_data[f"{name}-{sname}"] = 0
                     month_data["billed-import-kwh"] = 0
+                    month_data["import-grid-kwh"] = 0
+                    month_data["export-grid-kwh"] = 0
                     for suf in ("net-gbp", "vat-gbp", "gross-gbp"):
                         month_data[f"billed-import-{suf}"] = 0
+                        month_data[f"import-grid-{suf}"] = 0
+                        month_data[f"export-grid-{suf}"] = 0
                         month_data[f"billed-supplier-import-{suf}"] = 0
                         month_data[f"billed-dc-import-{suf}"] = 0
                         month_data[f"billed-mop-import-{suf}"] = 0
@@ -708,7 +724,7 @@ def _process_site(
     ] + [site_month_data[k] for k in summary_titles]
 
     site_rows.append([make_val(v) for v in site_row])
-    return normal_reads
+    return site_month_data
 
 
 class Object:
@@ -734,6 +750,7 @@ def content(scenario_props, base_name, user_id, compression, now):
         ind_cont = report_context["contract_names"] = {}
 
     sess = rf = None
+    org_rows = []
     site_rows = []
     era_rows = []
     normal_read_rows = []
@@ -806,7 +823,8 @@ def content(scenario_props, base_name, user_id, compression, now):
 
             user = User.get_by_id(sess, user_id)
 
-            rf = open_file("_".join(base_name) + ".ods", user, mode="wb")
+            fname = "_".join(base_name) + ".ods"
+            rf = open_file(fname, user, mode="wb")
 
             for rate_script in scenario_props.get("rates", []):
                 contract_id = rate_script["contract_id"]
@@ -858,7 +876,21 @@ def content(scenario_props, base_name, user_id, compression, now):
                     )
 
             by_hh = scenario_props.get("by_hh", False)
+            org_header_titles = [
+                "creation-date",
+                "month",
+            ]
 
+            site_header_titles = [
+                "creation-date",
+                "site-id",
+                "site-name",
+                "associated-site-ids",
+                "month",
+                "metering-type",
+                "sources",
+                "generator-types",
+            ]
             era_header_titles = [
                 "creation-date",
                 "imp-mpan-core",
@@ -888,16 +920,6 @@ def content(scenario_props, base_name, user_id, compression, now):
                 "associated-site-ids",
                 "month",
             ]
-            site_header_titles = [
-                "creation-date",
-                "site-id",
-                "site-name",
-                "associated-site-ids",
-                "month",
-                "metering-type",
-                "sources",
-                "generator-types",
-            ]
             summary_titles = [
                 "import-grid-kwh",
                 "export-grid-kwh",
@@ -909,7 +931,11 @@ def content(scenario_props, base_name, user_id, compression, now):
                 "used-kwh",
                 "used-3rd-party-kwh",
                 "import-grid-net-gbp",
+                "import-grid-vat-gbp",
+                "import-grid-gross-gbp",
                 "export-grid-net-gbp",
+                "export-grid-vat-gbp",
+                "export-grid-gross-gbp",
                 "import-gen-net-gbp",
                 "export-gen-net-gbp",
                 "import-3rd-party-net-gbp",
@@ -1003,6 +1029,7 @@ def content(scenario_props, base_name, user_id, compression, now):
                 + [None]
                 + ["exp-supplier-" + t for t in title_dict["exp-supplier"]]
             )
+            org_rows.append(org_header_titles + summary_titles)
             site_rows.append(site_header_titles + summary_titles)
             era_rows.append(era_titles)
 
@@ -1016,6 +1043,7 @@ def content(scenario_props, base_name, user_id, compression, now):
                     data_source_bill.finish_date = month_finish
                 else:
                     data_source_bill = None
+                org_month_data = defaultdict(int)
                 for site in sites:
                     if by_hh:
                         sf = [
@@ -1027,7 +1055,7 @@ def content(scenario_props, base_name, user_id, compression, now):
 
                     for start, finish in sf:
                         try:
-                            normal_reads = normal_reads | _process_site(
+                            site_month_data = _process_site(
                                 sess,
                                 report_context,
                                 forecast_from,
@@ -1039,10 +1067,16 @@ def content(scenario_props, base_name, user_id, compression, now):
                                 now,
                                 summary_titles,
                                 title_dict,
-                                era_rows,
+                                org_rows,
                                 site_rows,
+                                era_rows,
+                                normal_reads,
                                 data_source_bill,
                             )
+                            for k, v in site_month_data.items():
+                                if v is not None:
+                                    org_month_data[k] += v
+
                         except BadRequest as e:
                             raise BadRequest(f"Site Code {site.code}: {e.description}")
 
@@ -1053,9 +1087,29 @@ def content(scenario_props, base_name, user_id, compression, now):
                     row = [mpan_core, r.date, r.msn, r.type] + list(r.reads)
                     normal_read_rows.append(row)
 
+                org_row = [now, month_start] + [
+                    org_month_data[k] for k in summary_titles
+                ]
+                org_rows.append([make_val(v) for v in org_row])
+
                 write_spreadsheet(
                     rf, compression, site_rows, era_rows, normal_read_rows
                 )
+        if scenario_props.get("save_report_run", False):
+            report_run_id = ReportRun.w_insert(
+                "monthly_duration", user_id, fname, {"scenario": scenario_props}
+            )
+            for tab, rows in (
+                ("org", org_rows),
+                ("site", site_rows),
+                ("era", era_rows),
+            ):
+                titles = rows[0]
+                for row in rows[1:]:
+                    values = dict(zip(titles, row))
+                    ReportRun.w_insert_row(report_run_id, tab, titles, values, {})
+            ReportRun.w_update(report_run_id, "finished")
+
     except BadRequest as e:
         msg = e.description + traceback.format_exc()
         sys.stderr.write(msg + "\n")

chellow-1728658886.0.0/chellow/rrun.py

@@ -0,0 +1,77 @@
+import atexit
+import collections
+import threading
+import traceback
+
+from dateutil.relativedelta import relativedelta
+
+from sqlalchemy import select
+
+from chellow.models import ReportRun, Session
+from chellow.utils import utc_datetime_now
+
+rrun_deleter = None
+
+
+def startup():
+    global rrun_deleter
+
+    rrun_deleter = ReportRunDeleter()
+    rrun_deleter.start()
+
+
+class ReportRunDeleter(threading.Thread):
+    def __init__(self):
+        super(ReportRunDeleter, self).__init__(name="ReportRun Deleter")
+        self.messages = collections.deque(maxlen=1000)
+        self.stopped = threading.Event()
+        self.going = threading.Event()
+        self.global_alert = None
+
+    def stop(self):
+        self.stopped.set()
+        self.going.set()
+        self.join()
+
+    def go(self):
+        self.going.set()
+
+    def log(self, message):
+        self.messages.appendleft(
+            utc_datetime_now().strftime("%Y-%m-%d %H:%M:%S") + " - " + message
+        )
+
+    def run(self):
+        while not self.stopped.isSet():
+            try:
+                with Session() as sess:
+                    now = utc_datetime_now()
+                    cutoff_date = now - relativedelta(years=1)
+                    for report_run in sess.scalars(
+                        select(ReportRun).where(ReportRun.date_created < cutoff_date)
+                    ):
+                        report_run.delete(sess)
+                        sess.commit()
+                        self.log("Deleted report run")
+            except BaseException:
+                self.log("Outer problem " + traceback.format_exc())
+                self.global_alert = (
+                    "There's a problem with a "
+                    "<a href='/report_runs'>Report Runs</a>."
+                )
+            finally:
+                self.going.clear()
+                self.log("Finished deleting Report Runs.")
+
+            self.going.wait(24 * 60 * 60)
+            self.going.clear()
+
+
+def get_importer():
+    return rrun_deleter
+
+
+@atexit.register
+def shutdown():
+    if rrun_deleter is not None:
+        rrun_deleter.stop()

{chellow-1727274025.0.0 → chellow-1728658886.0.0}/chellow/templates/e/scenario_docs.html

@@ -137,6 +137,8 @@
     }
   }
   "is_bill_check": false,  /* Use the bill check rates */
+
+  "save_report_run": false,  /* Save the report as a Report Run */
 }</pre>
 
 <h3>HH Data</h3>

chellow-1728658886.0.0/chellow/templates/report_run_monthly_duration_org.html

@@ -0,0 +1,73 @@
+{% extends "base.html" %}
+
+{% block title %}
+	» Report Runs » {{run.id}}
+{% endblock %}
+
+{% block nav %}
+	<a href="/report_runs">Report Runs</a> &raquo; {{run.id}}
+{% endblock %}
+
+{% block content %}
+	{% if request.method == "GET" and request.values.delete %}
+		<form method="post" action="/report_runs/{{run.id}}">
+			<fieldset>
+				<legend>Are you sure you want to delete this report run?</legend>
+				<input type="submit" name="delete" value="delete"> 
+				<a href="/report_runs/{{run.id}}">Cancel</a>
+			</fieldset>
+		</form>
+	{% else %}
+		<table>
+			<caption>Monthly Duration Report</caption>
+			<thead>
+				<tr>
+					<th>Date Created</th>
+					<th>Created By</th>
+					<th>Title</th>
+					<th>State</th>
+					<th>Delete</th>
+					<th>Download Spreadsheet</th>
+				</tr>
+			</thead>
+			<tbody>
+				<tr>
+					<td>{{run.date_created|hh_format}}</td>
+					<td>{{run.creator}}</td>
+					<td>{{run.title}}</td>
+					<td>{{run.state}}</td>
+					<td>
+						<form action="/report_runs/{{run.id}}">
+							<fieldset style="border: none;">
+								<input type="submit" name="delete" value="Delete">
+							</fieldset>
+						</form>
+					</td>
+					<td><a href="/report_runs/{{run.id}}/spreadsheet">Download</a></td>
+				</tr>
+			</tbody>
+		</table>
+
+		<table class="sticky">
+			<caption>
+				Organisation Level
+			</caption>
+			<thead>
+				<tr>
+					{% for title in org_rows[0].data.titles %}
+						<th>{{title}}</th>
+					{% endfor %}
+				</tr>
+			</thead>
+			<tbody>
+				{% for org_row in org_rows %}
+					<tr>
+						{% for title in org_row.data.titles %}
+							<td>{{org_row.data['values'][title]}}</td>
+						{% endfor %}
+					</tr>
+				{% endfor %}
+			</tbody>
+		</table>
+	{% endif %}
+{% endblock %}

{chellow-1727274025.0.0 → chellow-1728658886.0.0}/chellow/templates/report_runs.html

@@ -35,4 +35,35 @@
 			{% endfor %}
 		</tbody>
 	</table>
+
+	<h2>Report Run Deleter</h2>
+
+	{% if importer %}
+
+		<table>
+			<tr>
+				<th>Is Going?</th>
+				<td>{{importer.going.is_set()}}</td>
+			</tr>
+			{% if importer.going.is_set() %}
+				<tr>
+					<th>Progress</th>
+					<td>{{importer.progress}}</td>
+				</tr>
+			{% endif %}
+		</table>
+
+
+		<h4 id="log">Log</h4>
+
+		<ul>
+			{% for message in importer.messages %}
+				<li>{{message}}</li>
+			{% endfor %}
+		</ul>
+
+
+	{% else %}
+		<p>Deleter not present.</p>
+	{% endif %}
 {% endblock %}

{chellow-1727274025.0.0 → chellow-1728658886.0.0}/chellow/views.py

@@ -1287,8 +1287,9 @@ def download_delete(fname):
 @home.route("/report_runs")
 def report_runs_get():
     runs = g.sess.query(ReportRun).order_by(ReportRun.date_created.desc())
+    importer = chellow.rrun.get_importer()
 
-    return render_template("report_runs.html", runs=runs)
+    return render_template("report_runs.html", runs=runs, importer=importer)
 
 
 @home.route("/report_runs/<int:run_id>")
@@ -1375,6 +1376,21 @@ def report_run_get(run_id):
             run=run,
             rows=rows,
         )
+    elif run.name == "monthly_duration":
+        org_rows = (
+            g.sess.execute(
+                select(ReportRunRow)
+                .filter(ReportRunRow.report_run == run, ReportRunRow.tab == "org")
+                .order_by(ReportRunRow.data["values"]["month"])
+            )
+            .scalars()
+            .all()
+        )
+        return render_template(
+            "report_run_monthly_duration_org.html",
+            run=run,
+            org_rows=org_rows,
+        )
 
     elif run.name == "ecoes_comparison":
         rows = (