chellow 1754570085.0.0__tar.gz → 1754580326.0.0__tar.gz

{chellow-1754570085.0.0 → chellow-1754580326.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: chellow
-Version: 1754570085.0.0
+Version: 1754580326.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)

chellow-1754580326.0.0/VERSION

@@ -0,0 +1 @@
+__version__ = "1754580326.0.0"

chellow-1754580326.0.0/chellow/e/energy_management.py

@@ -0,0 +1,278 @@
+import traceback
+from collections import defaultdict
+
+from sqlalchemy import null, or_, select
+from sqlalchemy.orm import joinedload
+
+from chellow.dloads import put_mem_val
+from chellow.e.computer import (
+    SiteSource,
+    SupplySource,
+    contract_func,
+    displaced_era,
+    forecast_date,
+)
+from chellow.models import Era, Session, Site, SiteEra, Source, Supply
+from chellow.utils import c_months_u, ct_datetime_now, hh_format, hh_max, hh_min
+
+
+CATS = ("imp_grid", "exp_grid", "used", "displaced")
+QUANTS = (
+    "kwh",
+    "net_gbp",
+    #    "p_per_kwh",
+    "triad_gbp",
+    "duos_gbp",
+    "red_duos_gbp",
+    "md_kw",
+)
+
+
+def _process(sess, log, start_year, start_month, mem_id, jval, site):
+    log("started thread")
+    f_date = forecast_date()
+    report_context = {}
+
+    for month_num, (month_start, month_finish) in enumerate(
+        c_months_u(start_year=start_year, start_month=start_month, months=12)
+    ):
+        jval["progress"] = month_num
+        put_mem_val(mem_id, jval)
+        vals = {cat: defaultdict(int) for cat in CATS}
+        vals["problem"] = ""
+
+        for era in sess.execute(
+            select(Era)
+            .join(SiteEra)
+            .join(Supply)
+            .join(Source)
+            .where(
+                SiteEra.site == site,
+                Era.start_date <= month_finish,
+                or_(Era.finish_date == null(), Era.finish_date >= month_start),
+                Era.imp_mpan_core != null(),
+                Source.code.in_(("grid", "gen-grid")),
+            )
+            .options(
+                joinedload(Era.imp_supplier_contract),
+                joinedload(Era.exp_supplier_contract),
+                joinedload(Era.mop_contract),
+                joinedload(Era.dc_contract),
+            )
+        ).scalars():
+            chunk_start = hh_max(era.start_date, month_start)
+            chunk_finish = hh_min(era.finish_date, month_finish)
+
+            log("about to call first supplysource")
+            supply_ds = SupplySource(
+                sess, chunk_start, chunk_finish, f_date, era, True, report_context
+            )
+
+            supplier_contract = era.imp_supplier_contract
+            import_vb_function = contract_func(
+                report_context, supplier_contract, "virtual_bill"
+            )
+            if import_vb_function is None:
+                raise Exception(
+                    "Can't find the import_virtual_bill function in the "
+                    "supplier contract. "
+                )
+            else:
+                import_vb_function(supply_ds)
+                v_bill = supply_ds.supplier_bill
+
+                if "problem" in v_bill and len(v_bill["problem"]) > 0:
+                    raise Exception("Supplier Problem: " + v_bill["problem"])
+
+                try:
+                    vals["imp_grid"]["net_gbp"] += v_bill["net-gbp"]
+                    if "triad-gbp" in v_bill:
+                        vals["imp_grid"]["triad_gbp"] += v_bill["triad-gbp"]
+                    if "duos-red-gbp" in v_bill:
+                        vals["imp_grid"]["red_duos_gbp"] += v_bill["duos-red-gbp"]
+                    vals["imp_grid"]["duos_gbp"] += sum(
+                        v
+                        for k, v in v_bill.items()
+                        if k.endswith("-gbp") and k.startswith("duos-")
+                    )
+                except KeyError:
+                    raise Exception(
+                        f"For the era {era.id} the virtual bill {v_bill} "
+                        f"from the contract {supplier_contract} does not "
+                        f"contain the net-gbp key."
+                    )
+
+            dc_contract = era.dc_contract
+            supply_ds.contract_func(dc_contract, "virtual_bill")(supply_ds)
+            dc_bill = supply_ds.dc_bill
+            vals["imp_grid"]["gbp"] += dc_bill["net-gbp"]
+            if "problem" in dc_bill and len(dc_bill["problem"]) > 0:
+                raise Exception(f'DC Problem: {dc_bill["problem"]}')
+
+            mop_contract = era.mop_contract
+            mop_bill_function = supply_ds.contract_func(mop_contract, "virtual_bill")
+            mop_bill_function(supply_ds)
+            mop_bill = supply_ds.mop_bill
+            vals["imp_grid"]["gbp"] += mop_bill["net-gbp"]
+            if "problem" in mop_bill and len(mop_bill["problem"]) > 0:
+                raise Exception(f'MOP Problem: {mop_bill["problem"]}')
+
+            exp_supplier_contract = era.exp_supplier_contract
+            if exp_supplier_contract is not None:
+                log("about to call second supply source")
+                exp_supply_ds = SupplySource(
+                    sess,
+                    chunk_start,
+                    chunk_finish,
+                    f_date,
+                    era,
+                    False,
+                    report_context,
+                )
+                exp_vb_function = contract_func(
+                    report_context, exp_supplier_contract, "virtual_bill"
+                )
+                if exp_vb_function is None:
+                    raise Exception(
+                        "Can't find the export_virtual_bill function in the "
+                        "supplier contract. "
+                    )
+                else:
+                    exp_vb_function(exp_supply_ds)
+                    v_bill = exp_supply_ds.supplier_bill
+
+                    if "problem" in v_bill and len(v_bill["problem"]) > 0:
+                        raise Exception(f"Export supplier Problem: {v_bill['problem']}")
+
+                    try:
+                        vals["exp_grid"]["net_gbp"] += v_bill["net-gbp"]
+                        if "triad-gbp" in v_bill:
+                            vals["exp_grid"]["triad_gbp"] += v_bill["triad-gbp"]
+                        if "duos-red-gbp" in v_bill:
+                            vals["exp_grid"]["red_duos_gbp"] += v_bill["duos-red-gbp"]
+                        vals["exp_grid"]["duos_gbp"] += sum(
+                            v
+                            for k, v in v_bill.items()
+                            if k.endswith("-gbp") and k.startswith("duos-")
+                        )
+                    except KeyError as e:
+                        raise Exception(
+                            f"For the era {era.id} the virtual bill {v_bill} from the "
+                            f"contract {supplier_contract} does not contain the key {e}"
+                        )
+
+        disp_era = displaced_era(
+            sess, report_context, site, month_start, month_finish, f_date
+        )
+        log("about to call sitesource")
+        site_ds = SiteSource(
+            sess,
+            site,
+            month_start,
+            month_finish,
+            f_date,
+            report_context,
+            disp_era,
+            exclude_virtual=False,
+        )
+
+        if disp_era is not None:
+            disp_supplier_contract = disp_era.imp_supplier_contract
+            disp_vb_function = contract_func(
+                report_context, disp_supplier_contract, "displaced_virtual_bill"
+            )
+            if disp_vb_function is None:
+                raise Exception(
+                    f"The supplier contract {disp_supplier_contract} "
+                    "doesn't have the displaced_virtual_bill() function."
+                )
+            disp_vb_function(site_ds)
+            disp_supplier_bill = site_ds.supplier_bill
+
+            try:
+                vals["displaced"]["net_gbp"] += disp_supplier_bill["net-gbp"]
+            except KeyError:
+                disp_supplier_bill["problem"] += (
+                    f"For the supply {site_ds.mpan_core} the virtual bill "
+                    f"{disp_supplier_bill} from the contract {disp_supplier_contract} "
+                    f"does not contain the net-gbp key."
+                )
+            log(str(disp_supplier_bill))
+            if "triad-gbp" in disp_supplier_bill:
+                vals["displaced"]["triad_gbp"] += disp_supplier_bill["triad-gbp"]
+            if "duos-red-gbp" in disp_supplier_bill:
+                vals["displaced"]["red_duos_gbp"] += disp_supplier_bill["duos-red-gbp"]
+            vals["displaced"]["duos_gbp"] += sum(
+                v
+                for k, v in disp_supplier_bill.items()
+                if k.endswith("-gbp") and k.startswith("duos-")
+            )
+
+        for hh in site.hh_data(sess, month_start, month_finish, exclude_virtual=True):
+            for cat in CATS:
+                vals_cat = vals[cat]
+                vals_cat["md_kw"] = max(vals_cat["md_kw"], hh[cat] * 2)
+                vals_cat["kwh"] += float(hh[cat])
+
+        for quant in ("net_gbp", "triad_gbp", "duos_gbp", "red_duos_gbp"):
+            vals["used"][quant] += vals["imp_grid"][quant] + vals["displaced"][quant]
+
+        """
+        for cat in CATS:
+            vals_cat = vals[cat]
+            if vals_cat["kwh"] > 0:
+                vals_cat["p_per_kwh"] = vals_cat["gbp"] / vals_cat["kwh"] * 100
+            else:
+                vals_cat["p_per_kwh"] = 0
+        """
+
+        FORMAT_LOOKUP = {
+            "kwh": "{:,.0f}",
+            "net_gbp": "{:,.2f}",
+            "p_per_kwh": "{:,.2f}",
+            "triad_gbp": "{:,.2f}",
+            "duos_gbp": "{:,.2f}",
+            "red_duos_gbp": "{:,.2f}",
+            "md_kw": "{:,.2f}",
+        }
+
+        for quant in QUANTS:
+            row = [hh_format(month_start)[:7], quant]
+            fmt = FORMAT_LOOKUP[quant]
+            for cat in CATS:
+                row.append(fmt.format(vals[cat][quant]))
+            jval["rows"].append(row)
+
+
+def totals_runner(mem_id, site_id, start_date):
+    sess = None
+    titles = ["month", "quantity"]
+    titles.extend(CATS)
+    log_list = []
+
+    jval = {"titles": titles, "rows": [], "status": "running", "log": log_list}
+    try:
+        with Session() as sess:
+            site = Site.get_by_id(sess, site_id)
+
+            def log(msg):
+                return
+                log_list.append(msg)
+                put_mem_val(mem_id, jval)
+
+            if start_date is None:
+                now = ct_datetime_now()
+                start_year = now.year - 1
+                start_month = now.month
+            else:
+                start_year = start_date.year
+                start_month = start_date.month
+
+            _process(sess, log, start_year, start_month, mem_id, jval, site)
+
+    except BaseException:
+        msg = traceback.format_exc()
+        jval["problem"] = msg
+    finally:
+        jval["status"] = "finished"
+        put_mem_val(mem_id, jval)

{chellow-1754570085.0.0 → chellow-1754580326.0.0}/chellow/e/views.py

@@ -6,7 +6,6 @@ from datetime import datetime as Datetime
 from decimal import Decimal
 from io import BytesIO, StringIO
 from itertools import chain, islice
-from random import random
 
 
 from dateutil.relativedelta import relativedelta
@@ -4156,16 +4155,10 @@ def em_totals(site_id):
     if "mem_id" in request.values:
         mem_id = req_int("mem_id")
         site_info = chellow.dloads.get_mem_val(mem_id)
-        random_number = random()
 
         status_code = 200 if site_info["status"] == "running" else 286
         return make_response(
-            render_template(
-                "em_totals.html",
-                site=site,
-                site_info=site_info,
-                random_number=random_number,
-            ),
+            render_template("em_totals.html", site=site, site_info=site_info),
             status_code,
         )
 

{chellow-1754570085.0.0 → chellow-1754580326.0.0}/chellow/templates/e/em_totals.html

@@ -2,6 +2,12 @@
 	<div
 			hx-get="/e/sites/{{site.id}}/energy_management/totals?mem_id={{mem_id}}"
 		 	hx-trigger="every 3s">
+		<label>
+			Calculating:
+			<progress value="0" max="12">
+				0 months
+			</progress>
+		</label>
   </div>
 {% elif site_info.status == 'running' %}
 	<label>
@@ -11,6 +17,11 @@
 		</progress>
 	</label>
 {% else %}
+	{% if site_info['problem']|length > 0 %}
+		<pre>
+			{{site_info['problem']}}
+		</pre>
+	{% endif %}
   <table>
 		<caption>Last 12 months</caption>
 		<thead>
@@ -30,4 +41,12 @@
 			{% endfor %}
 		</tbody>
 	</table>
+
+	<h3>Log</h3>
+
+	<ul>
+		{% for line in site_info.log %}
+			<li>>{{line}}</li>
+		{% endfor %}
+	</ul>
 {% endif %}

chellow-1754580326.0.0/test/e/test_energy_management.py

@@ -0,0 +1,218 @@
+from chellow.e.energy_management import _process
+from chellow.models import (
+    Comm,
+    Contract,
+    Cop,
+    DtcMeterType,
+    EnergisationStatus,
+    GspGroup,
+    MarketRole,
+    MeterPaymentType,
+    MeterType,
+    Mtc,
+    MtcLlfc,
+    MtcParticipant,
+    Participant,
+    Pc,
+    Site,
+    Source,
+    VoltageLevel,
+    insert_comms,
+    insert_cops,
+    insert_dtc_meter_types,
+    insert_energisation_statuses,
+    insert_sources,
+    insert_voltage_levels,
+)
+from chellow.utils import ct_datetime, to_utc, utc_datetime
+
+
+def test_process(mocker, sess):
+    vf = to_utc(ct_datetime(1996, 1, 1))
+    site = Site.insert(sess, "CI017", "Water Works")
+
+    market_role_Z = MarketRole.insert(sess, "Z", "Non-core")
+    participant = Participant.insert(sess, "CALB", "AK Industries")
+    participant.insert_party(sess, market_role_Z, "None core", vf, None, None)
+    bank_holiday_rate_script = {"bank_holidays": []}
+    Contract.insert_non_core(
+        sess, "bank_holidays", "", {}, vf, None, bank_holiday_rate_script
+    )
+    market_role_X = MarketRole.insert(sess, "X", "Supplier")
+    market_role_M = MarketRole.insert(sess, "M", "Mop")
+    market_role_C = MarketRole.insert(sess, "C", "HH Dc")
+    market_role_R = MarketRole.insert(sess, "R", "Distributor")
+    participant.insert_party(sess, market_role_M, "Fusion Mop Ltd", vf, None, None)
+    participant.insert_party(sess, market_role_X, "Fusion Ltc", vf, None, None)
+    participant.insert_party(sess, market_role_C, "Fusion DC", vf, None, None)
+
+    mop_charge_script = """
+from chellow.utils import reduce_bill_hhs
+
+def virtual_bill_titles():
+    return ['net-gbp', 'problem']
+
+def virtual_bill(ds):
+    for hh in ds.hh_data:
+        hh_start = hh['start-date']
+        bill_hh = ds.supplier_bill_hhs[hh_start]
+
+        bill_hh['net-gbp'] = sum(
+            v for k, v in bill_hh.items() if k.endswith('gbp'))
+
+    ds.mop_bill = reduce_bill_hhs(ds.supplier_bill_hhs)
+"""
+    mop_contract = Contract.insert_mop(
+        sess,
+        "Fusion Mop Contract",
+        participant,
+        mop_charge_script,
+        {},
+        utc_datetime(2000, 1, 1),
+        None,
+        {},
+    )
+
+    dc_charge_script = """
+from chellow.utils import reduce_bill_hhs
+
+def virtual_bill_titles():
+    return ['net-gbp', 'problem']
+
+def virtual_bill(ds):
+    for hh in ds.hh_data:
+        hh_start = hh['start-date']
+        bill_hh = ds.supplier_bill_hhs[hh_start]
+
+        bill_hh['net-gbp'] = sum(
+            v for k, v in bill_hh.items() if k.endswith('gbp'))
+
+    ds.dc_bill = reduce_bill_hhs(ds.supplier_bill_hhs)
+"""
+
+    dc_contract = Contract.insert_dc(
+        sess,
+        "Fusion DC 2000",
+        participant,
+        dc_charge_script,
+        {},
+        utc_datetime(2000, 1, 1),
+        None,
+        {},
+    )
+    pc = Pc.insert(sess, "00", "hh", utc_datetime(2000, 1, 1), None)
+    insert_cops(sess)
+    cop = Cop.get_by_code(sess, "5")
+    insert_comms(sess)
+    comm = Comm.get_by_code(sess, "GSM")
+
+    supplier_charge_script = """
+import chellow.e.ccl
+from chellow.utils import HH, reduce_bill_hhs, utc_datetime
+
+def virtual_bill_titles():
+    return [
+        'ccl-kwh', 'ccl-rate', 'ccl-gbp', 'net-gbp', 'vat-gbp', 'gross-gbp',
+        'sum-msp-kwh', 'sum-msp-gbp', 'problem']
+
+def virtual_bill(ds):
+    for hh in ds.hh_data:
+        hh_start = hh['start-date']
+        bill_hh = ds.supplier_bill_hhs[hh_start]
+        bill_hh['sum-msp-kwh'] = hh['msp-kwh']
+        bill_hh['sum-msp-gbp'] = hh['msp-kwh'] * 0.1
+        bill_hh['net-gbp'] = sum(
+            v for k, v in bill_hh.items() if k.endswith('gbp'))
+        bill_hh['vat-gbp'] = 0
+        bill_hh['gross-gbp'] = bill_hh['net-gbp'] + bill_hh['vat-gbp']
+
+    ds.supplier_bill = reduce_bill_hhs(ds.supplier_bill_hhs)
+"""
+    imp_supplier_contract = Contract.insert_supplier(
+        sess,
+        "Fusion Supplier 2000",
+        participant,
+        supplier_charge_script,
+        {},
+        utc_datetime(2000, 1, 1),
+        None,
+        {},
+    )
+    dno = participant.insert_party(sess, market_role_R, "WPD", vf, None, "22")
+    Contract.insert_dno(sess, dno.dno_code, participant, "", {}, vf, None, {})
+    meter_type = MeterType.insert(sess, "C5", "COP 1-5", vf, None)
+    meter_payment_type = MeterPaymentType.insert(sess, "CR", "Credit", vf, None)
+    mtc = Mtc.insert(sess, "845", False, True, vf, None)
+    mtc_participant = MtcParticipant.insert(
+        sess,
+        mtc,
+        participant,
+        "HH COP5 And Above With Comms",
+        False,
+        True,
+        meter_type,
+        meter_payment_type,
+        0,
+        vf,
+        None,
+    )
+    insert_voltage_levels(sess)
+    voltage_level = VoltageLevel.get_by_code(sess, "HV")
+    llfc = dno.insert_llfc(
+        sess, "510", "PC 5-8 & HH HV", voltage_level, False, True, vf, None
+    )
+    MtcLlfc.insert(sess, mtc_participant, llfc, vf, None)
+    insert_sources(sess)
+    source = Source.get_by_code(sess, "grid")
+    gsp_group = GspGroup.insert(sess, "_L", "South Western")
+    insert_energisation_statuses(sess)
+    energisation_status = EnergisationStatus.get_by_code(sess, "E")
+    insert_dtc_meter_types(sess)
+    dtc_meter_type = DtcMeterType.get_by_code(sess, "H")
+    site.insert_e_supply(
+        sess,
+        source,
+        None,
+        "Bob",
+        utc_datetime(2000, 1, 1),
+        None,
+        gsp_group,
+        mop_contract,
+        "773",
+        dc_contract,
+        "ghyy3",
+        "hgjeyhuw",
+        dno,
+        pc,
+        "845",
+        cop,
+        comm,
+        None,
+        energisation_status,
+        dtc_meter_type,
+        "22 7867 6232 781",
+        "510",
+        imp_supplier_contract,
+        "7748",
+        361,
+        None,
+        None,
+        None,
+        None,
+        None,
+    )
+
+    sess.commit()
+    mocker.patch("chellow.e.energy_management.put_mem_val")
+
+    start_year = 2009
+    start_month = 7
+
+    def log(x):
+        pass
+
+    mem_id = 0
+
+    jval = {"titles": [], "rows": [], "status": "running", "log": []}
+
+    _process(sess, log, start_year, start_month, mem_id, jval, site)

chellow-1754570085.0.0/VERSION

@@ -1 +0,0 @@
-__version__ = "1754570085.0.0"