chellow 1751619264.0.0__tar.gz → 1752483970.0.0__tar.gz

{chellow-1751619264.0.0 → chellow-1752483970.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: chellow
-Version: 1751619264.0.0
+Version: 1752483970.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
@@ -22,7 +22,7 @@ Requires-Dist: pypdf==4.3.1
 Requires-Dist: python-dateutil==2.8.2
 Requires-Dist: pytz==2022.6
 Requires-Dist: requests==2.32.4
-Requires-Dist: sqlalchemy==2.0.30
+Requires-Dist: sqlalchemy==2.0.41
 Requires-Dist: waitress==3.0.1
 Requires-Dist: xlrd==2.0.1
 Requires-Dist: zish==0.1.12
@@ -86,6 +86,19 @@ in Windows an environment variable can be set by doing:
   `admin@example.com` and the password `admin`, and then the home page should appear.
   Change the admin password from the `users` page.
 
+### Azure
+
+SSH into the machine and you'll be in ``/home/site/wwwroot``.
+
+* Create a virtual environment: ``python -m venv antenv``
+* Activate the virtual environment with: ``source antenv/bin/activate``
+*
+In ``/home/site/wwwroot`` create a file called ``app.py`` containing the following:
+
+    import chellow
+    
+    app = chellow.create_app()
+
 
 ### Manually Upgrading Chellow
 

{chellow-1751619264.0.0 → chellow-1752483970.0.0}/README.md

@@ -56,6 +56,19 @@ in Windows an environment variable can be set by doing:
   `admin@example.com` and the password `admin`, and then the home page should appear.
   Change the admin password from the `users` page.
 
+### Azure
+
+SSH into the machine and you'll be in ``/home/site/wwwroot``.
+
+* Create a virtual environment: ``python -m venv antenv``
+* Activate the virtual environment with: ``source antenv/bin/activate``
+*
+In ``/home/site/wwwroot`` create a file called ``app.py`` containing the following:
+
+    import chellow
+    
+    app = chellow.create_app()
+
 
 ### Manually Upgrading Chellow
 

chellow-1752483970.0.0/VERSION

@@ -0,0 +1 @@
+__version__ = "1752483970.0.0"

{chellow-1751619264.0.0 → chellow-1752483970.0.0}/chellow/__init__.py

@@ -1,4 +1,6 @@
+import json
 import os
+from base64 import b64decode
 from datetime import datetime as Datetime
 from importlib.metadata import version
 from pathlib import Path
@@ -127,17 +129,45 @@ def create_app(testing=False, instance_path=None):
             g.config = {}
         ad_props = g.config.get("ad_authentication", {})
         ad_auth_on = ad_props.get("on", False)
+        easy_auth_props = g.config.get("easy_auth", {})
+        easy_auth_on = easy_auth_props.get("on", False)
+        path = request.path
+        if path in ("/health", "/robots933456.txt"):
+            return
         if ad_auth_on:
             username = request.headers["X-Isrw-Proxy-Logon-User"].upper()
-            user = g.sess.query(User).filter(User.email_address == username).first()
+            user = g.sess.scalars(
+                select(User).where(User.email_address == username)
+            ).first()
             if user is None:
                 try:
                     username = ad_props["default_user"]
-                    user = (
-                        g.sess.query(User)
-                        .filter(User.email_address == username)
-                        .first()
-                    )
+                    user = g.sess.scalars(
+                        select(User).where(User.email_address == username)
+                    ).first()
+                except KeyError:
+                    user = None
+            if user is not None:
+                g.user = user
+        elif easy_auth_on:
+            jwt_enc = request.headers["X-MS-CLIENT-PRINCIPAL"]
+            jwt = json.loads(b64decode(jwt_enc))
+            for claim in jwt["claims"]:
+                if claim["typ"] == (
+                    "http://schemas.xmlsoap.org/ws/2005/05/identity/claims/"
+                    "emailaddress"
+                ):
+                    username = claim["val"]
+                    user = g.sess.scalars(
+                        select(User).where(User.email_address == username)
+                    ).first()
+                    break
+            if user is None:
+                try:
+                    username = easy_auth_props["default_user"]
+                    user = g.sess.scalars(
+                        select(User).where(User.email_address == username)
+                    ).first()
                 except KeyError:
                     user = None
             if user is not None:
@@ -156,25 +186,20 @@ def create_app(testing=False, instance_path=None):
                         key = None
 
                     email = ips[key]
-                    g.user = (
-                        g.sess.query(User).filter(User.email_address == email).first()
-                    )
+                    g.user = g.sess.scalars(
+                        select(User).where(User.email_address == email)
+                    ).first()
                 except KeyError:
                     pass
             else:
-                user = (
-                    g.sess.query(User)
-                    .filter(User.email_address == auth.username)
-                    .first()
-                )
+                user = g.sess.scalars(
+                    select(User).where(User.email_address == auth.username)
+                ).first()
                 if user is not None and user.password_matches(auth.password):
                     g.user = user
 
         # Got our user
-        path = request.path
         method = request.method
-        if path in ("/health",):
-            return
 
         if g.user is not None:
             if "X-Isrw-Proxy-Logon-User" in request.headers:
@@ -227,7 +252,9 @@ def create_app(testing=False, instance_path=None):
             g.sess.commit()
             return
 
-        if g.user is None or (not ad_auth_on and auth is None):
+        if (not easy_auth_on) and (
+            (g.user is None) or (not ad_auth_on and auth is None)
+        ):
             return Response(
                 "Could not verify your access level for that URL.\n"
                 "You have to login with proper credentials",

{chellow-1751619264.0.0 → chellow-1752483970.0.0}/chellow/e/bill_parsers/drax_edi.py

@@ -51,7 +51,11 @@ ELEMENT_MAP = {
                 "cfd-operational-kwh",
             ),
             "954379": ("cfd-interim-gbp", "cfd-interim-rate", "cfd-interim-kwh"),
-            "538249": ("capacity-gbp", "capacity-rate", "capacity-kwh"),
+            "538249": (
+                "cm-settlement-levy-gbp",
+                "cm-settlement-levy-rate",
+                "cm-settlement-levy-kwh",
+            ),
             "568307": ("capacity-gbp", "capacity-rate", "capacity-kwh"),
         },
     },
@@ -104,6 +108,7 @@ ELEMENT_MAP = {
                 "duos-availability-kva",
             ),
             "209269": ("tnuos-gbp", "tnuos-rate", "tnuos-days"),
+            "065950": ("eii-gbp", "eii-rate", "eii-kwh"),
         },
         "DUSDIS": {
             "122568": ("nrg-gsp-losses-gbp", "nrg-rate", "nrg-gsp-losses-kwh"),
@@ -149,7 +154,6 @@ ELEMENT_MAP = {
     "930504": ("eii-gbp", None, None),
     "331201": ("eii-gbp", None, None),
     "307253": ("eii-gbp", "eii-rate", "eii-kwh"),
-    "065950": ("eii-gbp", "eii-rate", "eii-kwh"),
 }
 
 TPR_LOOKUP = {

{chellow-1751619264.0.0 → chellow-1752483970.0.0}/chellow/e/bill_parsers/mm.py

@@ -47,7 +47,9 @@ def _handle_0050(headers, pre_record, record):
         unknown_4=12,
         late_payment=12,
     )
-    headers["late_payment"] = Decimal(parts["late_payment"]) / Decimal(100)
+    headers["breakdown"]["late_payment"] += Decimal(parts["late_payment"]) / Decimal(
+        100
+    )
     """
 
 
@@ -57,13 +59,10 @@ def _handle_0051(headers, pre_record, record):
 
 def _handle_0100(headers, pre_record, record):
     issue_date = headers["issue_date"]
-    late_payment = headers.get("late_payment")
     headers.clear()
     headers["issue_date"] = issue_date
-    if late_payment is not None:
-        headers["late_payment"] = late_payment
-    headers["account"] = pre_record[43:52]
-    headers["reference"] = pre_record[41:46]
+    headers["account"] = pre_record[42:52]
+    headers["reference"] = pre_record[52:64]
     headers["kwh"] = Decimal("0")
     headers["breakdown"] = defaultdict(int, {"vat": {}})
     headers["reads"] = []
@@ -245,6 +244,17 @@ def _handle_0860(headers, pre_record, record):
     bd["metering-gbp"] += Decimal(parts["metering_gbp"]) / Decimal("100")
 
 
+def _handle_0960(headers, pre_record, record):
+    parts = _chop_record(
+        record,
+        days=3,
+        payment=12,
+        rate=6,
+    )
+    bd = headers["breakdown"]
+    bd["late_payment_gbp"] += Decimal(parts["payment"]) / Decimal(100)
+
+
 def _handle_1455(headers, pre_record, record):
     parts = _chop_record(
         record, ccl_kwh=13, unknown_1=10, ccl_rate=13, ccl_gbp=12, unkown_2=8
@@ -294,12 +304,6 @@ def _handle_1500(headers, pre_record, record):
     breakdown = headers["breakdown"]
     net = Decimal("0.00") + Decimal(parts["net"]) / Decimal("100")
     gross = Decimal("0.00") + Decimal(parts["gross"]) / Decimal("100")
-    if "late_payment" in headers:
-        late_payment_gbp = headers["late_payment"]
-        net += late_payment_gbp
-        gross += late_payment_gbp
-        breakdown["late-payment-gbp"] += late_payment_gbp
-        del headers["late_payment"]
 
     return {
         "bill_type_code": "W" if net < 0 else "N",
@@ -354,6 +358,7 @@ LINE_HANDLERS = {
     "0461": _handle_0461,
     "0470": _handle_0470,
     "0860": _handle_0860,
+    "0960": _handle_0960,
     "1455": _handle_1455,
     "1460": _handle_1460,
     "1500": _handle_1500,

{chellow-1751619264.0.0 → chellow-1752483970.0.0}/chellow/e/computer.py

@@ -22,6 +22,7 @@ from chellow.models import (
     Channel,
     ClockInterval,
     Contract,
+    DtcMeterType,
     Era,
     HhDatum,
     Llfc,
@@ -694,7 +695,14 @@ class SiteSource(DataSource):
                 self.dtc_meter_type_code = self.era_map_dtc_meter_types[
                     era_dtc_meter_type_code
                 ]
+                if self.dtc_meter_type_code is None:
+                    self.dtc_meter_type = None
+                else:
+                    self.dtc_meter_type = DtcMeterType.get_by_code(
+                        sess, era_dtc_meter_type_code
+                    )
             else:
+                self.dtc_meter_type = era.dtc_meter_type
                 self.dtc_meter_type_code = era_dtc_meter_type_code
 
             self.is_import = True
@@ -1086,7 +1094,14 @@ class SupplySource(DataSource):
             self.dtc_meter_type_code = self.era_map_dtc_meter_types[
                 era_dtc_meter_type_code
             ]
+            if self.dtc_meter_type_code is None:
+                self.dtc_meter_type_code = None
+            else:
+                self.dtc_meter_type = DtcMeterType.get_by_code(
+                    sess, self.dtc_meter_type_code
+                )
         else:
+            self.dtc_meter_type = era.dtc_meter_type
             self.dtc_meter_type_code = era_dtc_meter_type_code
 
         self.id = self.mpan_core
@@ -1196,11 +1211,15 @@ class SupplySource(DataSource):
                         - Datetime(chunk_start.year, 1, 1)
                     ).total_seconds()
                 )
+                if hist_era.pc.code == "00":
+                    ssc = Ssc.get_by_code(sess, "0393", chunk_start)
+                else:
+                    ssc = hist_era.ssc
 
-                for tpr in (
-                    sess.query(Tpr)
+                for tpr in sess.scalars(
+                    select(Tpr)
                     .join(MeasurementRequirement)
-                    .filter(MeasurementRequirement.ssc == hist_era.ssc)
+                    .where(MeasurementRequirement.ssc == ssc)
                 ):
                     for hist_date in hh_range(self.caches, chunk_start, chunk_finish):
                         if is_tpr(sess, self.caches, tpr.code, hist_date):

{chellow-1751619264.0.0 → chellow-1752483970.0.0}/chellow/e/triad.py

@@ -27,7 +27,7 @@ from chellow.utils import (
 
 def hh(ds, rate_period="monthly", est_kw=None):
     for hh in ds.hh_data:
-        if hh["ct-is-month-end"]:
+        if hh["ct-is-month-end"] and ds.dtc_meter_type is not None:
             _process_triad_hh(ds, rate_period, est_kw, hh)
 
 

{chellow-1751619264.0.0 → chellow-1752483970.0.0}/chellow/reports/report_247.py

@@ -471,14 +471,13 @@ def _process_site(
     if supply_ids is not None:
         q = q.where(Supply.id.in_(supply_ids))
     for supply in sess.execute(q).scalars():
-        last_era = (
-            sess.execute(
-                select(Era).where(Era.supply == supply).order_by(Era.start_date.desc())
-            )
-            .scalars()
-            .first()
-        )
-        if last_era.finish_date is not None and last_era.start_date <= finish_date:
+        last_era = sess.scalars(
+            select(Era)
+            .join(SiteEra)
+            .where(Era.supply == supply)
+            .order_by(Era.start_date.desc())
+        ).first()
+        if last_era.finish_date is not None and last_era.finish_date < finish_date:
             site_era = sess.execute(
                 select(SiteEra).where(
                     SiteEra.era == last_era,

{chellow-1751619264.0.0 → chellow-1752483970.0.0}/chellow/views.py

@@ -198,6 +198,11 @@ def health():
     return Response("healthy\n", mimetype="text/plain")
 
 
+@home.route("/robots933456.txt")
+def robots():
+    return Response("healthy\n", mimetype="text/plain")
+
+
 @home.route("/local_reports/<int:report_id>/output")
 def local_report_output_get(report_id):
     report = g.sess.execute(select(Report).where(Report.id == report_id)).scalar_one()

{chellow-1751619264.0.0 → chellow-1752483970.0.0}/pyproject.toml

@@ -18,7 +18,7 @@ dependencies = [
 	"pypdf==4.3.1",
 	"pg8000==1.31.1",
 	"Flask==3.1.1",
-	"SQLAlchemy==2.0.30",
+	"SQLAlchemy==2.0.41",
 	"flask-restx==1.3.0",
   "jsonschema==4.17.3",  # Temporarily needed until RestX updated
 	"openpyxl==3.1.5",

{chellow-1751619264.0.0 → chellow-1752483970.0.0}/test/e/bill_parsers/test_bill_parser_mm.py

@@ -27,11 +27,11 @@ def test_handle_0100():
     _handle_0100(headers, pre_record, record)
     assert headers == {
         "issue_date": to_utc(ct_datetime(2023, 5, 2)),
-        "account": "023719050",
+        "account": "0023719050",
         "breakdown": {"vat": {}},
         "kwh": Decimal("0"),
         "reads": [],
-        "reference": " 0023",
+        "reference": "000023905961",
     }
 
 

{chellow-1751619264.0.0 → chellow-1752483970.0.0}/test/e/test_computer.py

@@ -819,6 +819,7 @@ def test_SupplySource_init_hh(sess, mocker):
     )
 
     assert not ss.full_channels
+    assert ss.dtc_meter_type == dtc_meter_type
 
 
 def test_SupplySource_init_nhh(sess, mocker):

{chellow-1751619264.0.0 → chellow-1752483970.0.0}/test/e/test_tnuos.py

@@ -16,6 +16,7 @@ from chellow.models import (
     Pc,
     Site,
     Source,
+    Ssc,
     VoltageLevel,
     insert_comms,
     insert_cops,
@@ -94,6 +95,7 @@ def test_process_banded_hh_ums(sess):
     insert_energisation_statuses(sess)
     energisation_status = EnergisationStatus.get_by_code(sess, "E")
     gsp_group = GspGroup.insert(sess, "_L", "South Western")
+    Ssc.insert(sess, "0393", "unrestricted", True, vf, None)
     supply = site.insert_e_supply(
         sess,
         source,

{chellow-1751619264.0.0 → chellow-1752483970.0.0}/test/e/test_triad.py

@@ -1,7 +1,12 @@
 from decimal import Decimal
 
 from chellow.e.computer import SupplySource
-from chellow.e.triad import _find_triad_dates, _process_triad_hh, national_grid_import
+from chellow.e.triad import (
+    _find_triad_dates,
+    _process_triad_hh,
+    hh,
+    national_grid_import,
+)
 from chellow.models import (
     Comm,
     Contract,
@@ -332,3 +337,222 @@ def test_national_grid_import(mocker, sess):
 
     s = mocker.Mock()
     national_grid_import(sess, log, set_progress, s)
+
+
+def test_hh(sess):
+    vf = to_utc(ct_datetime(1996, 1, 1))
+    site = Site.insert(sess, "CI017", "Water Works")
+    start_date = to_utc(ct_datetime(2023, 7, 31, 23, 30))
+    finish_date = to_utc(ct_datetime(2023, 7, 31, 23, 30))
+    forecast_from = to_utc(ct_datetime(2020, 1, 1))
+
+    market_role_Z = MarketRole.insert(sess, "Z", "Non-core")
+    participant = Participant.insert(sess, "CALB", "AK Industries")
+    participant.insert_party(sess, market_role_Z, "None core", vf, None, None)
+    bank_holiday_rate_script = {"bank_holidays": []}
+    Contract.insert_non_core(
+        sess, "bank_holidays", "", {}, vf, None, bank_holiday_rate_script
+    )
+    triad_rate_script = {
+        "triad_gbp_per_gsp_kw": {
+            "import": {"_L": {"HHTariff(Floored)_£/kW": Decimal(0)}}
+        },
+    }
+    Contract.insert_non_core(
+        sess,
+        "triad_rates",
+        "",
+        {},
+        vf,
+        None,
+        triad_rate_script,
+    )
+    triad_dates_rate_script = {
+        "triad_dates": [
+            utc_datetime(2021, 12, 2, 16, 30),
+            utc_datetime(2022, 1, 5, 17, 00),
+            utc_datetime(2022, 1, 20, 17, 00),
+        ],
+    }
+    Contract.insert_non_core(
+        sess,
+        "triad_dates",
+        "",
+        {},
+        vf,
+        None,
+        triad_dates_rate_script,
+    )
+    market_role_X = MarketRole.insert(sess, "X", "Supplier")
+    market_role_M = MarketRole.insert(sess, "M", "Mop")
+    market_role_C = MarketRole.insert(sess, "C", "HH Dc")
+    market_role_R = MarketRole.insert(sess, "R", "Distributor")
+    participant.insert_party(sess, market_role_M, "Fusion Mop Ltd", vf, None, None)
+    participant.insert_party(sess, market_role_X, "Fusion Ltc", vf, None, None)
+    participant.insert_party(sess, market_role_C, "Fusion DC", vf, None, None)
+    mop_contract = Contract.insert_mop(
+        sess, "Fusion", participant, "", {}, vf, None, {}
+    )
+    dc_contract = Contract.insert_dc(
+        sess, "Fusion DC 2000", participant, "", {}, vf, None, {}
+    )
+    pc = Pc.insert(sess, "00", "hh", vf, None)
+    insert_cops(sess)
+    cop = Cop.get_by_code(sess, "5")
+    insert_comms(sess)
+    comm = Comm.get_by_code(sess, "GSM")
+    imp_supplier_contract = Contract.insert_supplier(
+        sess,
+        "Fusion Supplier 2000",
+        participant,
+        "",
+        {},
+        vf,
+        None,
+        {},
+    )
+    dno = participant.insert_party(sess, market_role_R, "WPD", vf, None, "22")
+    dno_rate_script = {
+        "_L": {
+            "bands": {},
+            "tariffs": {
+                "510_00": {
+                    "description": "",
+                    "gbp-per-kvarh": 0,
+                    "green-gbp-per-kwh": 0,
+                }
+            },
+        },
+    }
+    Contract.insert_dno(
+        sess, dno.dno_code, participant, "", {}, vf, None, dno_rate_script
+    )
+    meter_type = MeterType.insert(sess, "UM", "Unmetered", vf, None)
+    meter_payment_type = MeterPaymentType.insert(sess, "CR", "Credit", vf, None)
+    mtc = Mtc.insert(sess, "845", True, False, vf, None)
+    mtc_participant = MtcParticipant.insert(
+        sess,
+        mtc,
+        participant,
+        "HH COP5",
+        False,
+        True,
+        meter_type,
+        meter_payment_type,
+        0,
+        vf,
+        None,
+    )
+    insert_voltage_levels(sess)
+    voltage_level = VoltageLevel.get_by_code(sess, "HV")
+    llfc = dno.insert_llfc(
+        sess, "510", "PC 5-8 & HH HV", voltage_level, False, True, vf, None
+    )
+    MtcLlfc.insert(sess, mtc_participant, llfc, vf, None)
+    insert_sources(sess)
+    source = Source.get_by_code(sess, "grid")
+    insert_energisation_statuses(sess)
+    energisation_status = EnergisationStatus.get_by_code(sess, "E")
+    gsp_group = GspGroup.insert(sess, "_L", "South Western")
+    insert_dtc_meter_types(sess)
+    dtc_meter_type = DtcMeterType.get_by_code(sess, "H")
+    supply = site.insert_e_supply(
+        sess,
+        source,
+        None,
+        "Bob",
+        to_utc(ct_datetime(2000, 1, 1)),
+        None,
+        gsp_group,
+        mop_contract,
+        "773",
+        dc_contract,
+        "ghyy3",
+        "hgjeyhuw",
+        dno,
+        pc,
+        "845",
+        cop,
+        comm,
+        None,
+        energisation_status,
+        dtc_meter_type,
+        "22 7867 6232 781",
+        "510",
+        imp_supplier_contract,
+        "7748",
+        361,
+        None,
+        None,
+        None,
+        None,
+        None,
+    )
+
+    sess.commit()
+    caches = {}
+    era = supply.eras[0]
+    is_import = True
+    ds = SupplySource(
+        sess, start_date, finish_date, forecast_from, era, is_import, caches
+    )
+    h = ds.hh_data[0]
+    h["duos-description"] = "Unmetered"
+
+    rate_period = "monthly"
+    est_kw = None
+    hh(ds, rate_period=rate_period, est_kw=est_kw)
+    assert h == {
+        "hist-start": utc_datetime(2019, 7, 31, 22, 30),
+        "start-date": utc_datetime(2023, 7, 31, 22, 30),
+        "ct-day": 31,
+        "utc-month": 7,
+        "utc-day": 31,
+        "utc-decimal-hour": 22.5,
+        "utc-year": 2023,
+        "utc-hour": 22,
+        "utc-minute": 30,
+        "ct-year": 2023,
+        "ct-month": 7,
+        "ct-decimal-hour": 23.5,
+        "ct-day-of-week": 0,
+        "utc-day-of-week": 0,
+        "utc-is-bank-holiday": False,
+        "ct-is-bank-holiday": False,
+        "utc-is-month-end": False,
+        "ct-is-month-end": True,
+        "status": "X",
+        "imp-msp-kvarh": 0,
+        "imp-msp-kvar": 0,
+        "exp-msp-kvarh": 0,
+        "exp-msp-kvar": 0,
+        "msp-kw": 0,
+        "msp-kwh": 0,
+        "msp-kva": 0,
+        "hist-import-grid-kvarh": 0,
+        "hist-export-grid-kvarh": 0,
+        "anti-msp-kwh": 0,
+        "anti-msp-kw": 0,
+        "hist-imp-msp-kvarh": 0,
+        "hist-kwh": 0,
+        "duos-description": "Unmetered",
+        "triad-estimate-1-date": utc_datetime(2021, 12, 2, 16, 30),
+        "triad-estimate-1-gsp-kw": 0,
+        "triad-estimate-1-laf": 1,
+        "triad-estimate-1-msp-kw": 0,
+        "triad-estimate-1-status": "X",
+        "triad-estimate-2-date": utc_datetime(2022, 1, 5, 17, 0),
+        "triad-estimate-2-gsp-kw": 0,
+        "triad-estimate-2-laf": 1,
+        "triad-estimate-2-msp-kw": 0,
+        "triad-estimate-2-status": "X",
+        "triad-estimate-3-date": utc_datetime(2022, 1, 20, 17, 0),
+        "triad-estimate-3-gsp-kw": 0,
+        "triad-estimate-3-laf": 1,
+        "triad-estimate-3-msp-kw": 0,
+        "triad-estimate-3-status": "X",
+        "triad-estimate-gbp": 0.0,
+        "triad-estimate-gsp-kw": 0.0,
+        "triad-estimate-months": 1,
+        "triad-estimate-rate": 0.0,
+    }