chellow 1753106469.0.0__tar.gz → 1753180539.0.0__tar.gz

{chellow-1753106469.0.0 → chellow-1753180539.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: chellow
-Version: 1753106469.0.0
+Version: 1753180539.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)

chellow-1753180539.0.0/VERSION

@@ -0,0 +1 @@
+__version__ = "1753180539.0.0"

{chellow-1753106469.0.0 → chellow-1753180539.0.0}/chellow/e/elexon.py

@@ -62,7 +62,12 @@ def run_import(sess, log, set_progress, scripting_key):
     s = requests.Session()
     s.verify = False
 
-    for mod_name in ("chellow.e.system_price", "chellow.e.tlms", "chellow.e.rcrc"):
+    for mod_name in (
+        "chellow.e.system_price",
+        "chellow.e.tlms",
+        "chellow.e.rcrc",
+        "chellow.e.lafs",
+    ):
         mod = import_module(mod_name)
         mod.elexon_import(sess, log, set_progress, s, scripting_key)
 

chellow-1753106469.0.0/chellow/e/laf_import.py → chellow-1753180539.0.0/chellow/e/lafs.py

@@ -4,10 +4,10 @@ from decimal import Decimal
 from io import BytesIO, StringIO
 from zipfile import ZipFile
 
-# from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy import text
 
 from werkzeug.exceptions import BadRequest
+from werkzeug.http import parse_options_header
 
 from chellow.models import Contract, Party
 from chellow.rate_server import download
@@ -30,7 +30,12 @@ DO UPDATE SET (llfc_id, timestamp, value) =
     )
     fname = name_list[0]
     csv_file = StringIO(zip_file.read(fname).decode("utf-8"))
-    csv_dt = to_utc(to_ct(Datetime.strptime(fname[-12:-4], "%Y%m%d")))
+    try:
+        csv_dt = to_utc(to_ct(Datetime.strptime(fname[-12:-4], "%Y%m%d")))
+    except ValueError as e:
+        log(f"Can't parse the date in the file name {fname} {e}")
+        return
+
     for llfc_ids, timestamps, values in laf_days(
         sess, log, set_progress, csv_file, csv_dt
     ):
@@ -110,6 +115,67 @@ LAF_START = "llf"
 LAF_END = "ptf.zip"
 
 
+def elexon_import(sess, log, set_progress, s, scripting_key):
+    log(
+        "Starting to check for new LAF files for participant codes in "
+        "configuration.properties.laf_importer.participant_codes"
+    )
+    conf = Contract.get_non_core_by_name(sess, "configuration")
+    props = conf.make_properties()
+    state = conf.make_state()
+
+    try:
+        laf_state = state["laf_importer"]
+    except KeyError:
+        laf_state = state["laf_importer"] = {}
+
+    url = "https://downloads.elexonportal.co.uk/svallf/download"
+
+    laf_props = props.get("laf_importer", {})
+    for participant_code in laf_props.get("participant_codes", []):
+        params = {"key": scripting_key, "ldso": participant_code, "format": "PTF"}
+
+        log(
+            f"Downloading from {url}?key={scripting_key}&ldso={participant_code}&"
+            f"format=PTF"
+        )
+        sess.rollback()  # Avoid long-running transactions
+        res = s.get(url, params=params)
+        log(f"Received {res.status_code} {res.reason}")
+        res.raise_for_status()
+        try:
+            cd_header = res.headers["Content-Disposition"]
+        except KeyError:
+            log("No Content-Disposition header found, so skipping.")
+            continue
+
+        cd = parse_options_header(cd_header)
+        file_name = cd[1]["filename"]
+
+        # File name llfetcl20220401ptf.zip
+        if not file_name.startswith(LAF_START):
+            raise BadRequest(f"A laf file must begin with '{LAF_START}'")
+        if not file_name.endswith(LAF_END):
+            raise BadRequest(f"A laf file must end with '{LAF_END}'")
+
+        participant_code = file_name[3:7]
+        timestamp = file_name[7:15]
+        log(f"File name {file_name}")
+
+        if timestamp > laf_state.get(participant_code, ""):
+            data = res.content
+            _process(sess, log, set_progress, BytesIO(data))
+            laf_state[participant_code] = timestamp
+            conf.update_state(state)
+            sess.commit()
+            log(f"Processed file {file_name}.")
+        else:
+            log(f"Already loaded {file_name}.")
+
+    log("Finished LAF files")
+    sess.commit()
+
+
 def find_participant_entries(paths, laf_state):
     participant_entries = {}
     for path, url in paths:

{chellow-1753106469.0.0 → chellow-1753180539.0.0}/chellow/e/views.py

@@ -2344,30 +2344,29 @@ def hh_datum_edit_post(datum_id):
 @e.route("/lafs")
 def lafs_get():
     llfc_id = req_int("llfc_id")
-    year = req_int("year")
-    month = req_int("month")
-
+    month_start = req_date("timestamp", resolution="month")
+    month_start_ct = to_ct(month_start)
     llfc = Llfc.get_by_id(g.sess, llfc_id)
     dno = llfc.dno
 
-    start_date, finish_date = next(
-        c_months_u(start_year=year, start_month=month, months=1)
-    )
-    lafs = (
-        g.sess.execute(
-            select(Laf)
-            .where(
-                Laf.llfc == llfc,
-                Laf.timestamp >= start_date,
-                Laf.timestamp <= finish_date,
-            )
-            .order_by(Laf.timestamp)
+    month_start, month_finish = next(
+        c_months_u(
+            start_year=month_start_ct.year, start_month=month_start_ct.month, months=1
         )
-        .scalars()
-        .all()
     )
+    lafs = g.sess.execute(
+        select(Laf)
+        .where(
+            Laf.llfc == llfc,
+            Laf.timestamp >= month_start,
+            Laf.timestamp <= month_finish,
+        )
+        .order_by(Laf.timestamp)
+    ).scalars()
 
-    return render_template("lafs.html", dno=dno, llfc=llfc, lafs=lafs)
+    return render_template(
+        "lafs.html", dno=dno, llfc=llfc, lafs=lafs, month_start=month_start
+    )
 
 
 @e.route("/lcc")

{chellow-1753106469.0.0 → chellow-1753180539.0.0}/chellow/gas/bill_parser_bgs_xlsx.py

@@ -5,12 +5,13 @@ from decimal import Decimal, InvalidOperation
 from enum import Enum, auto
 from io import BytesIO
 
+from dateutil.relativedelta import relativedelta
 
 from openpyxl import load_workbook
 
 from werkzeug.exceptions import BadRequest
 
-from chellow.utils import c_months_u, to_ct, to_utc
+from chellow.utils import to_ct, to_utc
 
 
 class Title(Enum):
@@ -151,13 +152,13 @@ def _parse_row(bills, sheet, row, title_row):
     column_map = make_column_map(title_row)
     mprn_raw = get_value(sheet, row, column_map[Title.MPRN])
     mprn = str(mprn_raw)
-    reference = get_value(sheet, row, column_map[Title.BILL])
+    reference = str(get_value(sheet, row, column_map[Title.BILL]))
     account = get_value(sheet, row, column_map[Title.ACCOUNT])
     issue_date = get_date(sheet, row, column_map[Title.BILL_DATE])
-    period_naive = get_date_naive(sheet, row, column_map[Title.BILLING_PERIOD])
-    start_date, finish_date = list(
-        c_months_u(start_year=period_naive.year, start_month=period_naive.month)
-    )[0]
+    charge_period_from = get_date(sheet, row, column_map[Title.CHARGE_PERIOD_FROM])
+    charge_period_end_naive = get_date_naive(
+        sheet, row, column_map[Title.CHARGE_PERIOD_END]
+    )
 
     try:
         mprn_values = bills[mprn]
@@ -165,16 +166,18 @@ def _parse_row(bills, sheet, row, title_row):
         mprn_values = bills[mprn] = {}
 
     try:
-        bill = mprn_values[period_naive]
+        bill = mprn_values[reference]
     except KeyError:
-        bill = mprn_values[period_naive] = {
-            "bill_type_code": "N",
+        bill = mprn_values[reference] = {
+            "bill_type_code": "W" if reference.startswith("CR") else "N",
             "mprn": mprn,
             "reference": reference,
             "account": account,
             "issue_date": issue_date,
-            "start_date": start_date,
-            "finish_date": finish_date,
+            "start_date": charge_period_from,
+            "finish_date": to_utc(
+                to_ct(charge_period_end_naive) + relativedelta(hours=23, minutes=30)
+            ),
             "kwh": Decimal("0"),
             "breakdown": defaultdict(int, {}),
             "net_gbp": Decimal("0.00"),
@@ -190,6 +193,8 @@ def _parse_row(bills, sheet, row, title_row):
     qunit = get_value(sheet, row, column_map[Title.QUNIT])
     charge = get_dec(sheet, row, column_map[Title.CHARGE]) / Decimal("100")
     total = get_dec(sheet, row, column_map[Title.TOTAL])
+    if bill["bill_type_code"] == "W":
+        quantity = quantity * -1
     if element_desc == "VAT":
         bill["net_gbp"] += quantity
         bill["vat_gbp"] += total

{chellow-1753106469.0.0 → chellow-1753180539.0.0}/chellow/rate_server.py

@@ -87,8 +87,8 @@ def run_import(sess, log, set_progress):
         "chellow.e.bsuos",
         "chellow.e.ccl",
         "chellow.e.dno_rate_parser",
-        "chellow.e.laf_import",
         "chellow.e.triad",
+        "chellow.e.lafs",
         "chellow.gas.ccl",
         "chellow.gas.dn_rate_parser",
     ):

{chellow-1753106469.0.0 → chellow-1753180539.0.0}/chellow/templates/e/lafs.html

@@ -17,7 +17,7 @@
 		<fieldset>
 			<legend>Show data</legend>	
 			<input type="hidden" name="llfc_id" value="{{llfc.id}}">
-			<label>Month</label> {{input_date(None, None, 'month')}}
+			<label>Month</label> {{input_date(resolution='month')}}
 			<input type="submit" value="Show">
 		</fieldset>
 	</form>

{chellow-1753106469.0.0 → chellow-1753180539.0.0}/chellow/templates/e/llfc.html

@@ -45,7 +45,7 @@
 		<tr>
 			<th>Loss Adjustment Factors</th>
 			<td>
-				<a href="/e/lafs?llfc_id={{llfc.id}}&amp;year={{now.year}}&amp;month={{now.month}}">LAFs</a>
+				<a href="/e/lafs?llfc_id={{llfc.id}}&amp;timestamp_year={{now.year}}&amp;timestamp_month={{now.month}}">LAFs</a>
 			</td>
 		</tr>
 	</table>

{chellow-1753106469.0.0 → chellow-1753180539.0.0}/chellow/views.py

@@ -61,7 +61,6 @@ import chellow.e.bill_importer
 import chellow.e.bsuos
 import chellow.e.computer
 import chellow.e.hh_importer
-import chellow.e.laf_import
 import chellow.e.lcc
 import chellow.e.mdd_importer
 import chellow.e.rcrc

{chellow-1753106469.0.0 → chellow-1753180539.0.0}/test/e/test_e_views.py

@@ -2023,6 +2023,37 @@ def test_get_era_bundles_bill_in_correct_era(sess, client):
     assert len(bundles[1]["imp_bills"]["bill_dicts"]) == 1
 
 
+def test_lafs_get(sess, client):
+    vf = to_utc(ct_datetime(2000, 1, 1))
+
+    market_role_Z = MarketRole.get_by_code(sess, "Z")
+    participant = Participant.insert(sess, "CALB", "AK Industries")
+    participant.insert_party(sess, market_role_Z, "None core", vf, None, None)
+    market_role_R = MarketRole.insert(sess, "R", "Distributor")
+    dno = participant.insert_party(sess, market_role_R, "WPD", vf, None, "22")
+    insert_voltage_levels(sess)
+    voltage_level = VoltageLevel.get_by_code(sess, "HV")
+    llfc = dno.insert_llfc(
+        sess,
+        "510",
+        "PC 5-8 & HH HV",
+        voltage_level,
+        False,
+        True,
+        vf,
+        None,
+    )
+    sess.commit()
+
+    query_string = {
+        "llfc_id": llfc.id,
+        "timestamp_year": 2020,
+        "timestamp_month": 1,
+    }
+    response = client.get("/e/lafs", query_string=query_string)
+    match(response, 200)
+
+
 def test_llfc_eidt_post(sess, client):
     vf = to_utc(ct_datetime(2000, 1, 1))
 

chellow-1753106469.0.0/test/e/test_laf_importer.py → chellow-1753180539.0.0/test/e/test_lafs.py

@@ -4,7 +4,7 @@ from zipfile import BadZipFile, ZipFile
 
 import pytest
 
-from chellow.e.laf_import import _process, find_participant_entries, laf_days
+from chellow.e.lafs import _process, find_participant_entries, laf_days
 from chellow.models import (
     Contract,
     MarketRole,

chellow-1753180539.0.0/test/gas/bill_parsers/test_bill_parser_bgs_xlsx.py

@@ -0,0 +1,296 @@
+from collections import defaultdict
+from datetime import datetime as Datetime
+from decimal import Decimal
+
+from openpyxl import Workbook
+
+
+from chellow.gas.bill_parser_bgs_xlsx import _make_raw_bills, _parse_row
+from chellow.utils import ct_datetime, to_utc, utc_datetime
+
+
+def test_parse_row():
+    wb = Workbook()
+    ws = wb.active
+
+    rows = [
+        [
+            "Customer",
+            "Account",
+            "MPRN",
+            "Bill",
+            "SupplyAddress",
+            "BillDate",
+            "BillingPeriod",
+            "kWhs",
+            "ChargeType",
+            "ChargePeriodFrom",
+            "ChargePeriodEnd",
+            "Quantity",
+            "QUnit",
+            "Charge",
+            "CUnit",
+            "Total",
+        ],
+        [
+            "Zorin Industries",
+            69389283,
+            86762618,
+            584773269,
+            "Extinct Volcano",
+            Datetime(2025, 6, 13),
+            Datetime(2025, 5, 1),
+            0,
+            "Gas",
+            Datetime(2025, 5, 1),
+            Datetime(2025, 5, 31),
+            0,
+            "kWh",
+            0.5311,
+            "p/kWh",
+            0.00,
+        ],
+    ]
+
+    for row, row_values in enumerate(rows, start=1):
+        for column, value in enumerate(row_values, start=1):
+            ws.cell(row=row, column=column, value=value)
+
+    row = 2
+    title_row = tuple(ws.rows)[0]
+    bills = {}
+    _parse_row(bills, ws, row, title_row)
+
+
+def test_make_raw_bills():
+    wb = Workbook()
+    ws = wb.active
+
+    rows = [
+        [
+            "Customer",
+            "Account",
+            "MPRN",
+            "Bill",
+            "SupplyAddress",
+            "BillDate",
+            "BillingPeriod",
+            "kWhs",
+            "ChargeType",
+            "ChargePeriodFrom",
+            "ChargePeriodEnd",
+            "Quantity",
+            "QUnit",
+            "Charge",
+            "CUnit",
+            "Total",
+        ],
+        [
+            "Zorin Industries",
+            69389283,
+            86762618,
+            584773269,
+            "Extinct Volcano",
+            Datetime(2025, 6, 13),
+            Datetime(2025, 5, 1),
+            0,
+            "Gas",
+            Datetime(2025, 5, 1),
+            Datetime(2025, 5, 31),
+            0,
+            "kWh",
+            0.5311,
+            "p/kWh",
+            0.00,
+        ],
+    ]
+
+    for row, row_vals in enumerate(rows, start=1):
+        for column, value in enumerate(row_vals, start=1):
+            ws.cell(row=row, column=column, value=value)
+
+    bills = _make_raw_bills(ws)
+    expected_bills = [
+        {
+            "bill_type_code": "N",
+            "mprn": "86762618",
+            "reference": "584773269",
+            "account": 69389283,
+            "issue_date": utc_datetime(2025, 6, 12, 23, 0),
+            "start_date": utc_datetime(2025, 4, 30, 23, 0),
+            "finish_date": utc_datetime(2025, 5, 31, 22, 30),
+            "kwh": Decimal("0"),
+            "breakdown": {
+                "admin_variable_gbp": Decimal("0.0"),
+                "admin_variable_kwh": Decimal("0"),
+                "admin_variable_rate": {Decimal("0.005311")},
+            },
+            "net_gbp": Decimal("0.00"),
+            "vat_gbp": Decimal("0.00"),
+            "gross_gbp": Decimal("0.00"),
+            "raw_lines": "['Zorin Industries', 69389283, 86762618, 584773269, 'Extinct "
+            "Volcano', datetime.datetime(2025, 6, 13, 0, 0), "
+            "datetime.datetime(2025, 5, 1, 0, 0), 0, 'Gas', "
+            "datetime.datetime(2025, 5, 1, 0, 0), datetime.datetime(2025, "
+            "5, 31, 0, 0), 0, 'kWh', 0.5311, 'p/kWh', 0.0]",
+            "reads": [],
+        }
+    ]
+    assert bills == expected_bills
+
+
+def test_make_raw_bills_part_month():
+    wb = Workbook()
+    ws = wb.active
+
+    rows = [
+        [
+            "Customer",
+            "Account",
+            "MPRN",
+            "Bill",
+            "SupplyAddress",
+            "BillDate",
+            "BillingPeriod",
+            "kWhs",
+            "ChargeType",
+            "ChargePeriodFrom",
+            "ChargePeriodEnd",
+            "Quantity",
+            "QUnit",
+            "Charge",
+            "CUnit",
+            "Total",
+        ],
+        [
+            "Zorin Industries",
+            69389283,
+            86762618,
+            584773269,
+            "Extinct Volcano",
+            Datetime(2025, 6, 13),
+            Datetime(2025, 5, 1),
+            0,
+            "Gas",
+            Datetime(2025, 5, 1),
+            Datetime(2025, 5, 15),
+            0,
+            "kWh",
+            0.5311,
+            "p/kWh",
+            0.00,
+        ],
+    ]
+
+    for row, row_vals in enumerate(rows, start=1):
+        for column, value in enumerate(row_vals, start=1):
+            ws.cell(row=row, column=column, value=value)
+
+    bills = _make_raw_bills(ws)
+    expected_bills = [
+        {
+            "bill_type_code": "N",
+            "mprn": "86762618",
+            "reference": "584773269",
+            "account": 69389283,
+            "issue_date": utc_datetime(2025, 6, 12, 23, 0),
+            "start_date": to_utc(ct_datetime(2025, 5, 1, 0, 0)),
+            "finish_date": to_utc(ct_datetime(2025, 5, 15, 23, 30)),
+            "kwh": Decimal("0"),
+            "breakdown": defaultdict(
+                int,
+                {
+                    "admin_variable_gbp": Decimal("0.0"),
+                    "admin_variable_kwh": Decimal("0"),
+                    "admin_variable_rate": {Decimal("0.005311")},
+                },
+            ),
+            "net_gbp": Decimal("0.00"),
+            "vat_gbp": Decimal("0.00"),
+            "gross_gbp": Decimal("0.00"),
+            "raw_lines": "['Zorin Industries', 69389283, 86762618, 584773269, 'Extinct "
+            "Volcano', datetime.datetime(2025, 6, 13, 0, 0), "
+            "datetime.datetime(2025, 5, 1, 0, 0), 0, 'Gas', "
+            "datetime.datetime(2025, 5, 1, 0, 0), datetime.datetime(2025, "
+            "5, 15, 0, 0), 0, 'kWh', 0.5311, 'p/kWh', 0.0]",
+            "reads": [],
+        }
+    ]
+    assert bills == expected_bills
+
+
+def test_make_raw_bills_credit():
+    wb = Workbook()
+    ws = wb.active
+
+    rows = [
+        [
+            "Customer",
+            "Account",
+            "MPRN",
+            "Bill",
+            "SupplyAddress",
+            "BillDate",
+            "BillingPeriod",
+            "kWhs",
+            "ChargeType",
+            "ChargePeriodFrom",
+            "ChargePeriodEnd",
+            "Quantity",
+            "QUnit",
+            "Charge",
+            "CUnit",
+            "Total",
+        ],
+        [
+            "Zorin Industries",
+            69389283,
+            86762618,
+            "CR584773269",
+            "Extinct Volcano",
+            Datetime(2025, 6, 13),
+            Datetime(2025, 5, 1),
+            0,
+            "Gas",
+            Datetime(2025, 5, 1),
+            Datetime(2025, 5, 31),
+            10,
+            "kWh",
+            0.5311,
+            "p/kWh",
+            -5.00,
+        ],
+    ]
+
+    for row, row_vals in enumerate(rows, start=1):
+        for column, value in enumerate(row_vals, start=1):
+            ws.cell(row=row, column=column, value=value)
+
+    bills = _make_raw_bills(ws)
+    expected_bills = [
+        {
+            "bill_type_code": "W",
+            "mprn": "86762618",
+            "reference": "CR584773269",
+            "account": 69389283,
+            "issue_date": utc_datetime(2025, 6, 12, 23, 0),
+            "start_date": utc_datetime(2025, 4, 30, 23, 0),
+            "finish_date": utc_datetime(2025, 5, 31, 22, 30),
+            "kwh": Decimal("0"),
+            "breakdown": {
+                "admin_variable_gbp": Decimal("-5.00"),
+                "admin_variable_kwh": Decimal("-10"),
+                "admin_variable_rate": {Decimal("0.005311")},
+            },
+            "net_gbp": Decimal("0.00"),
+            "vat_gbp": Decimal("0.00"),
+            "gross_gbp": Decimal("0.00"),
+            "raw_lines": "['Zorin Industries', 69389283, 86762618, 'CR584773269', "
+            "'Extinct Volcano', datetime.datetime(2025, 6, 13, 0, 0), "
+            "datetime.datetime(2025, 5, 1, 0, 0), 0, 'Gas', "
+            "datetime.datetime(2025, 5, 1, 0, 0), datetime.datetime(2025, "
+            "5, 31, 0, 0), 10, 'kWh', 0.5311, 'p/kWh', -5.0]",
+            "reads": [],
+        }
+    ]
+    assert bills == expected_bills

{chellow-1753106469.0.0 → chellow-1753180539.0.0}/test/test_rate_server.py

@@ -14,7 +14,7 @@ def test_run_import(mocker, sess):
 
     mocker.patch("chellow.e.bsuos.rate_server_import")
     mocker.patch("chellow.e.dno_rate_parser.rate_server_import")
-    mocker.patch("chellow.e.laf_import.rate_server_import")
+    mocker.patch("chellow.e.lafs.rate_server_import")
     mocker.patch("chellow.e.mdd_importer.rate_server_import")
     mocker.patch("chellow.gas.dn_rate_parser.rate_server_import")
 
@@ -52,7 +52,7 @@ def test_run_import_lowercase_readme(mocker, sess):
 
     mocker.patch("chellow.e.bsuos.rate_server_import")
     mocker.patch("chellow.e.dno_rate_parser.rate_server_import")
-    mocker.patch("chellow.e.laf_import.rate_server_import")
+    mocker.patch("chellow.e.lafs.rate_server_import")
     mocker.patch("chellow.e.mdd_importer.rate_server_import")
     mocker.patch("chellow.gas.dn_rate_parser.rate_server_import")
 

chellow-1753106469.0.0/VERSION

@@ -1 +0,0 @@
-__version__ = "1753106469.0.0"