chellow 1751036864.0.0__tar.gz → 1751459327.0.0__tar.gz

{chellow-1751036864.0.0 → chellow-1751459327.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: chellow
-Version: 1751036864.0.0
+Version: 1751459327.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)

chellow-1751459327.0.0/VERSION

@@ -0,0 +1 @@
+__version__ = "1751459327.0.0"

{chellow-1751036864.0.0 → chellow-1751459327.0.0}/chellow/gas/bill_parser_bgs_xlsx.py

@@ -1,16 +1,16 @@
 import csv
+from collections import defaultdict
 from datetime import datetime as Datetime
 from decimal import Decimal, InvalidOperation
 from enum import Enum, auto
 from io import BytesIO
 
-from dateutil.relativedelta import relativedelta
 
 from openpyxl import load_workbook
 
 from werkzeug.exceptions import BadRequest
 
-from chellow.utils import to_ct, to_utc
+from chellow.utils import c_months_u, to_ct, to_utc
 
 
 class Title(Enum):
@@ -35,17 +35,14 @@ class Title(Enum):
 
 COLUMNS = {
     Title.CUSTOMER: ["Customer"],
-    Title.PRODUCT: ["Product"],
-    Title.BROKER_NAME: ["Broker Name"],
     Title.ACCOUNT: ["Account"],
     Title.MPRN: ["MPRN"],
     Title.BILL: ["Bill"],
-    Title.SUPPLY_ADDRESS: ["Supply Address"],
-    Title.BILL_DATE: ["Bill Date"],
-    Title.BILLING_PERIOD: ["Billing Period"],
-    Title.CHARGE_TYPE: ["Charge Type"],
-    Title.CHARGE_PERIOD_FROM: ["Charge Period From"],
-    Title.CHARGE_PERIOD_END: ["Charge Period End"],
+    Title.BILL_DATE: ["Bill Date", "BillDate"],
+    Title.BILLING_PERIOD: ["Billing Period", "BillingPeriod"],
+    Title.CHARGE_TYPE: ["Charge Type", "ChargeType"],
+    Title.CHARGE_PERIOD_FROM: ["Charge Period From", "ChargePeriodFrom"],
+    Title.CHARGE_PERIOD_END: ["Charge Period End", "ChargePeriodEnd"],
     Title.QUANTITY: ["Quantity"],
     Title.QUNIT: ["QUnit"],
     Title.CHARGE: ["Charge"],
@@ -127,34 +124,21 @@ def get_int(sheet, row, col):
         )
 
 
-"""
-def _bd_add(bd, el_name, val):
-    if el_name.split("-")[-1] in ("rate", "kva"):
-        if el_name not in bd:
-            bd[el_name] = set()
-        bd[el_name].add(val)
-    else:
-        if el_name not in bd:
-            bd[el_name] = 0
-        try:
-            bd[el_name] += val
-        except TypeError as e:
-            raise BadRequest(
-                f"Problem with element name {el_name} and value '{val}': {e}"
-            )
-"""
-
 ELEMENT_LOOKUP = {
+    "CCL": "ccl",
+    "Fixed Management Fee": "admin_fixed",
+    "Gas": "admin_variable",
     "Management Fee": "admin_variable",
+    "Metering and Data": "metering",
     "LDZ Customer Capacity": "dn_customer_capacity_fixed",
+    "LDZ Customer Fixed": "dn_customer_fixed",
     "LDZ System Capacity": "dn_system_capacity_fixed",
-    "LDZ System Commodity": "dn_system_capacity",
-    "Metering Charges": "metering",
+    "LDZ System Commodity": "dn_system_commodity",
     "NTS Exit Capacity (ECN)": "dn_ecn_fixed",
     "NTS SO Exit": "so_exit_commodity",
     "NTS TO Exit": "to_exit_commodity",
     "Unidentified Gas": "ug",
-    "WAP": "wap",
+    "WAP": "commodity",
 }
 
 QUNIT_LOOKUP = {
@@ -165,15 +149,15 @@ QUNIT_LOOKUP = {
 
 def _parse_row(bills, sheet, row, title_row):
     column_map = make_column_map(title_row)
-    mprn = get_value(sheet, row, column_map[Title.MPRN])
+    mprn_raw = get_value(sheet, row, column_map[Title.MPRN])
+    mprn = str(mprn_raw)
     reference = get_value(sheet, row, column_map[Title.BILL])
     account = get_value(sheet, row, column_map[Title.ACCOUNT])
     issue_date = get_date(sheet, row, column_map[Title.BILL_DATE])
-    start_date = get_date(sheet, row, column_map[Title.CHARGE_PERIOD_FROM])
-    finish_date = to_utc(
-        to_ct(get_date_naive(sheet, row, column_map[Title.CHARGE_PERIOD_END]))
-        + relativedelta(hours=23, minutes=30)
-    )
+    period_naive = get_date_naive(sheet, row, column_map[Title.BILLING_PERIOD])
+    start_date, finish_date = list(
+        c_months_u(start_year=period_naive.year, start_month=period_naive.month)
+    )[0]
 
     try:
         mprn_values = bills[mprn]
@@ -181,14 +165,9 @@ def _parse_row(bills, sheet, row, title_row):
         mprn_values = bills[mprn] = {}
 
     try:
-        start_date_values = mprn_values[start_date]
-    except KeyError:
-        start_date_values = mprn_values[start_date] = {}
-
-    try:
-        bill = start_date_values[finish_date]
+        bill = mprn_values[period_naive]
     except KeyError:
-        bill = start_date_values[finish_date] = {
+        bill = mprn_values[period_naive] = {
             "bill_type_code": "N",
             "mprn": mprn,
             "reference": reference,
@@ -197,10 +176,12 @@ def _parse_row(bills, sheet, row, title_row):
             "start_date": start_date,
             "finish_date": finish_date,
             "kwh": Decimal("0"),
+            "breakdown": defaultdict(int, {}),
             "net_gbp": Decimal("0.00"),
             "vat_gbp": Decimal("0.00"),
             "gross_gbp": Decimal("0.00"),
-            "breakdown": {},
+            "raw_lines": str([(c.value) for c in sheet[row]]),
+            "reads": [],
         }
 
     bd = bill["breakdown"]
@@ -209,38 +190,42 @@ def _parse_row(bills, sheet, row, title_row):
     qunit = get_value(sheet, row, column_map[Title.QUNIT])
     charge = get_dec(sheet, row, column_map[Title.CHARGE]) / Decimal("100")
     total = get_dec(sheet, row, column_map[Title.TOTAL])
-    if element_desc.startswith("20% VAT on "):
+    if element_desc == "VAT":
         bill["net_gbp"] += quantity
         bill["vat_gbp"] += total
         bill["gross_gbp"] += quantity + total
+        if "vat_rate" not in bd:
+            bd["vat_rate"] = set()
+        bd["vat_rate"].add(charge)
     else:
         element_name = ELEMENT_LOOKUP[element_desc]
-        if element_name == "admin_variable":
+        if element_name == "commodity":
             bill["kwh"] += quantity
-        bd[f"{element_name}_gbp"] = total
+        bd[f"{element_name}_gbp"] += total
         element_qunit = QUNIT_LOOKUP[qunit]
-        bd[f"{element_name}_{element_qunit}"] = quantity
-        bd[f"{element_name}_rate"] = charge
+        bd[f"{element_name}_{element_qunit}"] += quantity
+        rate_name = f"{element_name}_rate"
+        if rate_name not in bd:
+            bd[rate_name] = set()
+        bd[rate_name].add(charge)
 
 
 def _make_raw_bills(sheet):
     bills = {}
     rows = tuple(sheet.rows)
-    title_row = rows[1]
-    for row_index, row in enumerate(rows[2:], start=3):
+    title_row = rows[0]
+    for row_index, row in enumerate(rows[1:], start=2):
         val = row[0].value
         if val not in (None, ""):
             try:
                 _parse_row(bills, sheet, row_index, title_row)
             except BadRequest as e:
-                raise BadRequest(f"On row {row_index + 1}: {e.description}")
-    print("bills", bills)
+                raise BadRequest(f"On row {row_index}: {e.description}")
 
     raw_bills = []
     for mprn, mprn_values in bills.items():
-        for period_stat, period_start_values in mprn_values.items():
-            for period_finish, bill in period_start_values.items():
-                raw_bills.append(bill)
+        for period, raw_bill in mprn_values.items():
+            raw_bills.append(raw_bill)
 
     return raw_bills
 

{chellow-1751036864.0.0 → chellow-1751459327.0.0}/chellow/general_import.py

@@ -21,6 +21,7 @@ from chellow.models import (
     DtcMeterType,
     EnergisationStatus,
     Era,
+    GBill,
     GContract,
     GEra,
     GExitZone,
@@ -1151,6 +1152,77 @@ def general_import_g_bill(sess, action, vals, args):
             )
 
 
+def general_import_g_register_read(sess, action, vals, args):
+    if action == "insert":
+        contract_name = add_arg(args, "Supplier Contract Name", vals, 0)
+
+        g_contract = GContract.get_supplier_by_name(sess, contract_name)
+
+        batch_reference = add_arg(args, "Batch Reference", vals, 1)
+
+        g_batch = g_contract.get_g_batch_by_reference(sess, batch_reference)
+
+        mprn = add_arg(args, "MPRN", vals, 2)
+        g_supply = GSupply.get_by_mprn(sess, mprn)
+
+        bill_start_date_str = add_arg(args, "Bill Start Date", vals, 3)
+        bill_start_date = parse_hh_start(bill_start_date_str)
+
+        g_bill = sess.scalars(
+            select(GBill).where(
+                GBill.g_batch == g_batch,
+                GBill.g_supply == g_supply,
+                GBill.start_date == bill_start_date,
+            )
+        ).first()
+
+        if g_bill is None:
+            raise BadRequest(
+                f"Can't find a bill in batch {batch_reference} in contract "
+                f"{contract_name} with MPRN {mprn} starting at "
+                f"{hh_format(bill_start_date)}"
+            )
+
+        for i in range(4, len(vals), 10):
+            msn = add_arg(args, "Meter Serial Number", vals, i)
+            g_unit_code = add_arg(args, "Unit", vals, i + 1)
+            g_unit = GUnit.get_by_code(sess, g_unit_code)
+            correction_factor_str = add_arg(args, "Correction Factor", vals, i + 2)
+            correction_factor = Decimal(correction_factor_str)
+            calorific_value_str = add_arg(args, "Calorific Value", vals, i + 3)
+            calorific_value = Decimal(calorific_value_str)
+
+            prev_date_str = add_arg(args, "Previous Date", vals, i + 4)
+            prev_date = parse_hh_start(prev_date_str)
+            prev_value_str = add_arg(args, "Previous Value", vals, i + 5)
+            prev_value = Decimal(prev_value_str)
+
+            prev_type_str = add_arg(args, "Previous Type", vals, i + 6)
+            prev_type = GReadType.get_by_code(sess, prev_type_str)
+
+            pres_date_str = add_arg(args, "Present Date", vals, i + 7)
+            pres_date = parse_hh_start(pres_date_str)
+            pres_value_str = add_arg(args, "Present Value", vals, i + 8)
+            pres_value = Decimal(pres_value_str)
+
+            pres_type_str = add_arg(args, "Present Type", vals, i + 9)
+            pres_type = GReadType.get_by_code(sess, pres_type_str)
+
+            g_bill.insert_g_read(
+                sess,
+                msn,
+                g_unit,
+                correction_factor,
+                calorific_value,
+                prev_value,
+                prev_date,
+                prev_type,
+                pres_value,
+                pres_date,
+                pres_type,
+            )
+
+
 def general_import_register_read(sess, action, vals, args):
     if action == "insert":
         pass

{chellow-1751036864.0.0 → chellow-1751459327.0.0}/chellow/templates/g/bill_import.html

@@ -33,7 +33,7 @@
 	</ul>
 	
 	{% if failed_bills|length > 0 %}
-		<table>
+		<table class="sticky">
 			<caption>Failed Bills</caption>
 			<thead>
 				<tr>
@@ -77,7 +77,7 @@
 						<td>{{bill.net_gbp}}</td>
 						<td>{{bill.vat_gbp}}</td>
 						<td>{{bill.gross_gbp}}</td>
-						<td>{{bill.breakdown|dumps}}</td>
+						<td><pre>{{bill.breakdown|dumps}}</pre></td>
 						{% for read in bill.reads %}
 							<td>{{read.calorific_value}}</td>
 							<td>{{read.correction_factor}}</td>
@@ -97,7 +97,7 @@
 	{% endif %}
 
 	{% if successful_bills|length > 0 %}
-		<table>
+		<table class="sticky">
 			<caption>Successful Bills</caption>
 			<thead>
 				<tr>
@@ -141,7 +141,7 @@
 						<td>{{bill.net_gbp}}</td>
 						<td>{{bill.vat_gbp}}</td>
 						<td>{{bill.gross_gbp}}</td>
-						<td>{{bill.breakdown|dumps}}</td>
+						<td><pre>{{bill.breakdown|dumps}}</pre></td>
 						{% for g_read in bill.reads %}
 							<td>{{g_read.msn}}</td>
 							<td>{{g_read.prev_date|hh_format}}</td>

{chellow-1751036864.0.0 → chellow-1751459327.0.0}/chellow/templates/general_imports.html

@@ -511,6 +511,24 @@
 				<td>Present Value</td>
 				<td>Present Type)*</td>
 			</td>
+			<tr>
+				<td><em>insert</em></td>
+				<td><em>g_register_read</em></td>
+				<td>Contract Name</td>
+				<td>Batch Reference</td>
+				<td>MPRN</td>
+				<td>Bill Start Date (YYYY-MM-dd hh:mm)</td>
+				<td>(Meter Serial Number</td>
+				<td>Unit</td>
+				<td>Correction Factor</td>
+				<td>Calorific Value</td>
+				<td>Previous Date (YYYY-MM-dd hh:mm)</td>
+				<td>Previous Value</td>
+				<td>Previous Type</td>
+				<td>Present Date (YYYY-MM-dd hh:mm)</td>
+				<td>Present Value</td>
+				<td>Present Type)*</td>
+			</td>
 			<tr>
 				<td><em>insert</em></td>
 				<td><em>valid_mtc_llfc_ssc_pc</em></td>

{chellow-1751036864.0.0 → chellow-1751459327.0.0}/chellow/views.py

@@ -1696,7 +1696,8 @@ def report_run_row_get(row_id):
             toks = t.split("_")
             if toks[0] in ("covered", "virtual", "difference"):
                 tail = "_".join(toks[1:])
-                for element in elements.keys():
+                for element in sorted(elements.keys(), key=len, reverse=True):
+
                     table = elements[element]
                     elstr = f"{element}_"
                     if tail.startswith(elstr):
@@ -1705,6 +1706,7 @@ def report_run_row_get(row_id):
                             table["parts"].add(part)
                         if t.startswith("difference_") and t.endswith("_gbp"):
                             table["order"] = abs(values[t])
+                        break
 
         tables.sort(key=lambda t: t["order"], reverse=True)
         return render_template(

chellow-1751459327.0.0/test/gas/bill_parsers/test_bill_parser_bgs_xlsx.py

@@ -0,0 +1,138 @@
+from datetime import datetime as Datetime
+from decimal import Decimal
+
+from openpyxl import Workbook
+
+
+from chellow.gas.bill_parser_bgs_xlsx import _make_raw_bills, _parse_row
+from chellow.utils import utc_datetime
+
+
+def test_parse_row():
+    wb = Workbook()
+    ws = wb.active
+
+    rows = [
+        [
+            "Customer",
+            "Account",
+            "MPRN",
+            "Bill",
+            "SupplyAddress",
+            "BillDate",
+            "BillingPeriod",
+            "kWhs",
+            "ChargeType",
+            "ChargePeriodFrom",
+            "ChargePeriodEnd",
+            "Quantity",
+            "QUnit",
+            "Charge",
+            "CUnit",
+            "Total",
+        ],
+        [
+            "Zorin Industries",
+            69389283,
+            86762618,
+            584773269,
+            "Extinct Volcano",
+            Datetime(2025, 6, 13),
+            Datetime(2025, 5, 1),
+            0,
+            "Gas",
+            Datetime(2025, 5, 1),
+            Datetime(2025, 5, 31),
+            0,
+            "kWh",
+            0.5311,
+            "p/kWh",
+            0.00,
+        ],
+    ]
+
+    for row, row_values in enumerate(rows, start=1):
+        for column, value in enumerate(row_values, start=1):
+            ws.cell(row=row, column=column, value=value)
+
+    row = 2
+    title_row = tuple(ws.rows)[0]
+    bills = {}
+    _parse_row(bills, ws, row, title_row)
+
+
+def test_make_raw_bills():
+    wb = Workbook()
+    ws = wb.active
+
+    rows = [
+        [
+            "Customer",
+            "Account",
+            "MPRN",
+            "Bill",
+            "SupplyAddress",
+            "BillDate",
+            "BillingPeriod",
+            "kWhs",
+            "ChargeType",
+            "ChargePeriodFrom",
+            "ChargePeriodEnd",
+            "Quantity",
+            "QUnit",
+            "Charge",
+            "CUnit",
+            "Total",
+        ],
+        [
+            "Zorin Industries",
+            69389283,
+            86762618,
+            584773269,
+            "Extinct Volcano",
+            Datetime(2025, 6, 13),
+            Datetime(2025, 5, 1),
+            0,
+            "Gas",
+            Datetime(2025, 5, 1),
+            Datetime(2025, 5, 31),
+            0,
+            "kWh",
+            0.5311,
+            "p/kWh",
+            0.00,
+        ],
+    ]
+
+    for row, row_vals in enumerate(rows, start=1):
+        for column, value in enumerate(row_vals, start=1):
+            ws.cell(row=row, column=column, value=value)
+
+    bills = _make_raw_bills(ws)
+    expected_bills = [
+        {
+            "bill_type_code": "N",
+            "mprn": "86762618",
+            "reference": 584773269,
+            "account": 69389283,
+            "issue_date": utc_datetime(2025, 6, 12, 23, 0),
+            "start_date": utc_datetime(2025, 4, 30, 23, 0),
+            "finish_date": utc_datetime(2025, 5, 31, 22, 30),
+            "kwh": Decimal("0"),
+            "breakdown": {
+                "admin_variable_gbp": Decimal("0.0"),
+                "admin_variable_kwh": Decimal("0"),
+                "admin_variable_rate": {Decimal("0.005311")},
+            },
+            "net_gbp": Decimal("0.00"),
+            "vat_gbp": Decimal("0.00"),
+            "gross_gbp": Decimal("0.00"),
+            "raw_lines": "['Zorin Industries', 69389283, 86762618, 584773269, 'Extinct "
+            "Volcano', datetime.datetime(2025, 6, 13, 0, 0), "
+            "datetime.datetime(2025, 5, 1, 0, 0), 0, 'Gas', "
+            "datetime.datetime(2025, 5, 1, 0, 0), datetime.datetime(2025, "
+            "5, 31, 0, 0), 0, 'kWh', 0.5311, 'p/kWh', 0.0]",
+            "reads": [],
+        }
+    ]
+    assert bills == expected_bills

{chellow-1751036864.0.0 → chellow-1751459327.0.0}/test/test_general_import.py

@@ -8,12 +8,14 @@ from chellow.general_import import (
     general_import_g_batch,
     general_import_g_bill,
     general_import_g_era,
+    general_import_g_register_read,
     general_import_g_supply,
     general_import_llfc,
     general_import_site,
     general_import_supply,
 )
 from chellow.models import (
+    BillType,
     Comm,
     Contract,
     Cop,
@@ -62,15 +64,16 @@ def test_general_import_g_batch(mocker):
     general_import_g_batch(sess, action, vals, args)
 
 
-def test_general_import_g_bill(sess):
+def test_general_import_g_register_reads(sess):
     vf = to_utc(ct_datetime(2000, 1, 1))
     site_code = "22488"
     site = Site.insert(sess, site_code, "Water Works")
     g_dn = GDn.insert(sess, "EE", "East of England")
     g_ldz = g_dn.insert_g_ldz(sess, "EA")
     g_exit_zone = g_ldz.insert_g_exit_zone(sess, "EA1")
+    g_unit_code = "M3"
     insert_g_units(sess)
-    g_unit_M3 = GUnit.get_by_code(sess, "M3")
+    g_unit_M3 = GUnit.get_by_code(sess, g_unit_code)
     participant = Participant.insert(sess, "CALB", "AK Industries")
     market_role_Z = MarketRole.insert(sess, "Z", "non core")
     participant.insert_party(sess, market_role_Z, "None core", vf, None, None)
@@ -80,7 +83,7 @@ def test_general_import_g_bill(sess):
     g_reading_frequency_M = GReadingFrequency.get_by_code(sess, "M")
     mprn = "87614362"
     msn = "hgeu8rhg"
-    site.insert_g_supply(
+    g_supply = site.insert_g_supply(
         sess,
         mprn,
         "main",
@@ -97,8 +100,26 @@ def test_general_import_g_bill(sess):
         1,
     )
     batch_name = "b1"
-    g_contract.insert_g_batch(sess, batch_name, "batch 1")
+    g_batch = g_contract.insert_g_batch(sess, batch_name, "batch 1")
     insert_bill_types(sess)
+    bill_type_N = BillType.get_by_code(sess, "N")
+    insert_g_read_types(sess)
+    g_batch.insert_g_bill(
+        sess,
+        g_supply,
+        bill_type_N,
+        "55h883",
+        "dhgh883",
+        utc_datetime(2019, 4, 3),
+        to_utc(ct_datetime(2015, 9, 1)),
+        utc_datetime(2015, 9, 30, 22, 30),
+        Decimal("45"),
+        Decimal("12.40"),
+        Decimal("1.20"),
+        Decimal("14.52"),
+        "",
+        {},
+    )
     sess.commit()
 
     action = "insert"
@@ -106,20 +127,20 @@ def test_general_import_g_bill(sess):
         g_contract_name,
         batch_name,
         mprn,
-        "2019-09-08 00:00",
-        "2019-10-01 00:00",
-        "2019-10-31 23:30",
+        "2015-09-01 00:00",
+        msn,
+        g_unit_code,
         "0.00",
         "0.00",
+        "2015-09-01 00:00",
         "0.00",
-        "77hwgtlll",
-        "7876hrwlju",
-        "N",
-        "{}",
-        "0",
+        "A",
+        "2015-10-01 00:00",
+        "10.00",
+        "A",
     ]
     args = []
-    general_import_g_bill(sess, action, vals, args)
+    general_import_g_register_read(sess, action, vals, args)
 
 
 def test_general_import_g_bill_reads(sess):

chellow-1751036864.0.0/VERSION

@@ -1 +0,0 @@
-__version__ = "1751036864.0.0"