chellow 1744720514.0.0__tar.gz → 1745911639.0.0__tar.gz

{chellow-1744720514.0.0 → chellow-1745911639.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: chellow
-Version: 1744720514.0.0
+Version: 1745911639.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)

chellow-1745911639.0.0/VERSION

@@ -0,0 +1 @@
+__version__ = "1745911639.0.0"

{chellow-1744720514.0.0 → chellow-1745911639.0.0}/chellow/e/bill_parsers/mm.py

@@ -59,7 +59,7 @@ def _handle_0100(headers, pre_record, record):
     headers["issue_date"] = issue_date
     if late_payment is not None:
         headers["late_payment"] = late_payment
-    headers["account"] = pre_record[33:41]
+    headers["account"] = pre_record[43:52]
     headers["reference"] = pre_record[41:46]
     headers["kwh"] = Decimal("0")
     headers["breakdown"] = defaultdict(int, {"vat": {}})

{chellow-1744720514.0.0 → chellow-1745911639.0.0}/chellow/e/bill_parsers/nonsettlement_dc_stark_xlsx.py

@@ -4,6 +4,7 @@ from openpyxl import load_workbook
 
 from werkzeug.exceptions import BadRequest
 
+from chellow.e.computer import hh_rate
 from chellow.models import Session
 from chellow.utils import ct_datetime, parse_mpan_core, to_utc
 
@@ -16,12 +17,6 @@ def get_start_date(title_row, row, name):
     return to_utc(get_ct_date(title_row, row, name))
 
 
-def get_finish_date(title_row, row, name):
-    d = get_ct_date(title_row, row, name)
-
-    return to_utc(ct_datetime(d.year, d.month, d.day, 23, 30))
-
-
 def get_value(title_row, row, name):
     idx = None
     name = name.strip().lower()
@@ -61,9 +56,6 @@ def get_int(title_row, row, name):
     return int(get_value(title_row, row, name))
 
 
-METER_RATE = Decimal("60.00")
-
-
 class Parser:
     def __init__(self, f):
         self.book = load_workbook(f)
@@ -86,6 +78,7 @@ class Parser:
 
     def make_raw_bills(self):
         row_index = None
+        caches = {}
         try:
             with Session() as sess:
                 bills = []
@@ -101,14 +94,30 @@ class Parser:
                     mpan_core = parse_mpan_core(
                         str(get_int(title_row, row, "mpan ref"))
                     )
-                    start_date = get_start_date(title_row, row, "start")
+                    start_date_ct = get_ct_date(title_row, row, "start")
+                    start_date = to_utc(start_date_ct)
                     issue_date = start_date
-                    finish_date = get_finish_date(title_row, row, "end")
+                    finish_date_ct = get_ct_date(title_row, row, "end")
+                    finish_date = to_utc(
+                        ct_datetime(
+                            finish_date_ct.year,
+                            finish_date_ct.month,
+                            finish_date_ct.day,
+                            23,
+                            30,
+                        )
+                    )
                     check = get_str(title_row, row, "check")
                     if check != "Billed":
                         continue
-
-                    net = METER_RATE / 12
+                    rates = hh_rate(sess, caches, 0, start_date)
+                    meter_rate = rates["annual_rates"]["non_settlement"]["*"]["IP"][
+                        "*"
+                    ]["gbp_per_meter"]
+                    months = (finish_date_ct.year - start_date_ct.year) * 12 + (
+                        finish_date_ct.month - start_date_ct.month + 1
+                    )
+                    net = round(Decimal(float(meter_rate) / 12 * months), 2)
                     vat = round(net * Decimal("0.2"), 2)
 
                     breakdown = {
@@ -116,7 +125,8 @@ class Parser:
                         "cop": ["5"],
                         "settlement-status": ["non_settlement"],
                         "msn": [msn],
-                        "meter-rate": [METER_RATE],
+                        "meter-rate": [meter_rate],
+                        "months": months,
                         "meter-gbp": net,
                     }
 

{chellow-1744720514.0.0 → chellow-1745911639.0.0}/chellow/e/dno_rate_parser.py

@@ -650,15 +650,15 @@ def update_vls(sess, logger, vls, dno_code, fy_start, rs_finish):
         llfc = sess.execute(q).scalar_one_or_none()
 
         if llfc is None:
-            raise BadRequest(
+            logger(
                 f"There is no LLFC with the code '{vl_code}' associated with the DNO "
-                f"{dno.code} from {hh_format(fy_start)} to {hh_format(rs_finish)}."
+                f"{dno.dno_code} from {hh_format(fy_start)} to {hh_format(rs_finish)}."
             )
-
-        vl_voltage_level = VoltageLevel.get_by_code(sess, vl["voltage_level"])
-        llfc.voltage_level = vl_voltage_level
-
-        llfc.is_substation = vl["is_substation"]
-        if sess.is_modified(llfc):
-            logger(f"Updated LLFC {llfc.code} of DNO {dno_code}")
-            sess.flush()
+        else:
+            vl_voltage_level = VoltageLevel.get_by_code(sess, vl["voltage_level"])
+            llfc.voltage_level = vl_voltage_level
+
+            llfc.is_substation = vl["is_substation"]
+            if sess.is_modified(llfc):
+                logger(f"Updated LLFC {llfc.code} of DNO {dno_code}")
+                sess.flush()

chellow-1745911639.0.0/chellow/gas/bill_parser_bgs_xlsx.py

@@ -0,0 +1,271 @@
+import csv
+from datetime import datetime as Datetime
+from decimal import Decimal, InvalidOperation
+from enum import Enum, auto
+from io import BytesIO
+
+from dateutil.relativedelta import relativedelta
+
+from openpyxl import load_workbook
+
+from werkzeug.exceptions import BadRequest
+
+from chellow.utils import to_ct, to_utc
+
+
+class Title(Enum):
+    CUSTOMER = auto()
+    PRODUCT = auto()
+    BROKER_NAME = auto()
+    ACCOUNT = auto()
+    MPRN = auto()
+    BILL = auto()
+    SUPPLY_ADDRESS = auto()
+    BILL_DATE = auto()
+    BILLING_PERIOD = auto()
+    CHARGE_TYPE = auto()
+    CHARGE_PERIOD_FROM = auto()
+    CHARGE_PERIOD_END = auto()
+    QUANTITY = auto()
+    QUNIT = auto()
+    CHARGE = auto()
+    CUNIT = auto()
+    TOTAL = auto()
+
+
+COLUMNS = {
+    Title.CUSTOMER: ["Customer"],
+    Title.PRODUCT: ["Product"],
+    Title.BROKER_NAME: ["Broker Name"],
+    Title.ACCOUNT: ["Account"],
+    Title.MPRN: ["MPRN"],
+    Title.BILL: ["Bill"],
+    Title.SUPPLY_ADDRESS: ["Supply Address"],
+    Title.BILL_DATE: ["Bill Date"],
+    Title.BILLING_PERIOD: ["Billing Period"],
+    Title.CHARGE_TYPE: ["Charge Type"],
+    Title.CHARGE_PERIOD_FROM: ["Charge Period From"],
+    Title.CHARGE_PERIOD_END: ["Charge Period End"],
+    Title.QUANTITY: ["Quantity"],
+    Title.QUNIT: ["QUnit"],
+    Title.CHARGE: ["Charge"],
+    Title.CUNIT: ["CUnit"],
+    Title.TOTAL: ["Total"],
+}
+
+
+def make_column_map(title_row):
+    titles = [cell.value for cell in title_row]
+    column_map = {}
+    for title, title_names in COLUMNS.items():
+        idx = None
+        for title_name in title_names:
+            try:
+                idx = titles.index(title_name)
+            except ValueError:
+                pass
+        if idx is None:
+            raise BadRequest(f"For the title {title} a column can't be found")
+
+        column_map[title] = idx + 1
+    return column_map
+
+
+def get_date_naive(sheet, row, col):
+    value = get_value(sheet, row, col)
+    if value in ("", None):
+        return None
+    elif not isinstance(value, Datetime):
+        raise BadRequest(
+            f"Problem reading {value} as a timestamp at row {row} col {col}."
+        )
+    return value
+
+
+def get_date(sheet, row, col_name):
+    dt = get_date_naive(sheet, row, col_name)
+    return None if dt is None else to_utc(to_ct(dt))
+
+
+def get_value(sheet, row, col):
+    try:
+        return sheet.cell(row=row, column=col).value
+    except IndexError:
+        raise BadRequest(
+            f"Can't find the cell at row {row} and col {col} on sheet {sheet}."
+        )
+
+
+def get_str(sheet, row, col):
+    value = get_value(sheet, row, col)
+    if value is None:
+        return None
+
+    return value.strip()
+
+
+def get_dec(sheet, row, col):
+    value = get_value(sheet, row, col)
+    if value in ("", None):
+        return None
+
+    try:
+        return Decimal(str(value))
+    except InvalidOperation as e:
+        raise BadRequest(
+            f"Problem parsing the number '{value}' at row {row} col {col}. {e}"
+        )
+
+
+def get_int(sheet, row, col):
+    value = get_value(sheet, row, col)
+    try:
+        return int(value)
+    except ValueError as e:
+        raise BadRequest(
+            f"Problem parsing the integer '{value}' at row {row} col {col}. {e}"
+        )
+
+
+"""
+def _bd_add(bd, el_name, val):
+    if el_name.split("-")[-1] in ("rate", "kva"):
+        if el_name not in bd:
+            bd[el_name] = set()
+        bd[el_name].add(val)
+    else:
+        if el_name not in bd:
+            bd[el_name] = 0
+        try:
+            bd[el_name] += val
+        except TypeError as e:
+            raise BadRequest(
+                f"Problem with element name {el_name} and value '{val}': {e}"
+            )
+"""
+
+ELEMENT_LOOKUP = {
+    "Management Fee": "admin_variable",
+    "LDZ Customer Capacity": "dn_customer_capacity_fixed",
+    "LDZ System Capacity": "dn_system_capacity_fixed",
+    "LDZ System Commodity": "dn_system_capacity",
+    "Metering Charges": "metering",
+    "NTS Exit Capacity (ECN)": "dn_ecn_fixed",
+    "NTS SO Exit": "so_exit_commodity",
+    "NTS TO Exit": "to_exit_commodity",
+    "Unidentified Gas": "ug",
+    "WAP": "wap",
+}
+
+QUNIT_LOOKUP = {
+    "kWh": "kwh",
+    "days": "days",
+}
+
+
+def _parse_row(bills, sheet, row, title_row):
+    column_map = make_column_map(title_row)
+    mprn = get_value(sheet, row, column_map[Title.MPRN])
+    reference = get_value(sheet, row, column_map[Title.BILL])
+    account = get_value(sheet, row, column_map[Title.ACCOUNT])
+    issue_date = get_date(sheet, row, column_map[Title.BILL_DATE])
+    start_date = get_date(sheet, row, column_map[Title.CHARGE_PERIOD_FROM])
+    finish_date = to_utc(
+        to_ct(get_date_naive(sheet, row, column_map[Title.CHARGE_PERIOD_END]))
+        + relativedelta(hours=23, minutes=30)
+    )
+
+    try:
+        mprn_values = bills[mprn]
+    except KeyError:
+        mprn_values = bills[mprn] = {}
+
+    try:
+        start_date_values = mprn_values[start_date]
+    except KeyError:
+        start_date_values = mprn_values[start_date] = {}
+
+    try:
+        bill = start_date_values[finish_date]
+    except KeyError:
+        bill = start_date_values[finish_date] = {
+            "bill_type_code": "N",
+            "mprn": mprn,
+            "reference": reference,
+            "account": account,
+            "issue_date": issue_date,
+            "start_date": start_date,
+            "finish_date": finish_date,
+            "kwh": Decimal("0"),
+            "net_gbp": Decimal("0.00"),
+            "vat_gbp": Decimal("0.00"),
+            "gross_gbp": Decimal("0.00"),
+            "breakdown": {},
+        }
+
+    bd = bill["breakdown"]
+    element_desc = get_value(sheet, row, column_map[Title.CHARGE_TYPE])
+    quantity = get_dec(sheet, row, column_map[Title.QUANTITY])
+    qunit = get_value(sheet, row, column_map[Title.QUNIT])
+    charge = get_dec(sheet, row, column_map[Title.CHARGE]) / Decimal("100")
+    total = get_dec(sheet, row, column_map[Title.TOTAL])
+    if element_desc.startswith("20% VAT on "):
+        bill["net_gbp"] += quantity
+        bill["vat_gbp"] += total
+        bill["gross_gbp"] += quantity + total
+    else:
+        element_name = ELEMENT_LOOKUP[element_desc]
+        if element_name == "admin_variable":
+            bill["kwh"] += quantity
+        bd[f"{element_name}_gbp"] = total
+        element_qunit = QUNIT_LOOKUP[qunit]
+        bd[f"{element_name}_{element_qunit}"] = quantity
+        bd[f"{element_name}_rate"] = charge
+
+
+def _make_raw_bills(sheet):
+    bills = {}
+    rows = tuple(sheet.rows)
+    title_row = rows[1]
+    for row_index, row in enumerate(rows[2:], start=3):
+        val = row[0].value
+        if val not in (None, ""):
+            try:
+                _parse_row(bills, sheet, row_index, title_row)
+            except BadRequest as e:
+                raise BadRequest(f"On row {row_index + 1}: {e.description}")
+    print("bills", bills)
+
+    raw_bills = []
+    for mprn, mprn_values in bills.items():
+        for period_stat, period_start_values in mprn_values.items():
+            for period_finish, bill in period_start_values.items():
+                raw_bills.append(bill)
+
+    return raw_bills
+
+
+class Parser:
+    def __init__(self, f):
+        self.book = load_workbook(BytesIO(f), data_only=True)
+        self.sheet = self.book.worksheets[0]
+
+        self.last_line = None
+        lines = (self._set_last_line(i, l) for i, l in enumerate(f))
+        self.reader = csv.reader(lines, skipinitialspace=True)
+        self._line_number = None
+        self._title_line = None
+
+    @property
+    def line_number(self):
+        return None if self._line_number is None else self._line_number + 1
+
+    def _set_last_line(self, i, line):
+        self._line_numer = i
+        self.last_line = line
+        if i == 0:
+            self._title_line = line
+        return line
+
+    def make_raw_bills(self):
+        return _make_raw_bills(self.sheet)

chellow-1745911639.0.0/chellow/reports/report_missing_bills.py

@@ -0,0 +1,158 @@
+import csv
+import threading
+import traceback
+
+from flask import g, redirect
+
+
+from sqlalchemy import null, or_, select
+from sqlalchemy.orm import joinedload
+
+from chellow.dloads import open_file
+from chellow.models import Batch, Bill, Contract, Era, RSession, ReportRun, User
+from chellow.utils import (
+    c_months_u,
+    csv_make_val,
+    hh_max,
+    hh_min,
+    hh_range,
+    req_date,
+    req_int,
+)
+
+
+def content(user_id, report_run_id, contract_id, months_length, finish_date):
+    f = writer = None
+    try:
+        with RSession() as sess:
+            caches = {}
+            contract = Contract.get_by_id(sess, contract_id)
+            user = User.get_by_id(sess, user_id)
+            f = open_file(
+                f"missing_bills_{contract.id}.csv", user, mode="w", newline=""
+            )
+            writer = csv.writer(f, lineterminator="\n")
+            titles = (
+                "contract_name",
+                "month_start",
+                "month_finish",
+                "site_code",
+                "site_name",
+                "imp_mpan_core",
+                "exp_mpan_core",
+                "account",
+            )
+            writer.writerow(titles)
+
+            months = list(
+                c_months_u(
+                    finish_year=finish_date.year,
+                    finish_month=finish_date.month,
+                    months=months_length,
+                )
+            )
+
+            for month_start, month_finish in months:
+                missing_bills = {}
+                missing_account = {}
+                account_missing_tuple = hh_range(caches, month_start, month_finish)
+                for era in sess.scalars(
+                    select(Era)
+                    .where(
+                        Era.start_date <= month_finish,
+                        or_(Era.finish_date == null(), Era.finish_date >= month_start),
+                        or_(
+                            Era.mop_contract == contract,
+                            Era.dc_contract == contract,
+                            Era.imp_supplier_contract == contract,
+                            Era.exp_supplier_contract == contract,
+                        ),
+                    )
+                    .options(joinedload(Era.supply))
+                ):
+                    chunk_start = hh_max(era.start_date, month_start)
+                    chunk_finish = hh_min(era.finish_date, month_finish)
+                    missing_set = set(hh_range(caches, chunk_start, chunk_finish))
+                    if era.mop_contract == contract:
+                        account = era.mop_account
+                    elif era.dc_contract == contract:
+                        account = era.dc_account
+                    elif era.imp_supplier_contract == contract:
+                        account = era.imp_supplier_account
+                    elif era.imp_supplier_contract == contract:
+                        account = era.exp_supplier_account
+
+                    try:
+                        account_missing_set = missing_account[account]
+                    except KeyError:
+                        account_missing_set = missing_account[account] = set(
+                            account_missing_tuple
+                        )
+                    supply = era.supply
+
+                    for bill in sess.scalars(
+                        select(Bill)
+                        .join(Batch)
+                        .where(
+                            Batch.contract == contract,
+                            Bill.supply == supply,
+                            Bill.start_date <= chunk_finish,
+                            Bill.finish_date >= chunk_start,
+                        )
+                    ):
+                        found_set = set(
+                            hh_range(caches, bill.start_date, bill.finish_date)
+                        )
+                        missing_set.difference_update(found_set)
+                        account_missing_set.difference_update(found_set)
+
+                    if len(missing_set) > 0:
+                        site = era.get_physical_site(sess)
+
+                        values = {
+                            "contract_id": contract.id,
+                            "contract_name": contract.name,
+                            "month_start": month_start,
+                            "month_finish": month_finish,
+                            "era_id": era.id,
+                            "supply_id": supply.id,
+                            "imp_mpan_core": era.imp_mpan_core,
+                            "exp_mpan_core": era.exp_mpan_core,
+                            "site_id": site.id,
+                            "site_code": site.code,
+                            "site_name": site.name,
+                            "account": account,
+                            "market_role_code": contract.market_role.code,
+                        }
+                        missing_bills[era.id] = values
+        for era_id, values in missing_bills.items():
+            if len(missing_account[values["account"]]) > 0:
+                writer.writerow(csv_make_val(values[t]) for t in titles)
+                ReportRun.w_insert_row(report_run_id, "", titles, values, {})
+
+    except BaseException:
+        msg = traceback.format_exc()
+        print(msg)
+        if writer is not None:
+            writer.writerow([msg])
+    finally:
+        ReportRun.w_update(report_run_id, "finished")
+        if f is not None:
+            f.close()
+
+
+def do_get(sess):
+    contract_id = req_int("contract_id")
+    months = req_int("months")
+    finish_date = req_date("finish", resolution="month")
+    report_run = ReportRun.insert(
+        sess,
+        "missing_e_bills",
+        g.user,
+        "missing_e_bills",
+        {},
+    )
+    sess.commit()
+    args = g.user.id, report_run.id, contract_id, months, finish_date
+    threading.Thread(target=content, args=args).start()
+    return redirect(f"/report_runs/{report_run.id}", 303)

{chellow-1744720514.0.0 → chellow-1745911639.0.0}/chellow/templates/e/dc_contract.html

@@ -97,7 +97,18 @@
 			<input type="submit" value="Download">
 		</fieldset>
 	</form>
-	<br>
+
+	<form action="/reports/missing_bills">
+		<fieldset>
+			<legend>Download Missing Bills</legend>
+			<input type="hidden" name="contract_id" value="{{dc_contract.id}}">
+			
+			<label>Months</label> <input name="months" value="1" maxlength="2" size="2">
+			<label>Last Month</label> {{input_date('finish', last_month_finish, 'month')}}
+			<input type="submit" value="Download">
+		</fieldset>
+	</form>
+
 	<h3>Script</h3>
 
 	<pre>{{dc_contract.charge_script}}</pre>

{chellow-1744720514.0.0 → chellow-1745911639.0.0}/chellow/templates/e/mop_contract.html

@@ -76,6 +76,17 @@
 			<input type="submit" value="Download"/>
 		</fieldset>
 	</form>
+
+	<form action="/reports/missing_bills">
+		<fieldset>
+			<legend>Download Missing Bills</legend>
+			<input type="hidden" name="contract_id" value="{{contract.id}}">
+			
+			<label>Months</label> <input name="months" value="1" maxlength="2" size="2">
+			<label>Last Month</label> {{input_date('finish', last_month_finish, 'month')}}
+			<input type="submit" value="Download">
+		</fieldset>
+	</form>
 	<br>
 	<h2>Script</h2>
 		<pre>{{contract.charge_script}}</pre>

{chellow-1744720514.0.0 → chellow-1745911639.0.0}/chellow/templates/e/supplier_contract.html

@@ -97,6 +97,17 @@
 		</fieldset>
 	</form>
 
+	<form action="/reports/missing_bills">
+		<fieldset>
+			<legend>Download Missing Bills</legend>
+			<input type="hidden" name="contract_id" value="{{contract.id}}">
+			
+			<label>Months</label> <input name="months" value="1" maxlength="2" size="2">
+			<label>Last Month</label> {{input_date('finish', month_finish, 'month')}}
+			<input type="submit" value="Download">
+		</fieldset>
+	</form>
+
 	<h2>Script</h2>	
 
 	<pre>{{contract.charge_script}}</pre>