chellow 1715873476.0.0__tar.gz → 1716366171.0.0__tar.gz

{chellow-1715873476.0.0 → chellow-1716366171.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: chellow
-Version: 1715873476.0.0
+Version: 1716366171.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)

chellow-1716366171.0.0/VERSION

@@ -0,0 +1 @@
+__version__ = "1716366171.0.0"

{chellow-1715873476.0.0 → chellow-1716366171.0.0}/chellow/edi_lib.py

@@ -6,54 +6,56 @@ from werkzeug.exceptions import BadRequest
 from chellow.utils import ct_datetime, to_ct, to_utc
 
 
+def line_iter(f):
+    line = []
+    while c := f.read(1) != "":
+        if c == "'":
+            yield "".join(line)
+            line.clear()
+        else:
+            line.append(c)
+
+
 class EdiParser:
     def __init__(self, f):
-        self.f_iterator = iter(f)
+
+        self.line_iterator = line_iter(f)
         self.line_number = 0
 
     def __iter__(self):
         return self
 
     def __next__(self):
-        self.line = next(self.f_iterator).strip()
+        self.line = next(self.line_iterator).strip()
         self.line_number += 1
 
-        if self.line[-1] != "'":
-            raise BadRequest(
-                f"The parser expects each line to end with a ', but line number "
-                f"{self.line_number} doesn't: {self.line}."
-            )
-        self.elements = [element.split(":") for element in self.line[4:-1].split("+")]
+        self.elements = [element.split(":") for element in self.line[4].split("+")]
         return self.line[:3]
 
 
 def parse_edi(edi_str):
-    for line_number, raw_line in enumerate(edi_str.splitlines(), start=1):
-        if raw_line[-1] != "'":
-            raise BadRequest(
-                f"The parser expects each line to end with a ', but line "
-                f"number {line_number} doesn't: {raw_line}."
-            )
-
+    for line_number, raw_line in enumerate(edi_str.split("'"), start=1):
         line = raw_line.strip()
+        if len(line) == 0:
+            continue
         code = line[:3]
 
-        els = [el.split(":") for el in line[4:-1].split("+")]
+        els = [el.split(":") for el in line[4:].split("+")]
 
-        segment_name = code + els[1][0] if code == "CCD" else code
+        segment_name = code + els[1][0].strip() if code == "CCD" else code
 
         try:
             elem_data = SEGMENTS[segment_name]
         except KeyError:
             raise BadRequest(
                 f"At line number {line_number} the segment name {segment_name} isn't "
-                f"recognized."
+                f"recognized. {raw_line}"
             )
         elem_codes = [m["code"] for m in elem_data["elements"]]
 
         elements = dict(zip(elem_codes, els))
 
-        yield line_number, line, segment_name, elements
+        yield line_number, f"{line}'", segment_name, elements
 
 
 def to_decimal(components):
@@ -67,11 +69,16 @@ def to_decimal(components):
 
 
 def to_ct_date(component):
+    if len(component) == 0:
+        return None
     return to_ct(Datetime.strptime(component, "%y%m%d"))
 
 
 def to_date(component):
-    return to_utc(to_ct_date(component))
+    dt = to_ct_date(component)
+    if dt is None:
+        return None
+    return to_utc(dt)
 
 
 def to_finish_date(component):

{chellow-1715873476.0.0 → chellow-1716366171.0.0}/chellow/gas/bill_parser_engie_edi.py

@@ -301,8 +301,10 @@ class Parser:
                 func = CODE_FUNCS[seg_name]
             except KeyError:
                 raise BadRequest(f"Code {seg_name} not recognized.")
-
-            bill = func(elements, headers)
+            try:
+                bill = func(elements, headers)
+            except BaseException as e:
+                raise BadRequest(f"Propblem with segment {line}: {e}") from e
 
             if "raw_lines" in headers:
                 headers["raw_lines"].append(line)

chellow-1716366171.0.0/chellow/gas/bill_parser_total_edi.py

@@ -0,0 +1,251 @@
+from collections import defaultdict
+from datetime import datetime as Datetime
+from decimal import Decimal
+
+from dateutil.relativedelta import relativedelta
+
+from werkzeug.exceptions import BadRequest
+
+from chellow.edi_lib import parse_edi, to_date, to_decimal
+from chellow.utils import HH, to_ct, to_utc
+
+
+READ_TYPE_MAP = {"00": "A", "01": "E", "02": "E"}
+
+
+SUPPLIER_CODE_MAP = {
+    "STD": ("standing", Decimal("1000"), Decimal("1")),
+    "MET": ("commodity", Decimal("100000"), Decimal("1000")),
+    "CCL": ("ccl", Decimal("100000"), Decimal("1000")),
+}
+
+UNIT_MAP = {"M3": "M3", "HH": "HCUF", "HCUF": "HCUF"}
+
+
+def _to_finish_date(date_str):
+    if len(date_str) == 0:
+        return None
+    return to_utc(
+        to_ct(Datetime.strptime(date_str, "%y%m%d") + relativedelta(days=1) - HH)
+    )
+
+
+def _process_ADJ(elements, headers):
+    adjf = elements["ADJF"]
+    if adjf[0] == "CV":
+        headers["cv"] = Decimal(adjf[1]) / Decimal(100000)
+
+
+def _process_BCD(elements, headers):
+    ivdt = elements["IVDT"]
+    headers["issue_date"] = to_date(ivdt[0])
+
+    invn = elements["INVN"]
+    headers["reference"] = invn[0]
+
+    btcd = elements["BTCD"]
+    headers["bill_type_code"] = btcd[0]
+
+
+def _process_MHD(elements, headers):
+    headers.clear()
+
+    typ = elements["TYPE"]
+    message_type = headers["message_type"] = typ[0]
+    if message_type == "UTLBIL":
+        headers["reads"] = []
+        headers["raw_lines"] = []
+        headers["breakdown"] = defaultdict(int, {"units_consumed": Decimal(0)})
+        headers["kwh"] = Decimal("0.00")
+        headers["net"] = Decimal("0.00")
+        headers["vat"] = Decimal("0.00")
+        headers["gross"] = Decimal("0.00")
+
+
+NUCT_LOOKUP = {"DAY": "days", "KWH": "kwh"}
+
+
+def _process_CCD2(elements, headers):
+    breakdown = headers["breakdown"]
+    ccde = elements["CCDE"]
+    nuct = elements["NUCT"]
+    mtnr = elements["MTNR"]
+    conb = elements["CONB"]
+    adjf = elements["ADJF"]
+    prdt = elements["PRDT"]
+    pvdt = elements["PVDT"]
+    prrd = elements["PRRD"]
+    cppu = elements["CPPU"]
+    ctot = elements["CTOT"]
+    csdt = elements["CSDT"]
+    cedt = elements["CEDT"]
+    mloc = elements["MLOC"]
+
+    ccde_supplier_code = ccde[2]
+    tpref, rate_divisor, units_divisor = SUPPLIER_CODE_MAP[ccde_supplier_code]
+
+    rate_str = cppu[0]
+
+    rate_key = f"{tpref}_rate"
+    if rate_key not in breakdown:
+        breakdown[rate_key] = set()
+    rate = Decimal(rate_str) / rate_divisor
+    breakdown[rate_key].add(rate)
+
+    breakdown[f"{tpref}_gbp"] += to_decimal(ctot) / Decimal("100")
+
+    suff = NUCT_LOOKUP[nuct[1].strip()]
+    key = f"{tpref}_{suff}"
+    units_billed = to_decimal(nuct) / units_divisor
+    breakdown[key] += units_billed
+
+    if "start_date" not in headers:
+        headers["start_date"] = to_date(csdt[0])
+
+    if "finish_date" not in headers:
+        headers["finish_date"] = _to_finish_date(cedt[0])
+
+    if "mprn" not in headers:
+        headers["mprn"] = mloc[0]
+
+    if len(conb) > 0 and len(conb[0]) > 0:
+        headers["breakdown"]["units_consumed"] += to_decimal(conb) / Decimal("1000")
+
+    if tpref == "commodity":
+        headers["kwh"] += units_billed
+
+    if len(prrd) > 0 and len(prrd[0]) > 0:
+        pres_read_date = to_date(prdt[0])
+        prev_read_date = to_date(pvdt[0])
+
+        pres_read_value = Decimal(prrd[0])
+        pres_read_type = READ_TYPE_MAP[prrd[1]]
+        prev_read_value = Decimal(prrd[2])
+        prev_read_type = READ_TYPE_MAP[prrd[3]]
+        msn = mtnr[0]
+        unit = UNIT_MAP[conb[1]]
+        correction_factor = Decimal(adjf[1]) / Decimal(100000)
+
+        headers["reads"].append(
+            {
+                "msn": msn,
+                "unit": unit,
+                "correction_factor": correction_factor,
+                "prev_date": prev_read_date,
+                "prev_value": prev_read_value,
+                "prev_type_code": prev_read_type,
+                "pres_date": pres_read_date,
+                "pres_value": pres_read_value,
+                "pres_type_code": pres_read_type,
+            }
+        )
+
+
+def _process_MTR(elements, headers):
+    if headers["message_type"] == "UTLBIL":
+        breakdown = headers["breakdown"]
+        for k, v in tuple(breakdown.items()):
+            if isinstance(v, set):
+                breakdown[k] = sorted(v)
+
+        for read in headers["reads"]:
+            read["calorific_value"] = headers["cv"]
+
+        return {
+            "raw_lines": "\n".join(headers["raw_lines"]),
+            "mprn": headers["mprn"],
+            "reference": headers["reference"],
+            "account": headers["mprn"],
+            "reads": headers["reads"],
+            "kwh": headers["kwh"],
+            "breakdown": headers["breakdown"],
+            "net_gbp": headers["net"],
+            "vat_gbp": headers["vat"],
+            "gross_gbp": headers["gross"],
+            "bill_type_code": headers["bill_type_code"],
+            "start_date": headers["start_date"],
+            "finish_date": headers["finish_date"],
+            "issue_date": headers["issue_date"],
+        }
+
+
+def _process_VAT(elements, headers):
+    breakdown = headers["breakdown"]
+    vatp = elements["VATP"]
+    if "vat" in breakdown:
+        vat = breakdown["vat"]
+    else:
+        vat = breakdown["vat"] = {}
+
+    vat_perc = to_decimal(vatp) / Decimal(1000)
+    try:
+        vat_bd = vat[vat_perc]
+    except KeyError:
+        vat_bd = vat[vat_perc] = defaultdict(int)
+
+    uvtt = elements["UVTT"]
+    vat_gbp = to_decimal(uvtt) / Decimal("100")
+
+    vat_bd["vat"] += vat_gbp
+
+    uvla = elements["UVLA"]
+    net_gbp = to_decimal(uvla) / Decimal("100")
+    vat_bd["net"] += net_gbp
+    headers["net"] += net_gbp
+    headers["vat"] += vat_gbp
+    ucsi = elements["UCSI"]
+    headers["gross"] += to_decimal(ucsi) / Decimal("100")
+
+
+def _process_NOOP(elements, headers):
+    pass
+
+
+CODE_FUNCS = {
+    "ADJ": _process_ADJ,
+    "BCD": _process_BCD,
+    "BTL": _process_NOOP,
+    "CCD1": _process_NOOP,
+    "CCD2": _process_CCD2,
+    "CCD3": _process_NOOP,
+    "CCD4": _process_NOOP,
+    "CDT": _process_NOOP,
+    "CLO": _process_NOOP,
+    "END": _process_NOOP,
+    "FIL": _process_NOOP,
+    "MHD": _process_MHD,
+    "MTR": _process_MTR,
+    "SDT": _process_NOOP,
+    "STX": _process_NOOP,
+    "TYP": _process_NOOP,
+    "TTL": _process_NOOP,
+    "VAT": _process_VAT,
+    "VTS": _process_NOOP,
+}
+
+
+class Parser:
+    def __init__(self, file_bytes):
+        self.edi_str = str(file_bytes, "utf-8", errors="ignore")
+        self.line_number = None
+
+    def make_raw_bills(self):
+        bills = []
+        headers = {}
+        bill = None
+        for self.line_number, line, seg_name, elements in parse_edi(self.edi_str):
+            try:
+                func = CODE_FUNCS[seg_name]
+            except KeyError:
+                raise BadRequest(f"Code {seg_name} not recognized.")
+            try:
+                bill = func(elements, headers)
+            except BaseException as e:
+                raise Exception(f"Propblem with segment {line}: {e}") from e
+
+            if "raw_lines" in headers:
+                headers["raw_lines"].append(line)
+            if bill is not None:
+                bills.append(bill)
+
+        return bills

{chellow-1715873476.0.0 → chellow-1716366171.0.0}/chellow/gas/engine.py

@@ -502,13 +502,6 @@ class GDataSource:
                 * h["calorific_value"]
                 / 3.6
             )
-            h["kwh_avg"] = (
-                h["units_consumed"]
-                * h["unit_factor"]
-                * h["correction_factor"]
-                * h["avg_cv"]
-                / 3.6
-            )
             h["ug_rate"] = float(
                 g_rates(sess, self.caches, "ug", h["start_date"], True)[
                     "ug_gbp_per_kwh"
@@ -703,9 +696,7 @@ def find_cv(sess, caches, dt, g_ldz_code):
             avg_cv = year_cache[ct.month]
         except KeyError:
             cv_list = [float(v["cv"]) for v in cvs.values()]
-            avg_cv = year_cache[ct.month] = round(
-                sum(cv_list) / len(cv_list), ndigits=1
-            )
+            avg_cv = year_cache[ct.month] = sum(cv_list) / len(cv_list)
     return cv, avg_cv
 
 

{chellow-1715873476.0.0 → chellow-1716366171.0.0}/chellow/gas/views.py

@@ -289,15 +289,37 @@ def supply_edit_get(g_supply_id):
     )
 
 
+@gas.route("/supplies/<int:g_supply_id>/edit", methods=["DELETE"])
+def supply_edit_delete(g_supply_id):
+    g_supply = GSupply.get_by_id(g.sess, g_supply_id)
+    try:
+        g_supply.delete(g.sess)
+        g.sess.commit()
+        return hx_redirect("/supplies")
+    except BadRequest as e:
+        flash(e.description)
+        g_eras = g.sess.scalars(
+            select(GEra)
+            .where(GEra.g_supply == g_supply)
+            .order_by(GEra.start_date.desc())
+        )
+        g_exit_zones = g.sess.query(GExitZone).order_by(GExitZone.code).all()
+        return make_response(
+            render_template(
+                "supply_edit.html",
+                g_supply=g_supply,
+                g_eras=g_eras,
+                g_exit_zones=g_exit_zones,
+            ),
+            400,
+        )
+
+
 @gas.route("/supplies/<int:g_supply_id>/edit", methods=["POST"])
 def supply_edit_post(g_supply_id):
     g_supply = GSupply.get_by_id(g.sess, g_supply_id)
     try:
-        if "delete" in request.values:
-            g_supply.delete(g.sess)
-            g.sess.commit()
-            return chellow_redirect("/supplies", 303)
-        elif "insert_g_era" in request.values:
+        if "insert_g_era" in request.values:
             start_date = req_date("start")
             g_supply.insert_g_era_at(g.sess, start_date)
             g.sess.commit()
@@ -418,6 +440,16 @@ def batch_get(g_batch_id):
                 vbd["vat"] += g_bill.vat
                 vbd["net"] += g_bill.net
 
+            if "vat" in bd:
+                for vat_percentage, vat_bd in bd["vat"].items():
+                    try:
+                        vbd = vat_breakdown[vat_percentage]
+                    except KeyError:
+                        vbd = vat_breakdown[vat_percentage] = defaultdict(int)
+
+                    vbd["vat"] += vat_bd["vat"]
+                    vbd["net"] += vat_bd["net"]
+
     config_contract = Contract.get_non_core_by_name(g.sess, "configuration")
     properties = config_contract.make_properties()
 

{chellow-1715873476.0.0 → chellow-1716366171.0.0}/chellow/templates/g/bill_import.html

@@ -45,18 +45,21 @@
 					<th>Issue Date</th>
 					<th>Start Date</th>
 					<th>Finish Date</th>
+					<th>kWh</th>
 					<th>Net GBP</th>
-					<th>VAT 5%</th>
-					<th>VAT 15%</th>
-					<th>VAT 17.5%</th>
-					<th>VAT 20%</th>
 					<th>VAT</th>
 					<th>Gross GBP</th>
-					{% for read in failed_bills[0]['reads'] %}
-						{% for k in read|sort %}
-							<th>{{k}}</th>
-						{% endfor %}
-					{% endfor %}
+					<th>Breakdown</th>
+					<th>R1 calorific_value</th>
+					<th>R1 correction_factor</th>
+					<th>R1 msn</th>
+					<th>R1 unit</th>
+					<th>R1 prev_date</th>
+					<th>R1 prev_value</th>
+					<th>R1 prev_type_code</th>
+					<th>R1 pres_date</th>
+					<th>R1 pres_value</th>
+					<th>R1 pres_type_code</th>
 				</tr>
 			</thead>
 			<tbody>
@@ -70,17 +73,22 @@
 						<td>{{bill.issue_date|hh_format}}</td>
 						<td>{{bill.start_date|hh_format}}</td>
 						<td>{{bill.finish_date|hh_format}}</td>
+						<td>{{bill.kwh}}</td>
 						<td>{{bill.net_gbp}}</td>
-						<td>{{bill.vat_5pc}}</td>
-						<td>{{bill.vat_15pc}}</td>
-						<td>{{bill.vat_17_5pc}}</td>
-						<td>{{bill.vat_20pc}}</td>
 						<td>{{bill.vat_gbp}}</td>
 						<td>{{bill.gross_gbp}}</td>
+						<td>{{bill.breakdown|dumps}}</td>
 						{% for read in bill.reads %}
-							{% for k, v in read|dictsort %}
-								<td>{{v}}</td>
-							{% endfor %}
+							<td>{{read.calorific_value}}</td>
+							<td>{{read.correction_factor}}</td>
+							<td>{{read.msn}}</td>
+							<td>{{read. unit}}</td>
+							<td>{{read.prev_date|hh_format}}</td>
+							<td>{{read.prev_value}}</td>
+							<td>{{read.prev_type_code}}</td>
+							<td>{{read.pres_date|hh_format}}</td>
+							<td>{{read.pres_value}}</td>
+							<td>{{read.pres_type_code}}</td>
 						{% endfor %}
 					</tr>
 				{% endfor %}

chellow-1716366171.0.0/chellow/templates/g/supply_edit.html

@@ -0,0 +1,62 @@
+{% extends "base.html" %}
+
+{% block title %}
+	Gas Supplies » {{g_supply.id}} » Edit
+{% endblock %}
+
+{% block nav %}
+	<a href="/g/supplies">Gas Supplies</a> &raquo;
+	<a href="/g/supplies/{{g_supply.id}}">{{g_supply.id}}</a> &raquo; Edit
+{% endblock %}
+
+{% block content %}
+	<form method="post" action="/g/supplies/{{g_supply.id}}/edit">
+		<fieldset>
+			<legend>Update this supply</legend>
+			<label>Name</label> {{input_text('name', g_supply.name)}}
+			<label>MPRN</label> {{input_text('mprn', g_supply.mprn)}}
+			<label>Exit Zone</label>
+			<select name="g_exit_zone_id">
+				{% for g_exit_zone in g_exit_zones %}
+					{{input_option(
+						'g_exit_zone_id', g_exit_zone.id, g_exit_zone.code, initial=g_supply.g_exit_zone.id)}}
+				{% endfor %}
+			</select>
+			<input type="submit" value="Update">
+		</fieldset>
+	</form>
+
+	<form hx-delete="/g/supplies/{{g_supply.id}}/edit"
+			hx-confirm="Are you sure you want to delete this supply?">
+		<fieldset>
+			<legend>Delete this supply</legend>
+			<input type="submit" value="Delete">
+		</fieldset>
+	</form>
+
+	<form method="post" action="/g/supplies/{{g_supply.id}}/edit">
+		<fieldset>
+			<legend>Insert a new era</legend>
+			<label>Start date</label> {{input_date('start', None)}}
+			<input type="submit" name="insert_g_era" value="Insert">
+		</fieldset>
+	</form>
+
+	<table>
+		<caption>Existing Eras</caption>
+		<thead>
+			<tr>
+				<th>Start date</th>
+				<th>Finish date</th>
+			</tr>
+		</thead>
+		<tbody>
+			{% for g_era in g_eras %}
+				<tr>
+					<td>{{g_era.start_date|hh_format}}</td>
+					<td>{{g_era.finish_date|hh_format}}</td>
+				</tr>
+			{% endfor %}
+		</tbody>
+	</table>
+{% endblock %}

{chellow-1715873476.0.0 → chellow-1716366171.0.0}/chellow/views.py

@@ -67,10 +67,10 @@ import chellow.e.mdd_importer
 import chellow.e.rcrc
 import chellow.e.system_price
 import chellow.e.tlms
-import chellow.edi_lib
 import chellow.general_import
 import chellow.national_grid
 import chellow.rate_server
+from chellow.edi_lib import SEGMENTS, parse_edi
 from chellow.models import (
     BillType,
     Channel,
@@ -644,23 +644,19 @@ def edi_viewer_post():
         else:
             edi_str = req_str("edi_fragment")
 
-        f = StringIO(edi_str)
-        f.seek(0)
-        parser = chellow.edi_lib.EdiParser(f)
-        for segment_name in parser:
-            elements = parser.elements
+        for line_number, line, segment_name, elements in parse_edi(edi_str):
 
             if segment_name == "CCD":
-                segment_name = segment_name + elements[1][0]
+                segment_name = segment_name + elements["CCDE"][0]
                 try:
-                    seg = chellow.edi_lib.SEGMENTS[segment_name]
+                    seg = SEGMENTS[segment_name]
                 except KeyError:
                     raise BadRequest(
                         f"The segment name {segment_name} is not recognized."
                     )
             else:
                 try:
-                    seg = chellow.edi_lib.SEGMENTS[segment_name]
+                    seg = SEGMENTS[segment_name]
                 except KeyError:
                     raise BadRequest(
                         f"The segment name {segment_name} is not recognized."
@@ -669,13 +665,14 @@ def edi_viewer_post():
             titles_element = []
             titles_component = []
             values = []
-            elems = seg["elements"]
+            elems = {el["code"]: el for el in seg["elements"]}
             if len(elements) > len(elems):
                 raise BadRequest(
                     f"There are more elements than recognized for the segment "
-                    f"{segment_name}."
+                    f"{segment_name}. {line}"
                 )
-            for element, elem in zip(elements, elems):
+            for element_code, element in elements.items():
+                elem = elems[element_code]
                 comps = elem["components"]
                 colspan = len(comps)
                 titles_element.append(
@@ -688,8 +685,9 @@ def edi_viewer_post():
                 )
                 if len(element) > len(comps):
                     raise BadRequest(
-                        f"There are more components than recognized for the segment "
-                        f"{segment_name} and element {element}."
+                        f"For the segment {segment_name} the number of components "
+                        f"{element} is greater than the expected components {comps}. "
+                        f"{line}"
                     )
 
                 for i, (title, typ) in enumerate(comps):
@@ -726,7 +724,7 @@ def edi_viewer_post():
                     "titles_element": titles_element,
                     "titles_component": titles_component,
                     "vals": values,
-                    "raw_line": parser.line,
+                    "raw_line": line,
                 }
             )