chellow 1715873476.0.0__tar.gz → 1715955542.0.0__tar.gz

{chellow-1715873476.0.0 → chellow-1715955542.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: chellow
-Version: 1715873476.0.0
+Version: 1715955542.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)

chellow-1715955542.0.0/VERSION

@@ -0,0 +1 @@
+__version__ = "1715955542.0.0"

{chellow-1715873476.0.0 → chellow-1715955542.0.0}/chellow/edi_lib.py

@@ -6,54 +6,56 @@ from werkzeug.exceptions import BadRequest
 from chellow.utils import ct_datetime, to_ct, to_utc
 
 
+def line_iter(f):
+    line = []
+    while c := f.read(1) != "":
+        if c == "'":
+            yield "".join(line)
+            line.clear()
+        else:
+            line.append(c)
+
+
 class EdiParser:
     def __init__(self, f):
-        self.f_iterator = iter(f)
+
+        self.line_iterator = line_iter(f)
         self.line_number = 0
 
     def __iter__(self):
         return self
 
     def __next__(self):
-        self.line = next(self.f_iterator).strip()
+        self.line = next(self.line_iterator).strip()
         self.line_number += 1
 
-        if self.line[-1] != "'":
-            raise BadRequest(
-                f"The parser expects each line to end with a ', but line number "
-                f"{self.line_number} doesn't: {self.line}."
-            )
-        self.elements = [element.split(":") for element in self.line[4:-1].split("+")]
+        self.elements = [element.split(":") for element in self.line[4].split("+")]
         return self.line[:3]
 
 
 def parse_edi(edi_str):
-    for line_number, raw_line in enumerate(edi_str.splitlines(), start=1):
-        if raw_line[-1] != "'":
-            raise BadRequest(
-                f"The parser expects each line to end with a ', but line "
-                f"number {line_number} doesn't: {raw_line}."
-            )
-
+    for line_number, raw_line in enumerate(edi_str.split("'"), start=1):
         line = raw_line.strip()
+        if len(line) == 0:
+            continue
         code = line[:3]
 
-        els = [el.split(":") for el in line[4:-1].split("+")]
+        els = [el.split(":") for el in line[4:].split("+")]
 
-        segment_name = code + els[1][0] if code == "CCD" else code
+        segment_name = code + els[1][0].strip() if code == "CCD" else code
 
         try:
             elem_data = SEGMENTS[segment_name]
         except KeyError:
             raise BadRequest(
                 f"At line number {line_number} the segment name {segment_name} isn't "
-                f"recognized."
+                f"recognized. {raw_line}"
             )
         elem_codes = [m["code"] for m in elem_data["elements"]]
 
         elements = dict(zip(elem_codes, els))
 
-        yield line_number, line, segment_name, elements
+        yield line_number, f"{line}'", segment_name, elements
 
 
 def to_decimal(components):
@@ -67,11 +69,16 @@ def to_decimal(components):
 
 
 def to_ct_date(component):
+    if len(component) == 0:
+        return None
     return to_ct(Datetime.strptime(component, "%y%m%d"))
 
 
 def to_date(component):
-    return to_utc(to_ct_date(component))
+    dt = to_ct_date(component)
+    if dt is None:
+        return None
+    return to_utc(dt)
 
 
 def to_finish_date(component):

chellow-1715955542.0.0/chellow/gas/bill_parser_edi.py

@@ -0,0 +1,403 @@
+from collections import defaultdict
+from datetime import datetime as Datetime
+from decimal import Decimal
+
+from dateutil.relativedelta import relativedelta
+
+from werkzeug.exceptions import BadRequest
+
+from chellow.edi_lib import parse_edi, to_date, to_decimal
+from chellow.utils import HH, to_ct, to_utc
+
+
+READ_TYPE_MAP = {"00": "A", "01": "E", "02": "E"}
+
+TCOD_MAP = {
+    "Energy Bill Discount Scheme": {"PPK": "ebrs"},
+    "Energy Bill Relief Scheme": {"PPK": "ebrs"},
+    "Energy Bill Relief Scheme Discount": {"PPK": "ebrs"},
+    "Unidentified Gas": {"PPK": "ug"},
+    "Commodity": {"PPK": "commodity"},
+    "Transportation": {"PPD": "transportation_fixed", "PPK": "transportation_variable"},
+    "Gas Flexi": {"PPK": "commodity"},
+    "Flex - Gas Flexi (New)": {"PPK": "commodity"},
+    "Meter Reading": {"PPD": "meter_read"},
+    "Meter Reading Credit Oct 19": {"FIX": "meter_read"},
+    "Meter Rental": {"PPD": "metering"},
+    "CCL": {"PPK": "ccl"},
+    "Consumption Based Administration": {"PPK": "admin_variable"},
+    "Swing": {"PPK": "swing"},
+}
+
+SUPPLIER_CODE_MAP = {
+    "STD": "standing",
+    "MET": "commodity",
+    "CCL": "ccl",
+}
+
+UNIT_MAP = {"M3": "M3", "HH": "HCUF", "HCUF": "HCUF"}
+
+
+def _to_finish_date(date_str):
+    if len(date_str) == 0:
+        return None
+    return to_utc(
+        to_ct(Datetime.strptime(date_str, "%y%m%d") + relativedelta(days=1) - HH)
+    )
+
+
+def _process_ADJ(elements, headers):
+    adjf = elements["ADJF"]
+    if adjf[0] == "CV":
+        headers["cv"] = Decimal(adjf[1]) / Decimal(100000)
+
+
+def _process_BCD(elements, headers):
+    ivdt = elements["IVDT"]
+    headers["issue_date"] = to_date(ivdt[0])
+
+    invn = elements["INVN"]
+    headers["reference"] = invn[0]
+
+    btcd = elements["BTCD"]
+    headers["bill_type_code"] = btcd[0]
+
+    sumo = elements["SUMO"]
+    start_date = to_date(sumo[0])
+    if start_date is not None:
+        headers["start_date"] = start_date
+
+    if len(sumo) > 1:
+        finish_date = _to_finish_date(sumo[1])
+        if finish_date is not None:
+            headers["finish_date"] = finish_date
+
+
+def _process_MHD(elements, headers):
+    headers.clear()
+
+    typ = elements["TYPE"]
+    message_type = headers["message_type"] = typ[0]
+    if message_type == "UTLBIL":
+        headers["reads"] = []
+        headers["raw_lines"] = []
+        headers["breakdown"] = defaultdict(int, {"units_consumed": Decimal(0)})
+        headers["kwh"] = Decimal("0.00")
+        headers["net"] = Decimal("0.00")
+        headers["vat"] = Decimal("0.00")
+        headers["gross"] = Decimal("0.00")
+
+
+def _process_CCD1(elements, headers):
+    mtnr = elements["MTNR"]
+    msn = mtnr[0]
+
+    mloc = elements["MLOC"]
+
+    # Bug in EDI where MPRN missing in second CCD 1
+    if "mprn" not in headers:
+        headers["mprn"] = mloc[0]
+
+    prdt = elements["PRDT"]
+    pvdt = elements["PVDT"]
+
+    pres_read_date = to_date(prdt[0])
+    prev_read_date = to_date(pvdt[0])
+
+    prrd = elements["PRRD"]
+    pres_read_value = Decimal(prrd[0])
+    pres_read_type = READ_TYPE_MAP[prrd[1]]
+    prev_read_value = Decimal(prrd[2])
+    prev_read_type = READ_TYPE_MAP[prrd[3]]
+
+    conb = elements["CONB"]
+    unit = UNIT_MAP[conb[1]]
+    headers["breakdown"]["units_consumed"] += to_decimal(conb) / Decimal("1000")
+
+    adjf = elements["ADJF"]
+    correction_factor = Decimal(adjf[1]) / Decimal(100000)
+
+    nuct = elements["NUCT"]
+
+    headers["kwh"] += to_decimal(nuct) / Decimal("1000")
+
+    headers["reads"].append(
+        {
+            "msn": msn,
+            "unit": unit,
+            "correction_factor": correction_factor,
+            "prev_date": prev_read_date,
+            "prev_value": prev_read_value,
+            "prev_type_code": prev_read_type,
+            "pres_date": pres_read_date,
+            "pres_value": pres_read_value,
+            "pres_type_code": pres_read_type,
+        }
+    )
+
+
+NUCT_LOOKUP = {"DAY": "days", "KWH": "kwh"}
+
+
+def _process_CCD2(elements, headers):
+    breakdown = headers["breakdown"]
+    ccde = elements["CCDE"]
+    ccde_supplier_code = ccde[2]
+    tcod = elements["TCOD"]
+    nuct = elements["NUCT"]
+    mtnr = elements["MTNR"]
+    conb = elements["CONB"]
+    adjf = elements["ADJF"]
+    prdt = elements["PRDT"]
+    pvdt = elements["PVDT"]
+    prrd = elements["PRRD"]
+
+    if len(tcod) > 1:
+        tpref_lookup = TCOD_MAP[tcod[1]]
+    else:
+        tpref_lookup = SUPPLIER_CODE_MAP
+
+    tpref = tpref_lookup[ccde_supplier_code]
+
+    bpri = elements["BPRI"]
+    if len(bpri[0]) > 0:
+        rate_key = f"{tpref}_rate"
+        if rate_key not in breakdown:
+            breakdown[rate_key] = set()
+        rate = Decimal(bpri[0]) / Decimal("10000000")
+        breakdown[rate_key].add(rate)
+
+    try:
+        ctot = elements["CTOT"]
+        breakdown[f"{tpref}_gbp"] += to_decimal(ctot) / Decimal("100")
+
+        if len(nuct) > 1:
+            key = NUCT_LOOKUP[nuct[1]]
+        else:
+            if ccde_supplier_code == "PPK":
+                key = f"{tpref}_kwh"
+            elif ccde_supplier_code == "PPD":
+                key = f"{tpref}_days"
+
+        breakdown[key] += to_decimal(nuct) / Decimal("1000")
+    except KeyError:
+        pass
+
+    if "start_date" not in headers:
+        csdt = elements["CSDT"]
+        start_date = to_date(csdt[0])
+        if start_date is not None:
+            headers["start_date"] = start_date
+
+    if "finish_date" not in headers:
+        cedt = elements["CSDT"]
+        finish_date = _to_finish_date(cedt[0])
+        if finish_date is not None:
+            headers["finish_date"] = finish_date
+
+    if "mprn" not in headers:
+        mloc = elements["MLOC"]
+        headers["mprn"] = mloc[0]
+
+    if len(conb) > 0 and len(conb[0]) > 0:
+        headers["breakdown"]["units_consumed"] += to_decimal(conb) / Decimal("1000")
+
+    if len(prrd) > 0 and len(prrd[0]) > 0:
+        pres_read_date = to_date(prdt[0])
+        prev_read_date = to_date(pvdt[0])
+
+        pres_read_value = Decimal(prrd[0])
+        pres_read_type = READ_TYPE_MAP[prrd[1]]
+        prev_read_value = Decimal(prrd[2])
+        prev_read_type = READ_TYPE_MAP[prrd[3]]
+        msn = mtnr[0]
+        unit = UNIT_MAP[conb[1]]
+        correction_factor = Decimal(adjf[1]) / Decimal(100000)
+
+        headers["reads"].append(
+            {
+                "msn": msn,
+                "unit": unit,
+                "correction_factor": correction_factor,
+                "prev_date": prev_read_date,
+                "prev_value": prev_read_value,
+                "prev_type_code": prev_read_type,
+                "pres_date": pres_read_date,
+                "pres_value": pres_read_value,
+                "pres_type_code": pres_read_type,
+            }
+        )
+
+
+def _process_CCD3(elements, headers):
+    breakdown = headers["breakdown"]
+    ccde = elements["CCDE"]
+    ccde_supplier_code = ccde[2]
+    tcod = elements["TCOD"]
+
+    tpref = TCOD_MAP[tcod[1]][ccde_supplier_code]
+
+    bpri = elements["BPRI"]
+    bpri_str = bpri[0]
+    if len(bpri_str) > 0:
+        rate_key = f"{tpref}_rate"
+        if rate_key not in breakdown:
+            breakdown[rate_key] = set()
+        rate = Decimal(bpri_str) / Decimal("10000000")
+        breakdown[rate_key].add(rate)
+
+    nuct = elements["NUCT"]
+
+    try:
+        ctot = elements["CTOT"]
+        breakdown[f"{tpref}_gbp"] += to_decimal(ctot) / Decimal("100")
+
+        if ccde_supplier_code == "PPK":
+            key = f"{tpref}_kwh"
+        elif ccde_supplier_code == "PPD":
+            key = f"{tpref}_days"
+
+        breakdown[key] += to_decimal(nuct) / Decimal("1000")
+    except KeyError:
+        pass
+
+
+def _process_CCD4(elements, headers):
+    breakdown = headers["breakdown"]
+    ccde = elements["ccde"]
+    ccde_supplier_code = ccde[2]
+    tcod = elements["TCOD"]
+
+    tpref = TCOD_MAP[tcod[1]][ccde_supplier_code]
+
+    bpri = elements["BPRI"]
+    rate_key = f"{tpref}_rate"
+    if rate_key not in breakdown:
+        breakdown[rate_key] = set()
+    rate = Decimal(bpri[0]) / Decimal("10000000")
+    breakdown[rate_key].add(rate)
+
+    nuct = elements["NUCT"]
+
+    try:
+        ctot = elements["CTOT"]
+        breakdown[tpref + "_gbp"] += to_decimal(ctot) / Decimal("100")
+
+        if ccde_supplier_code == "PPK":
+            key = f"{tpref}_kwh"
+        elif ccde_supplier_code == "PPD":
+            key = f"{tpref}_days"
+
+        breakdown[key] += to_decimal(nuct) / Decimal("1000")
+    except KeyError:
+        pass
+
+
+def _process_MTR(elements, headers):
+    if headers["message_type"] == "UTLBIL":
+        breakdown = headers["breakdown"]
+        for k, v in tuple(breakdown.items()):
+            if isinstance(v, set):
+                breakdown[k] = sorted(v)
+
+        for read in headers["reads"]:
+            read["calorific_value"] = headers["cv"]
+
+        return {
+            "raw_lines": "\n".join(headers["raw_lines"]),
+            "mprn": headers["mprn"],
+            "reference": headers["reference"],
+            "account": headers["mprn"],
+            "reads": headers["reads"],
+            "kwh": headers["kwh"],
+            "breakdown": headers["breakdown"],
+            "net_gbp": headers["net"],
+            "vat_gbp": headers["vat"],
+            "gross_gbp": headers["gross"],
+            "bill_type_code": headers["bill_type_code"],
+            "start_date": headers["start_date"],
+            "finish_date": headers["finish_date"],
+            "issue_date": headers["issue_date"],
+        }
+
+
+def _process_VAT(elements, headers):
+    breakdown = headers["breakdown"]
+    vatp = elements["VATP"]
+    if "vat" in breakdown:
+        vat = breakdown["vat"]
+    else:
+        vat = breakdown["vat"] = {}
+
+    vat_perc = to_decimal(vatp) / Decimal(1000)
+    try:
+        vat_bd = vat[vat_perc]
+    except KeyError:
+        vat_bd = vat[vat_perc] = defaultdict(int)
+
+    uvtt = elements["UVTT"]
+    vat_gbp = to_decimal(uvtt) / Decimal("100")
+
+    vat_bd["vat"] += vat_gbp
+
+    uvla = elements["UVLA"]
+    net_gbp = to_decimal(uvla) / Decimal("100")
+    vat_bd["net"] += net_gbp
+    headers["net"] += net_gbp
+    headers["vat"] += vat_gbp
+    ucsi = elements["UCSI"]
+    headers["gross"] += to_decimal(ucsi) / Decimal("100")
+
+
+def _process_NOOP(elements, headers):
+    pass
+
+
+CODE_FUNCS = {
+    "ADJ": _process_ADJ,
+    "BCD": _process_BCD,
+    "BTL": _process_NOOP,
+    "CCD1": _process_CCD1,
+    "CCD2": _process_CCD2,
+    "CCD3": _process_CCD3,
+    "CCD4": _process_CCD4,
+    "CDT": _process_NOOP,
+    "CLO": _process_NOOP,
+    "END": _process_NOOP,
+    "FIL": _process_NOOP,
+    "MHD": _process_MHD,
+    "MTR": _process_MTR,
+    "SDT": _process_NOOP,
+    "STX": _process_NOOP,
+    "TYP": _process_NOOP,
+    "TTL": _process_NOOP,
+    "VAT": _process_VAT,
+    "VTS": _process_NOOP,
+}
+
+
+class Parser:
+    def __init__(self, file_bytes):
+        self.edi_str = str(file_bytes, "utf-8", errors="ignore")
+        self.line_number = None
+
+    def make_raw_bills(self):
+        bills = []
+        headers = {}
+        bill = None
+        for self.line_number, line, seg_name, elements in parse_edi(self.edi_str):
+            try:
+                func = CODE_FUNCS[seg_name]
+            except KeyError:
+                raise BadRequest(f"Code {seg_name} not recognized.")
+            try:
+                bill = func(elements, headers)
+            except BaseException as e:
+                raise Exception(f"Propblem with segment {line}: {e}") from e
+
+            if "raw_lines" in headers:
+                headers["raw_lines"].append(line)
+            if bill is not None:
+                bills.append(bill)
+
+        return bills

{chellow-1715873476.0.0 → chellow-1715955542.0.0}/chellow/gas/bill_parser_engie_edi.py

@@ -301,8 +301,10 @@ class Parser:
                 func = CODE_FUNCS[seg_name]
             except KeyError:
                 raise BadRequest(f"Code {seg_name} not recognized.")
-
-            bill = func(elements, headers)
+            try:
+                bill = func(elements, headers)
+            except BaseException as e:
+                raise BadRequest(f"Propblem with segment {line}: {e}") from e
 
             if "raw_lines" in headers:
                 headers["raw_lines"].append(line)

{chellow-1715873476.0.0 → chellow-1715955542.0.0}/chellow/gas/views.py

@@ -418,6 +418,16 @@ def batch_get(g_batch_id):
                 vbd["vat"] += g_bill.vat
                 vbd["net"] += g_bill.net
 
+            if "vat" in bd:
+                for vat_percentage, vat_bd in bd["vat"].items():
+                    try:
+                        vbd = vat_breakdown[vat_percentage]
+                    except KeyError:
+                        vbd = vat_breakdown[vat_percentage] = defaultdict(int)
+
+                    vbd["vat"] += vat_bd["vat"]
+                    vbd["net"] += vat_bd["net"]
+
     config_contract = Contract.get_non_core_by_name(g.sess, "configuration")
     properties = config_contract.make_properties()
 

{chellow-1715873476.0.0 → chellow-1715955542.0.0}/chellow/templates/g/bill_import.html

@@ -46,17 +46,19 @@
 					<th>Start Date</th>
 					<th>Finish Date</th>
 					<th>Net GBP</th>
-					<th>VAT 5%</th>
-					<th>VAT 15%</th>
-					<th>VAT 17.5%</th>
-					<th>VAT 20%</th>
 					<th>VAT</th>
 					<th>Gross GBP</th>
-					{% for read in failed_bills[0]['reads'] %}
-						{% for k in read|sort %}
-							<th>{{k}}</th>
-						{% endfor %}
-					{% endfor %}
+					<th>Breakdown</th>
+					<th>R1 calorific_value</th>
+					<th>R1 correction_factor</th>
+					<th>R1 msn</th>
+					<th>R1 unit</th>
+					<th>R1 prev_date</th>
+					<th>R1 prev_value</th>
+					<th>R1 prev_type_code</th>
+					<th>R1 pres_date</th>
+					<th>R1 pres_value</th>
+					<th>R1 pres_type_code</th>
 				</tr>
 			</thead>
 			<tbody>
@@ -71,16 +73,20 @@
 						<td>{{bill.start_date|hh_format}}</td>
 						<td>{{bill.finish_date|hh_format}}</td>
 						<td>{{bill.net_gbp}}</td>
-						<td>{{bill.vat_5pc}}</td>
-						<td>{{bill.vat_15pc}}</td>
-						<td>{{bill.vat_17_5pc}}</td>
-						<td>{{bill.vat_20pc}}</td>
 						<td>{{bill.vat_gbp}}</td>
 						<td>{{bill.gross_gbp}}</td>
+						<td>{{bill.breakdown|dumps}}</td>
 						{% for read in bill.reads %}
-							{% for k, v in read|dictsort %}
-								<td>{{v}}</td>
-							{% endfor %}
+							<td>{{read.calorific_value}}</td>
+							<td>{{read.correction_factor}}</td>
+							<td>{{read.msn}}</td>
+							<td>{{read. unit}}</td>
+							<td>{{read.prev_date|hh_format}}</td>
+							<td>{{read.prev_value}}</td>
+							<td>{{read.prev_type_code}}</td>
+							<td>{{read.pres_date|hh_format}}</td>
+							<td>{{read.pres_value}}</td>
+							<td>{{read.pres_type_code}}</td>
 						{% endfor %}
 					</tr>
 				{% endfor %}

{chellow-1715873476.0.0 → chellow-1715955542.0.0}/chellow/views.py

@@ -67,10 +67,10 @@ import chellow.e.mdd_importer
 import chellow.e.rcrc
 import chellow.e.system_price
 import chellow.e.tlms
-import chellow.edi_lib
 import chellow.general_import
 import chellow.national_grid
 import chellow.rate_server
+from chellow.edi_lib import SEGMENTS, parse_edi
 from chellow.models import (
     BillType,
     Channel,
@@ -644,23 +644,19 @@ def edi_viewer_post():
         else:
             edi_str = req_str("edi_fragment")
 
-        f = StringIO(edi_str)
-        f.seek(0)
-        parser = chellow.edi_lib.EdiParser(f)
-        for segment_name in parser:
-            elements = parser.elements
+        for line_number, line, segment_name, elements in parse_edi(edi_str):
 
             if segment_name == "CCD":
-                segment_name = segment_name + elements[1][0]
+                segment_name = segment_name + elements["CCDE"][0]
                 try:
-                    seg = chellow.edi_lib.SEGMENTS[segment_name]
+                    seg = SEGMENTS[segment_name]
                 except KeyError:
                     raise BadRequest(
                         f"The segment name {segment_name} is not recognized."
                     )
             else:
                 try:
-                    seg = chellow.edi_lib.SEGMENTS[segment_name]
+                    seg = SEGMENTS[segment_name]
                 except KeyError:
                     raise BadRequest(
                         f"The segment name {segment_name} is not recognized."
@@ -669,13 +665,14 @@ def edi_viewer_post():
             titles_element = []
             titles_component = []
             values = []
-            elems = seg["elements"]
+            elems = {el["code"]: el for el in seg["elements"]}
             if len(elements) > len(elems):
                 raise BadRequest(
                     f"There are more elements than recognized for the segment "
-                    f"{segment_name}."
+                    f"{segment_name}. {line}"
                 )
-            for element, elem in zip(elements, elems):
+            for element_code, element in elements.items():
+                elem = elems[element_code]
                 comps = elem["components"]
                 colspan = len(comps)
                 titles_element.append(
@@ -688,8 +685,9 @@ def edi_viewer_post():
                 )
                 if len(element) > len(comps):
                     raise BadRequest(
-                        f"There are more components than recognized for the segment "
-                        f"{segment_name} and element {element}."
+                        f"For the segment {segment_name} the number of components "
+                        f"{element} is greater than the expected components {comps}. "
+                        f"{line}"
                     )
 
                 for i, (title, typ) in enumerate(comps):
@@ -726,7 +724,7 @@ def edi_viewer_post():
                     "titles_element": titles_element,
                     "titles_component": titles_component,
                     "vals": values,
-                    "raw_line": parser.line,
+                    "raw_line": line,
                 }
             )
 

chellow-1715873476.0.0/VERSION

@@ -1 +0,0 @@
-__version__ = "1715873476.0.0"