chellow 1715858374.0.0__tar.gz → 1715955542.0.0__tar.gz

{chellow-1715858374.0.0 → chellow-1715955542.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: chellow
-Version: 1715858374.0.0
+Version: 1715955542.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)

chellow-1715955542.0.0/VERSION

@@ -0,0 +1 @@
+__version__ = "1715955542.0.0"

{chellow-1715858374.0.0 → chellow-1715955542.0.0}/chellow/edi_lib.py

@@ -6,54 +6,56 @@ from werkzeug.exceptions import BadRequest
 from chellow.utils import ct_datetime, to_ct, to_utc
 
 
+def line_iter(f):
+    line = []
+    while c := f.read(1) != "":
+        if c == "'":
+            yield "".join(line)
+            line.clear()
+        else:
+            line.append(c)
+
+
 class EdiParser:
     def __init__(self, f):
-        self.f_iterator = iter(f)
+
+        self.line_iterator = line_iter(f)
         self.line_number = 0
 
     def __iter__(self):
         return self
 
     def __next__(self):
-        self.line = next(self.f_iterator).strip()
+        self.line = next(self.line_iterator).strip()
         self.line_number += 1
 
-        if self.line[-1] != "'":
-            raise BadRequest(
-                f"The parser expects each line to end with a ', but line number "
-                f"{self.line_number} doesn't: {self.line}."
-            )
-        self.elements = [element.split(":") for element in self.line[4:-1].split("+")]
+        self.elements = [element.split(":") for element in self.line[4].split("+")]
         return self.line[:3]
 
 
 def parse_edi(edi_str):
-    for line_number, raw_line in enumerate(edi_str.splitlines(), start=1):
-        if raw_line[-1] != "'":
-            raise BadRequest(
-                f"The parser expects each line to end with a ', but line "
-                f"number {line_number} doesn't: {raw_line}."
-            )
-
+    for line_number, raw_line in enumerate(edi_str.split("'"), start=1):
         line = raw_line.strip()
+        if len(line) == 0:
+            continue
         code = line[:3]
 
-        els = [el.split(":") for el in line[4:-1].split("+")]
+        els = [el.split(":") for el in line[4:].split("+")]
 
-        segment_name = code + els[1][0] if code == "CCD" else code
+        segment_name = code + els[1][0].strip() if code == "CCD" else code
 
         try:
             elem_data = SEGMENTS[segment_name]
         except KeyError:
             raise BadRequest(
                 f"At line number {line_number} the segment name {segment_name} isn't "
-                f"recognized."
+                f"recognized. {raw_line}"
             )
         elem_codes = [m["code"] for m in elem_data["elements"]]
 
         elements = dict(zip(elem_codes, els))
 
-        yield line_number, line, segment_name, elements
+        yield line_number, f"{line}'", segment_name, elements
 
 
 def to_decimal(components):
@@ -67,11 +69,16 @@ def to_decimal(components):
 
 
 def to_ct_date(component):
+    if len(component) == 0:
+        return None
     return to_ct(Datetime.strptime(component, "%y%m%d"))
 
 
 def to_date(component):
-    return to_utc(to_ct_date(component))
+    dt = to_ct_date(component)
+    if dt is None:
+        return None
+    return to_utc(dt)
 
 
 def to_finish_date(component):

chellow-1715955542.0.0/chellow/gas/bill_parser_edi.py

@@ -0,0 +1,403 @@
+from collections import defaultdict
+from datetime import datetime as Datetime
+from decimal import Decimal
+
+from dateutil.relativedelta import relativedelta
+
+from werkzeug.exceptions import BadRequest
+
+from chellow.edi_lib import parse_edi, to_date, to_decimal
+from chellow.utils import HH, to_ct, to_utc
+
+
+READ_TYPE_MAP = {"00": "A", "01": "E", "02": "E"}
+
+TCOD_MAP = {
+    "Energy Bill Discount Scheme": {"PPK": "ebrs"},
+    "Energy Bill Relief Scheme": {"PPK": "ebrs"},
+    "Energy Bill Relief Scheme Discount": {"PPK": "ebrs"},
+    "Unidentified Gas": {"PPK": "ug"},
+    "Commodity": {"PPK": "commodity"},
+    "Transportation": {"PPD": "transportation_fixed", "PPK": "transportation_variable"},
+    "Gas Flexi": {"PPK": "commodity"},
+    "Flex - Gas Flexi (New)": {"PPK": "commodity"},
+    "Meter Reading": {"PPD": "meter_read"},
+    "Meter Reading Credit Oct 19": {"FIX": "meter_read"},
+    "Meter Rental": {"PPD": "metering"},
+    "CCL": {"PPK": "ccl"},
+    "Consumption Based Administration": {"PPK": "admin_variable"},
+    "Swing": {"PPK": "swing"},
+}
+
+SUPPLIER_CODE_MAP = {
+    "STD": "standing",
+    "MET": "commodity",
+    "CCL": "ccl",
+}
+
+UNIT_MAP = {"M3": "M3", "HH": "HCUF", "HCUF": "HCUF"}
+
+
+def _to_finish_date(date_str):
+    if len(date_str) == 0:
+        return None
+    return to_utc(
+        to_ct(Datetime.strptime(date_str, "%y%m%d") + relativedelta(days=1) - HH)
+    )
+
+
+def _process_ADJ(elements, headers):
+    adjf = elements["ADJF"]
+    if adjf[0] == "CV":
+        headers["cv"] = Decimal(adjf[1]) / Decimal(100000)
+
+
+def _process_BCD(elements, headers):
+    ivdt = elements["IVDT"]
+    headers["issue_date"] = to_date(ivdt[0])
+
+    invn = elements["INVN"]
+    headers["reference"] = invn[0]
+
+    btcd = elements["BTCD"]
+    headers["bill_type_code"] = btcd[0]
+
+    sumo = elements["SUMO"]
+    start_date = to_date(sumo[0])
+    if start_date is not None:
+        headers["start_date"] = start_date
+
+    if len(sumo) > 1:
+        finish_date = _to_finish_date(sumo[1])
+        if finish_date is not None:
+            headers["finish_date"] = finish_date
+
+
+def _process_MHD(elements, headers):
+    headers.clear()
+
+    typ = elements["TYPE"]
+    message_type = headers["message_type"] = typ[0]
+    if message_type == "UTLBIL":
+        headers["reads"] = []
+        headers["raw_lines"] = []
+        headers["breakdown"] = defaultdict(int, {"units_consumed": Decimal(0)})
+        headers["kwh"] = Decimal("0.00")
+        headers["net"] = Decimal("0.00")
+        headers["vat"] = Decimal("0.00")
+        headers["gross"] = Decimal("0.00")
+
+
+def _process_CCD1(elements, headers):
+    mtnr = elements["MTNR"]
+    msn = mtnr[0]
+
+    mloc = elements["MLOC"]
+
+    # Bug in EDI where MPRN missing in second CCD 1
+    if "mprn" not in headers:
+        headers["mprn"] = mloc[0]
+
+    prdt = elements["PRDT"]
+    pvdt = elements["PVDT"]
+
+    pres_read_date = to_date(prdt[0])
+    prev_read_date = to_date(pvdt[0])
+
+    prrd = elements["PRRD"]
+    pres_read_value = Decimal(prrd[0])
+    pres_read_type = READ_TYPE_MAP[prrd[1]]
+    prev_read_value = Decimal(prrd[2])
+    prev_read_type = READ_TYPE_MAP[prrd[3]]
+
+    conb = elements["CONB"]
+    unit = UNIT_MAP[conb[1]]
+    headers["breakdown"]["units_consumed"] += to_decimal(conb) / Decimal("1000")
+
+    adjf = elements["ADJF"]
+    correction_factor = Decimal(adjf[1]) / Decimal(100000)
+
+    nuct = elements["NUCT"]
+
+    headers["kwh"] += to_decimal(nuct) / Decimal("1000")
+
+    headers["reads"].append(
+        {
+            "msn": msn,
+            "unit": unit,
+            "correction_factor": correction_factor,
+            "prev_date": prev_read_date,
+            "prev_value": prev_read_value,
+            "prev_type_code": prev_read_type,
+            "pres_date": pres_read_date,
+            "pres_value": pres_read_value,
+            "pres_type_code": pres_read_type,
+        }
+    )
+
+
+NUCT_LOOKUP = {"DAY": "days", "KWH": "kwh"}
+
+
+def _process_CCD2(elements, headers):
+    breakdown = headers["breakdown"]
+    ccde = elements["CCDE"]
+    ccde_supplier_code = ccde[2]
+    tcod = elements["TCOD"]
+    nuct = elements["NUCT"]
+    mtnr = elements["MTNR"]
+    conb = elements["CONB"]
+    adjf = elements["ADJF"]
+    prdt = elements["PRDT"]
+    pvdt = elements["PVDT"]
+    prrd = elements["PRRD"]
+
+    if len(tcod) > 1:
+        tpref_lookup = TCOD_MAP[tcod[1]]
+    else:
+        tpref_lookup = SUPPLIER_CODE_MAP
+
+    tpref = tpref_lookup[ccde_supplier_code]
+
+    bpri = elements["BPRI"]
+    if len(bpri[0]) > 0:
+        rate_key = f"{tpref}_rate"
+        if rate_key not in breakdown:
+            breakdown[rate_key] = set()
+        rate = Decimal(bpri[0]) / Decimal("10000000")
+        breakdown[rate_key].add(rate)
+
+    try:
+        ctot = elements["CTOT"]
+        breakdown[f"{tpref}_gbp"] += to_decimal(ctot) / Decimal("100")
+
+        if len(nuct) > 1:
+            key = NUCT_LOOKUP[nuct[1]]
+        else:
+            if ccde_supplier_code == "PPK":
+                key = f"{tpref}_kwh"
+            elif ccde_supplier_code == "PPD":
+                key = f"{tpref}_days"
+
+        breakdown[key] += to_decimal(nuct) / Decimal("1000")
+    except KeyError:
+        pass
+
+    if "start_date" not in headers:
+        csdt = elements["CSDT"]
+        start_date = to_date(csdt[0])
+        if start_date is not None:
+            headers["start_date"] = start_date
+
+    if "finish_date" not in headers:
+        cedt = elements["CSDT"]
+        finish_date = _to_finish_date(cedt[0])
+        if finish_date is not None:
+            headers["finish_date"] = finish_date
+
+    if "mprn" not in headers:
+        mloc = elements["MLOC"]
+        headers["mprn"] = mloc[0]
+
+    if len(conb) > 0 and len(conb[0]) > 0:
+        headers["breakdown"]["units_consumed"] += to_decimal(conb) / Decimal("1000")
+
+    if len(prrd) > 0 and len(prrd[0]) > 0:
+        pres_read_date = to_date(prdt[0])
+        prev_read_date = to_date(pvdt[0])
+
+        pres_read_value = Decimal(prrd[0])
+        pres_read_type = READ_TYPE_MAP[prrd[1]]
+        prev_read_value = Decimal(prrd[2])
+        prev_read_type = READ_TYPE_MAP[prrd[3]]
+        msn = mtnr[0]
+        unit = UNIT_MAP[conb[1]]
+        correction_factor = Decimal(adjf[1]) / Decimal(100000)
+
+        headers["reads"].append(
+            {
+                "msn": msn,
+                "unit": unit,
+                "correction_factor": correction_factor,
+                "prev_date": prev_read_date,
+                "prev_value": prev_read_value,
+                "prev_type_code": prev_read_type,
+                "pres_date": pres_read_date,
+                "pres_value": pres_read_value,
+                "pres_type_code": pres_read_type,
+            }
+        )
+
+
+def _process_CCD3(elements, headers):
+    breakdown = headers["breakdown"]
+    ccde = elements["CCDE"]
+    ccde_supplier_code = ccde[2]
+    tcod = elements["TCOD"]
+
+    tpref = TCOD_MAP[tcod[1]][ccde_supplier_code]
+
+    bpri = elements["BPRI"]
+    bpri_str = bpri[0]
+    if len(bpri_str) > 0:
+        rate_key = f"{tpref}_rate"
+        if rate_key not in breakdown:
+            breakdown[rate_key] = set()
+        rate = Decimal(bpri_str) / Decimal("10000000")
+        breakdown[rate_key].add(rate)
+
+    nuct = elements["NUCT"]
+
+    try:
+        ctot = elements["CTOT"]
+        breakdown[f"{tpref}_gbp"] += to_decimal(ctot) / Decimal("100")
+
+        if ccde_supplier_code == "PPK":
+            key = f"{tpref}_kwh"
+        elif ccde_supplier_code == "PPD":
+            key = f"{tpref}_days"
+
+        breakdown[key] += to_decimal(nuct) / Decimal("1000")
+    except KeyError:
+        pass
+
+
+def _process_CCD4(elements, headers):
+    breakdown = headers["breakdown"]
+    ccde = elements["ccde"]
+    ccde_supplier_code = ccde[2]
+    tcod = elements["TCOD"]
+
+    tpref = TCOD_MAP[tcod[1]][ccde_supplier_code]
+
+    bpri = elements["BPRI"]
+    rate_key = f"{tpref}_rate"
+    if rate_key not in breakdown:
+        breakdown[rate_key] = set()
+    rate = Decimal(bpri[0]) / Decimal("10000000")
+    breakdown[rate_key].add(rate)
+
+    nuct = elements["NUCT"]
+
+    try:
+        ctot = elements["CTOT"]
+        breakdown[tpref + "_gbp"] += to_decimal(ctot) / Decimal("100")
+
+        if ccde_supplier_code == "PPK":
+            key = f"{tpref}_kwh"
+        elif ccde_supplier_code == "PPD":
+            key = f"{tpref}_days"
+
+        breakdown[key] += to_decimal(nuct) / Decimal("1000")
+    except KeyError:
+        pass
+
+
+def _process_MTR(elements, headers):
+    if headers["message_type"] == "UTLBIL":
+        breakdown = headers["breakdown"]
+        for k, v in tuple(breakdown.items()):
+            if isinstance(v, set):
+                breakdown[k] = sorted(v)
+
+        for read in headers["reads"]:
+            read["calorific_value"] = headers["cv"]
+
+        return {
+            "raw_lines": "\n".join(headers["raw_lines"]),
+            "mprn": headers["mprn"],
+            "reference": headers["reference"],
+            "account": headers["mprn"],
+            "reads": headers["reads"],
+            "kwh": headers["kwh"],
+            "breakdown": headers["breakdown"],
+            "net_gbp": headers["net"],
+            "vat_gbp": headers["vat"],
+            "gross_gbp": headers["gross"],
+            "bill_type_code": headers["bill_type_code"],
+            "start_date": headers["start_date"],
+            "finish_date": headers["finish_date"],
+            "issue_date": headers["issue_date"],
+        }
+
+
+def _process_VAT(elements, headers):
+    breakdown = headers["breakdown"]
+    vatp = elements["VATP"]
+    if "vat" in breakdown:
+        vat = breakdown["vat"]
+    else:
+        vat = breakdown["vat"] = {}
+
+    vat_perc = to_decimal(vatp) / Decimal(1000)
+    try:
+        vat_bd = vat[vat_perc]
+    except KeyError:
+        vat_bd = vat[vat_perc] = defaultdict(int)
+
+    uvtt = elements["UVTT"]
+    vat_gbp = to_decimal(uvtt) / Decimal("100")
+
+    vat_bd["vat"] += vat_gbp
+
+    uvla = elements["UVLA"]
+    net_gbp = to_decimal(uvla) / Decimal("100")
+    vat_bd["net"] += net_gbp
+    headers["net"] += net_gbp
+    headers["vat"] += vat_gbp
+    ucsi = elements["UCSI"]
+    headers["gross"] += to_decimal(ucsi) / Decimal("100")
+
+
+def _process_NOOP(elements, headers):
+    pass
+
+
+CODE_FUNCS = {
+    "ADJ": _process_ADJ,
+    "BCD": _process_BCD,
+    "BTL": _process_NOOP,
+    "CCD1": _process_CCD1,
+    "CCD2": _process_CCD2,
+    "CCD3": _process_CCD3,
+    "CCD4": _process_CCD4,
+    "CDT": _process_NOOP,
+    "CLO": _process_NOOP,
+    "END": _process_NOOP,
+    "FIL": _process_NOOP,
+    "MHD": _process_MHD,
+    "MTR": _process_MTR,
+    "SDT": _process_NOOP,
+    "STX": _process_NOOP,
+    "TYP": _process_NOOP,
+    "TTL": _process_NOOP,
+    "VAT": _process_VAT,
+    "VTS": _process_NOOP,
+}
+
+
+class Parser:
+    def __init__(self, file_bytes):
+        self.edi_str = str(file_bytes, "utf-8", errors="ignore")
+        self.line_number = None
+
+    def make_raw_bills(self):
+        bills = []
+        headers = {}
+        bill = None
+        for self.line_number, line, seg_name, elements in parse_edi(self.edi_str):
+            try:
+                func = CODE_FUNCS[seg_name]
+            except KeyError:
+                raise BadRequest(f"Code {seg_name} not recognized.")
+            try:
+                bill = func(elements, headers)
+            except BaseException as e:
+                raise Exception(f"Propblem with segment {line}: {e}") from e
+
+            if "raw_lines" in headers:
+                headers["raw_lines"].append(line)
+            if bill is not None:
+                bills.append(bill)
+
+        return bills

{chellow-1715858374.0.0 → chellow-1715955542.0.0}/chellow/gas/bill_parser_engie_edi.py

@@ -301,8 +301,10 @@ class Parser:
                 func = CODE_FUNCS[seg_name]
             except KeyError:
                 raise BadRequest(f"Code {seg_name} not recognized.")
-
-            bill = func(elements, headers)
+            try:
+                bill = func(elements, headers)
+            except BaseException as e:
+                raise BadRequest(f"Propblem with segment {line}: {e}") from e
 
             if "raw_lines" in headers:
                 headers["raw_lines"].append(line)

{chellow-1715858374.0.0 → chellow-1715955542.0.0}/chellow/gas/views.py

@@ -418,6 +418,16 @@ def batch_get(g_batch_id):
                 vbd["vat"] += g_bill.vat
                 vbd["net"] += g_bill.net
 
+            if "vat" in bd:
+                for vat_percentage, vat_bd in bd["vat"].items():
+                    try:
+                        vbd = vat_breakdown[vat_percentage]
+                    except KeyError:
+                        vbd = vat_breakdown[vat_percentage] = defaultdict(int)
+
+                    vbd["vat"] += vat_bd["vat"]
+                    vbd["net"] += vat_bd["net"]
+
     config_contract = Contract.get_non_core_by_name(g.sess, "configuration")
     properties = config_contract.make_properties()
 

{chellow-1715858374.0.0 → chellow-1715955542.0.0}/chellow/reports/report_233.py

@@ -7,7 +7,7 @@ from dateutil.relativedelta import relativedelta
 
 from flask import g
 
-from sqlalchemy.sql.expression import true
+from sqlalchemy.sql.expression import false, select, true
 
 from chellow.dloads import open_file
 from chellow.models import (
@@ -21,11 +21,11 @@ from chellow.models import (
     Supply,
     User,
 )
-from chellow.utils import csv_make_val, hh_before, req_int, utc_datetime_now
+from chellow.utils import csv_make_val, hh_before, req_bool, req_int, utc_datetime_now
 from chellow.views import chellow_redirect
 
 
-def content(contract_id, days_hidden, user_id):
+def content(contract_id, days_hidden, is_ignored, user_id):
     f = writer = None
     try:
         with Session() as sess:
@@ -54,15 +54,14 @@ def content(contract_id, days_hidden, user_id):
 
             now = utc_datetime_now()
             cutoff_date = now - relativedelta(days=days_hidden)
-
-            for snag, channel, era, supply, site_era, site in (
-                sess.query(Snag, Channel, Era, Supply, SiteEra, Site)
+            q = (
+                select(Snag, Channel, Era, Supply, SiteEra, Site)
                 .join(Channel, Snag.channel_id == Channel.id)
                 .join(Era, Channel.era_id == Era.id)
                 .join(Supply, Era.supply_id == Supply.id)
                 .join(SiteEra, Era.site_eras)
                 .join(Site, SiteEra.site_id == Site.id)
-                .filter(
+                .where(
                     SiteEra.is_physical == true(),
                     Era.dc_contract == contract,
                     Snag.start_date < cutoff_date,
@@ -76,7 +75,11 @@ def content(contract_id, days_hidden, user_id):
                     Snag.start_date,
                     Snag.id,
                 )
-            ):
+            )
+            if not is_ignored:
+                q = q.where(Snag.is_ignored == false())
+
+            for snag, channel, era, supply, site_era, site in sess.execute(q):
                 snag_start = snag.start_date
                 snag_finish = snag.finish_date
                 imp_mc = "" if era.imp_mpan_core is None else era.imp_mpan_core
@@ -123,7 +126,8 @@ def content(contract_id, days_hidden, user_id):
 def do_get(sess):
     contract_id = req_int("dc_contract_id")
     days_hidden = req_int("days_hidden")
+    is_ignored = req_bool("is_ignored")
 
-    args = contract_id, days_hidden, g.user.id
+    args = contract_id, days_hidden, is_ignored, g.user.id
     threading.Thread(target=content, args=args).start()
     return chellow_redirect("/downloads", 303)

{chellow-1715858374.0.0 → chellow-1715955542.0.0}/chellow/templates/e/channel_snags.html

@@ -17,7 +17,8 @@
 			<legend>Download CSV</legend>	
 			<input type="hidden" name="dc_contract_id" value="{{contract.id}}">
 			<label>Hide snags &lt; days old</label>
-			<input size="3" maxlength="3" name="days_hidden" value="0">
+		 	{{input_text('days_hidden', '0', 3, 3)}}
+			<label>Include ignored snags</label> {{input_checkbox('is_ignored', False)}} 
 			<input type="submit" value="Download">
 		</fieldset>
 	</form>

{chellow-1715858374.0.0 → chellow-1715955542.0.0}/chellow/templates/g/bill_import.html

@@ -46,17 +46,19 @@
 					<th>Start Date</th>
 					<th>Finish Date</th>
 					<th>Net GBP</th>
-					<th>VAT 5%</th>
-					<th>VAT 15%</th>
-					<th>VAT 17.5%</th>
-					<th>VAT 20%</th>
 					<th>VAT</th>
 					<th>Gross GBP</th>
-					{% for read in failed_bills[0]['reads'] %}
-						{% for k in read|sort %}
-							<th>{{k}}</th>
-						{% endfor %}
-					{% endfor %}
+					<th>Breakdown</th>
+					<th>R1 calorific_value</th>
+					<th>R1 correction_factor</th>
+					<th>R1 msn</th>
+					<th>R1 unit</th>
+					<th>R1 prev_date</th>
+					<th>R1 prev_value</th>
+					<th>R1 prev_type_code</th>
+					<th>R1 pres_date</th>
+					<th>R1 pres_value</th>
+					<th>R1 pres_type_code</th>
 				</tr>
 			</thead>
 			<tbody>
@@ -71,16 +73,20 @@
 						<td>{{bill.start_date|hh_format}}</td>
 						<td>{{bill.finish_date|hh_format}}</td>
 						<td>{{bill.net_gbp}}</td>
-						<td>{{bill.vat_5pc}}</td>
-						<td>{{bill.vat_15pc}}</td>
-						<td>{{bill.vat_17_5pc}}</td>
-						<td>{{bill.vat_20pc}}</td>
 						<td>{{bill.vat_gbp}}</td>
 						<td>{{bill.gross_gbp}}</td>
+						<td>{{bill.breakdown|dumps}}</td>
 						{% for read in bill.reads %}
-							{% for k, v in read|dictsort %}
-								<td>{{v}}</td>
-							{% endfor %}
+							<td>{{read.calorific_value}}</td>
+							<td>{{read.correction_factor}}</td>
+							<td>{{read.msn}}</td>
+							<td>{{read. unit}}</td>
+							<td>{{read.prev_date|hh_format}}</td>
+							<td>{{read.prev_value}}</td>
+							<td>{{read.prev_type_code}}</td>
+							<td>{{read.pres_date|hh_format}}</td>
+							<td>{{read.pres_value}}</td>
+							<td>{{read.pres_type_code}}</td>
 						{% endfor %}
 					</tr>
 				{% endfor %}