chellow 1716452334.0.0__tar.gz → 1716903746.0.0__tar.gz

{chellow-1716452334.0.0 → chellow-1716903746.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: chellow
-Version: 1716452334.0.0
+Version: 1716903746.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)

chellow-1716903746.0.0/VERSION

@@ -0,0 +1 @@
+__version__ = "1716903746.0.0"

{chellow-1716452334.0.0 → chellow-1716903746.0.0}/chellow/e/bill_importer.py

@@ -179,7 +179,7 @@ def _process_batch_file(sess, batch_file, log_f):
         raise BadRequest(f"Can't find a parser with the name '{parser_name}'.")
 
     parser = imp_mod.Parser(BytesIO(data))
-    log_f(f"Starting to parse the file with '{parser_name}'.")
+    log_f(f"Starting to parse the file {batch_file.filename} with '{parser_name}'.")
 
     return parser
 

chellow-1716903746.0.0/chellow/e/bill_parsers/mm.py

@@ -0,0 +1,298 @@
+import datetime
+from collections import defaultdict
+from decimal import Decimal
+from io import StringIO
+
+from werkzeug.exceptions import BadRequest
+
+from chellow.utils import HH, to_ct, to_utc
+
+
+def parse_date(date_str):
+    return to_utc(to_ct(parse_date_naive(date_str)))
+
+
+def parse_date_naive(date_str):
+    return datetime.datetime.strptime(date_str, "%Y%m%d")
+
+
+def _chop_record(record, **kwargs):
+    parts = {}
+    idx = 0
+    for name, length in kwargs.items():
+        parts[name] = record[idx : idx + length]
+        idx += length
+    return parts
+
+
+DATE_LENGTH = 8
+
+
+def _handle_0000(headers, pre_record, record):
+    parts = _chop_record(record, unknown_1=9, issue_date=DATE_LENGTH)
+    headers["issue_date"] = parse_date(parts["issue_date"])
+
+
+def _handle_0050(headers, pre_record, record):
+    pass
+
+
+def _handle_0051(headers, pre_record, record):
+    pass
+
+
+def _handle_0100(headers, pre_record, record):
+    issue_date = headers["issue_date"]
+    headers.clear()
+    headers["issue_date"] = issue_date
+    headers["account"] = pre_record[33:41]
+    headers["reference"] = pre_record[41:46]
+    headers["kwh"] = Decimal("0")
+    headers["breakdown"] = defaultdict(int, {"vat": {}})
+    headers["reads"] = []
+
+
+def _handle_0101(headers, pre_record, record):
+    parts = _chop_record(record, start_date=DATE_LENGTH, finish_date=DATE_LENGTH)
+    headers["start_date"] = parse_date(parts["start_date"])
+    headers["finish_date"] = to_utc(to_ct(parse_date_naive(parts["finish_date"]) - HH))
+
+
+CHARGE_UNITS_LOOKUP = {
+    "STDG": "days",
+    "UNIT": "kwh",
+    "AVAL": "kva",
+    "EXAVAL": "kva",
+    "MD": "kw",
+    "LOADU": "kw",
+    "SAG": "days",
+    "TNUOS": "days",
+    "REAP": "kvarh",
+}
+
+ELEMENT_LOOKUP = {
+    "10ANNUAL": "standing",
+    "20RS0108": "unrestricted",
+    "9WANNUAL": "site_fee",
+    "20RS0123": "day",
+    "30RS0123": "night",
+    "90ANNUAL": "duos-fixed",
+    "9QANNUAL": "duos-availability",
+    "9UANNUAL": "duos-excess-availability",
+    "40ANNUAL": "maximum-demand",
+    "20ANNUAL": "triad",
+    "70ANNUAL": "elexon",
+    "10RS0050": "duos-red",
+    "20RS0050": "duos-amber",
+    "30RS0050": "duos-red",
+    "9CANNUAL": "duos-reactive",
+}
+
+
+def _handle_0460(headers, pre_record, record):
+    parts = _chop_record(
+        record,
+        unknown_1=12,
+        unknown_2=12,
+        code=8,
+        quantity=12,
+        units=22,
+        rate=16,
+        unknown_date=DATE_LENGTH,
+        gbp=12,
+        charge_description=35,
+        unknown_3=51,
+        days=2,
+    )
+    units = CHARGE_UNITS_LOOKUP[parts["units"].strip()]
+    gbp = Decimal(parts["gbp"]) / 100
+    quantity = Decimal(parts["quantity"])
+    rate = Decimal(parts["rate"])
+    element_name = ELEMENT_LOOKUP[parts["code"]]
+    breakdown = headers["breakdown"]
+    breakdown[f"{element_name}-{units}"] += quantity
+    rate_name = f"{element_name}-rate"
+    if rate_name in breakdown:
+        rates = breakdown[rate_name]
+    else:
+        rates = breakdown[rate_name] = set()
+
+    rates.add(rate)
+    breakdown[f"{element_name}-gbp"] += gbp
+    if element_name in ("duos-availability", "duos-excess-availability"):
+        breakdown[f"{element_name}-days"] += Decimal(parts["days"])
+
+
+CONSUMPTION_UNITS_LOOKUP = {"KWH": "kwh", "KVA": "kva", "KVARH": "kvarh", "KW": "kw"}
+
+REGISTER_CODE_LOOKUP = {"DAY": "00040", "NIGHT": "00206", "SINGLE": "00001"}
+
+
+def _handle_0461(headers, pre_record, record):
+    parts = _chop_record(
+        record,
+        msn=11,
+        unknown_1=2,
+        prev_read_value=12,
+        pres_read_value=12,
+        coefficient=6,
+        units=6,
+        quantity=12,
+        charge=6,
+        prev_read_type=1,
+        pres_read_type=1,
+        mpan_core=13,
+        mpan_top=8,
+        register_code=19,
+        pres_read_date=DATE_LENGTH,
+        prev_read_date=DATE_LENGTH,
+    )
+    mpan_core = parts["mpan_core"]
+    headers["mpan_core"] = mpan_core
+    units = CONSUMPTION_UNITS_LOOKUP[parts["units"].strip()]
+    if units == "kwh":
+        headers["kwh"] += Decimal(parts["quantity"])
+
+    prev_read_date_str = parts["prev_read_date"].strip()
+    if len(prev_read_date_str) > 0:
+        tpr_code = REGISTER_CODE_LOOKUP[parts["register_code"].strip()]
+
+        headers["reads"].append(
+            {
+                "msn": parts["msn"].strip(),
+                "mpan": f"{parts['mpan_top']} {mpan_core}",
+                "coefficient": Decimal(parts["coefficient"]),
+                "units": units,
+                "tpr_code": tpr_code,
+                "prev_date": parse_date(parts["prev_read_date"]),
+                "prev_value": Decimal(parts["prev_read_value"]),
+                "prev_type_code": parts["prev_read_type"],
+                "pres_date": parse_date(parts["pres_read_date"]),
+                "pres_value": Decimal(parts["pres_read_value"]),
+                "pres_type_code": parts["pres_read_type"],
+            }
+        )
+
+
+def _handle_0470(headers, pre_record, record):
+    pass
+
+
+def _handle_1455(headers, pre_record, record):
+    parts = _chop_record(record, ccl_kwh=13, unknown_1=8, ccl_rate=15, ccl_gbp=13)
+    bd = headers["breakdown"]
+    bd["ccl_kwh"] += Decimal(parts["ccl_kwh"])
+    if "ccl_rate" in bd:
+        ccl_rates = bd["ccl_rate"]
+    else:
+        ccl_rates = bd["ccl_rate"] = set()
+
+    ccl_rates.add(Decimal(parts["ccl_rate"]) / Decimal("100"))
+    bd["ccl_gbp"] += Decimal(parts["ccl_gbp"]) / Decimal("100")
+
+
+def _handle_1460(headers, pre_record, record):
+    parts = _chop_record(record, unknown_1=1, net=12, vat_rate=6, vat=12)
+    net = Decimal(parts["net"]) / Decimal(100)
+    vat_rate = int(Decimal(parts["vat_rate"]))
+    vat = Decimal(parts["vat"]) / Decimal(100)
+
+    vat_breakdown = headers["breakdown"]["vat"]
+    try:
+        vat_bd = vat_breakdown[vat_rate]
+    except KeyError:
+        vat_bd = vat_breakdown[vat_rate] = {"vat": Decimal("0"), "net": Decimal("0")}
+
+    vat_bd["vat"] += vat
+    vat_bd["net"] += net
+
+
+def _handle_1500(headers, pre_record, record):
+    parts = _chop_record(
+        record,
+        unknown_1=8,
+        unknown_2=10,
+        unknown_3=10,
+        unknown_4=10,
+        unknown_5=20,
+        unknown_6=10,
+        unknown_7=10,
+        unknown_8=20,
+        gross=12,
+        net=12,
+        vat=12,
+    )
+    return {
+        "bill_type_code": "N",
+        "mpan_core": headers["mpan_core"],
+        "account": headers["account"],
+        "reference": headers["reference"],
+        "issue_date": headers["issue_date"],
+        "start_date": headers["start_date"],
+        "finish_date": headers["finish_date"],
+        "kwh": headers["kwh"],
+        "net": Decimal("0.00") + Decimal(parts["net"]) / Decimal("100"),
+        "vat": Decimal("0.00") + Decimal(parts["vat"]) / Decimal("100"),
+        "gross": Decimal("0.00") + Decimal(parts["gross"]) / Decimal("100"),
+        "breakdown": headers["breakdown"],
+        "reads": headers["reads"],
+    }
+
+
+def _handle_2000(headers, pre_record, record):
+    pass
+
+
+def _handle_9999(headers, pre_record, record):
+    pass
+
+
+LINE_HANDLERS = {
+    "0000": _handle_0000,
+    "0050": _handle_0050,
+    "0051": _handle_0051,
+    "0100": _handle_0100,
+    "0101": _handle_0101,
+    "0460": _handle_0460,
+    "0461": _handle_0461,
+    "0470": _handle_0470,
+    "1455": _handle_1455,
+    "1460": _handle_1460,
+    "1500": _handle_1500,
+    "2000": _handle_2000,
+    "9999": _handle_9999,
+}
+
+
+class Parser:
+    def __init__(self, f):
+        self.f = StringIO(str(f.read(), "utf-8", errors="ignore"))
+        self.line_number = None
+
+    def make_raw_bills(self):
+        raw_bills = []
+        headers = {}
+        for self.line_number, line in enumerate(self.f, 1):
+            pre_record, record_type, record = line[:80], line[80:84], line[84:]
+            try:
+                handler = LINE_HANDLERS[record_type]
+            except KeyError:
+                raise BadRequest(
+                    f"Record type {record_type} not recognized on line "
+                    f"{self.line_number} {line}"
+                )
+
+            try:
+                bill = handler(headers, pre_record, record)
+            except BadRequest as e:
+                raise BadRequest(
+                    f"Problem at line {self.line_number} {line}: {e.description}"
+                )
+            except BaseException as e:
+                raise Exception(
+                    f"Problem at line {self.line_number} {line}: {e}"
+                ) from e
+            if bill is not None:
+                raw_bills.append(bill)
+
+        return raw_bills

{chellow-1716452334.0.0 → chellow-1716903746.0.0}/chellow/e/views.py

@@ -5252,6 +5252,10 @@ def supplier_bill_import_get(import_id):
             fields["successful_max_registers"] = max(
                 len(bill["reads"]) for bill in imp_fields["successful_bills"]
             )
+        if "failed_bills" in imp_fields and len(imp_fields["failed_bills"]) > 0:
+            fields["failed_max_registers"] = max(
+                len(bill["reads"]) for bill in imp_fields["failed_bills"]
+            )
         fields.update(imp_fields)
         fields["status"] = importer.status()
     return render_template(

{chellow-1716452334.0.0 → chellow-1716903746.0.0}/chellow/templates/e/supplier_bill_import.html

@@ -50,7 +50,19 @@
 					<th>VAT</th>
 					<th>Gross</th>
 					<th>Breakdown</th>
-					<th>Reads</th>
+					{% for i in range(failed_max_registers) %}
+						<th>R{{loop.index}} MPAN</th>
+						<th>R{{loop.index}} Meter Serial Number</th>
+						<th>R{{loop.index}} Coefficient</th>
+						<th>R{{loop.index}} Units</th>
+						<th>R{{loop.index}} TPR</th>
+						<th>R{{loop.index}} Previous Read Date</th>
+						<th>R{{loop.index}} Previous Read Value</th>
+						<th>R{{loop.index}} Previous Read Type</th>
+						<th>R{{loop.index}} Present Read Date</th>
+						<th>R{{loop.index}} Present Read Value</th>
+						<th>R{{loop.index}} Present Read Type</th>
+					{% endfor %}
 				</tr>
 			</thead>
 			<tbody>
@@ -87,7 +99,19 @@
 								<pre>{{bill.breakdown|dumps}}</pre>
 							{% endif %}
 						</td>
-						<td>{{bill.reads}}</td>
+						{% for read in bill.reads %}
+							<td>{{read.mpan}}</td>
+							<td>{{read.msn}}</td>
+							<td>{{read.coefficient}}</td>
+							<td>{{read.units}}</td>
+							<td>{{read.tpr_code}}</td>
+							<td>{{read.prev_date|hh_format}}</td>
+							<td>{{read.prev_value}}</td>
+							<td>{{read.prev_type_code}}</td>
+							<td>{{read.pres_date|hh_format}}</td>
+							<td>{{read.pres_value}}</td>
+							<td>{{read.pres_type_code}}</td>
+						{% endfor %}
 					</tr>
 				{% endfor %}
 			</tbody>

{chellow-1716452334.0.0 → chellow-1716903746.0.0}/test/e/test_bill_importer.py

@@ -17,8 +17,12 @@ def test_supplier_bill_import_html(mocker, app):
             g.config = {}
             batch = mocker.Mock()
             failed_bills = [{"error": "MPAN not found"}]
+            failed_max_registers = 0
             render_template(
-                "/e/supplier_bill_import.html", batch=batch, failed_bills=failed_bills
+                "/e/supplier_bill_import.html",
+                batch=batch,
+                failed_bills=failed_bills,
+                failed_max_registers=failed_max_registers,
             )
 
 

chellow-1716452334.0.0/VERSION

@@ -1 +0,0 @@
-__version__ = "1716452334.0.0"

chellow-1716452334.0.0/chellow/e/bill_parsers/mm.py

@@ -1,56 +0,0 @@
-import datetime
-from decimal import Decimal
-from io import StringIO
-
-from chellow.utils import parse_mpan_core, validate_hh_start
-
-
-def parse_date(date_string):
-    return validate_hh_start(datetime.datetime.strptime(date_string, "%Y%m%d"))
-
-
-class Parser:
-    def __init__(self, f):
-        self.f = StringIO(str(f.read(), "utf-8", errors="ignore"))
-        self.line_number = None
-
-    def make_raw_bills(self):
-        raw_bills = []
-        for i, line in enumerate(self.f):
-            self.line_number = i
-            record_type = line[62:66]
-            if record_type == "0100":
-                account = line[33:41]
-                reference = line[41:46]
-                start_date = None
-                finish_date = None
-                net = Decimal(0)
-                vat = Decimal(0)
-                mpan_core = None
-            elif record_type == "1460":
-                net += Decimal(line[67:79]) / Decimal(100)
-                vat += Decimal(line[85:97]) / Decimal(100)
-            elif record_type == "0461":
-                mpan_core = parse_mpan_core(line[135:148])
-            elif record_type == "0101":
-                start_date = parse_date(line[66:74])
-                finish_date = parse_date(line[74:82])
-            elif record_type == "1500":
-                raw_bill = {
-                    "bill_type_code": "N",
-                    "mpan_core": mpan_core,
-                    "account": account,
-                    "reference": reference,
-                    "issue_date": start_date,
-                    "start_date": start_date,
-                    "finish_date": finish_date,
-                    "kwh": Decimal("0.00"),
-                    "net": net,
-                    "vat": vat,
-                    "gross": Decimal("0.00"),
-                    "breakdown": {},
-                    "reads": [],
-                }
-                raw_bills.append(raw_bill)
-
-        return raw_bills