chellow 1716452334.0.0__py3-none-any.whl → 1716903746.0.0__py3-none-any.whl

chellow/e/bill_importer.py

@@ -179,7 +179,7 @@ def _process_batch_file(sess, batch_file, log_f):
         raise BadRequest(f"Can't find a parser with the name '{parser_name}'.")
 
     parser = imp_mod.Parser(BytesIO(data))
-    log_f(f"Starting to parse the file with '{parser_name}'.")
+    log_f(f"Starting to parse the file {batch_file.filename} with '{parser_name}'.")
 
     return parser
 

chellow/e/bill_parsers/mm.py

@@ -1,12 +1,267 @@
 import datetime
+from collections import defaultdict
 from decimal import Decimal
 from io import StringIO
 
-from chellow.utils import parse_mpan_core, validate_hh_start
+from werkzeug.exceptions import BadRequest
 
+from chellow.utils import HH, to_ct, to_utc
 
-def parse_date(date_string):
-    return validate_hh_start(datetime.datetime.strptime(date_string, "%Y%m%d"))
+
+def parse_date(date_str):
+    return to_utc(to_ct(parse_date_naive(date_str)))
+
+
+def parse_date_naive(date_str):
+    return datetime.datetime.strptime(date_str, "%Y%m%d")
+
+
+def _chop_record(record, **kwargs):
+    parts = {}
+    idx = 0
+    for name, length in kwargs.items():
+        parts[name] = record[idx : idx + length]
+        idx += length
+    return parts
+
+
+DATE_LENGTH = 8
+
+
+def _handle_0000(headers, pre_record, record):
+    parts = _chop_record(record, unknown_1=9, issue_date=DATE_LENGTH)
+    headers["issue_date"] = parse_date(parts["issue_date"])
+
+
+def _handle_0050(headers, pre_record, record):
+    pass
+
+
+def _handle_0051(headers, pre_record, record):
+    pass
+
+
+def _handle_0100(headers, pre_record, record):
+    issue_date = headers["issue_date"]
+    headers.clear()
+    headers["issue_date"] = issue_date
+    headers["account"] = pre_record[33:41]
+    headers["reference"] = pre_record[41:46]
+    headers["kwh"] = Decimal("0")
+    headers["breakdown"] = defaultdict(int, {"vat": {}})
+    headers["reads"] = []
+
+
+def _handle_0101(headers, pre_record, record):
+    parts = _chop_record(record, start_date=DATE_LENGTH, finish_date=DATE_LENGTH)
+    headers["start_date"] = parse_date(parts["start_date"])
+    headers["finish_date"] = to_utc(to_ct(parse_date_naive(parts["finish_date"]) - HH))
+
+
+CHARGE_UNITS_LOOKUP = {
+    "STDG": "days",
+    "UNIT": "kwh",
+    "AVAL": "kva",
+    "EXAVAL": "kva",
+    "MD": "kw",
+    "LOADU": "kw",
+    "SAG": "days",
+    "TNUOS": "days",
+    "REAP": "kvarh",
+}
+
+ELEMENT_LOOKUP = {
+    "10ANNUAL": "standing",
+    "20RS0108": "unrestricted",
+    "9WANNUAL": "site_fee",
+    "20RS0123": "day",
+    "30RS0123": "night",
+    "90ANNUAL": "duos-fixed",
+    "9QANNUAL": "duos-availability",
+    "9UANNUAL": "duos-excess-availability",
+    "40ANNUAL": "maximum-demand",
+    "20ANNUAL": "triad",
+    "70ANNUAL": "elexon",
+    "10RS0050": "duos-red",
+    "20RS0050": "duos-amber",
+    "30RS0050": "duos-red",
+    "9CANNUAL": "duos-reactive",
+}
+
+
+def _handle_0460(headers, pre_record, record):
+    parts = _chop_record(
+        record,
+        unknown_1=12,
+        unknown_2=12,
+        code=8,
+        quantity=12,
+        units=22,
+        rate=16,
+        unknown_date=DATE_LENGTH,
+        gbp=12,
+        charge_description=35,
+        unknown_3=51,
+        days=2,
+    )
+    units = CHARGE_UNITS_LOOKUP[parts["units"].strip()]
+    gbp = Decimal(parts["gbp"]) / 100
+    quantity = Decimal(parts["quantity"])
+    rate = Decimal(parts["rate"])
+    element_name = ELEMENT_LOOKUP[parts["code"]]
+    breakdown = headers["breakdown"]
+    breakdown[f"{element_name}-{units}"] += quantity
+    rate_name = f"{element_name}-rate"
+    if rate_name in breakdown:
+        rates = breakdown[rate_name]
+    else:
+        rates = breakdown[rate_name] = set()
+
+    rates.add(rate)
+    breakdown[f"{element_name}-gbp"] += gbp
+    if element_name in ("duos-availability", "duos-excess-availability"):
+        breakdown[f"{element_name}-days"] += Decimal(parts["days"])
+
+
+CONSUMPTION_UNITS_LOOKUP = {"KWH": "kwh", "KVA": "kva", "KVARH": "kvarh", "KW": "kw"}
+
+REGISTER_CODE_LOOKUP = {"DAY": "00040", "NIGHT": "00206", "SINGLE": "00001"}
+
+
+def _handle_0461(headers, pre_record, record):
+    parts = _chop_record(
+        record,
+        msn=11,
+        unknown_1=2,
+        prev_read_value=12,
+        pres_read_value=12,
+        coefficient=6,
+        units=6,
+        quantity=12,
+        charge=6,
+        prev_read_type=1,
+        pres_read_type=1,
+        mpan_core=13,
+        mpan_top=8,
+        register_code=19,
+        pres_read_date=DATE_LENGTH,
+        prev_read_date=DATE_LENGTH,
+    )
+    mpan_core = parts["mpan_core"]
+    headers["mpan_core"] = mpan_core
+    units = CONSUMPTION_UNITS_LOOKUP[parts["units"].strip()]
+    if units == "kwh":
+        headers["kwh"] += Decimal(parts["quantity"])
+
+    prev_read_date_str = parts["prev_read_date"].strip()
+    if len(prev_read_date_str) > 0:
+        tpr_code = REGISTER_CODE_LOOKUP[parts["register_code"].strip()]
+
+        headers["reads"].append(
+            {
+                "msn": parts["msn"].strip(),
+                "mpan": f"{parts['mpan_top']} {mpan_core}",
+                "coefficient": Decimal(parts["coefficient"]),
+                "units": units,
+                "tpr_code": tpr_code,
+                "prev_date": parse_date(parts["prev_read_date"]),
+                "prev_value": Decimal(parts["prev_read_value"]),
+                "prev_type_code": parts["prev_read_type"],
+                "pres_date": parse_date(parts["pres_read_date"]),
+                "pres_value": Decimal(parts["pres_read_value"]),
+                "pres_type_code": parts["pres_read_type"],
+            }
+        )
+
+
+def _handle_0470(headers, pre_record, record):
+    pass
+
+
+def _handle_1455(headers, pre_record, record):
+    parts = _chop_record(record, ccl_kwh=13, unknown_1=8, ccl_rate=15, ccl_gbp=13)
+    bd = headers["breakdown"]
+    bd["ccl_kwh"] += Decimal(parts["ccl_kwh"])
+    if "ccl_rate" in bd:
+        ccl_rates = bd["ccl_rate"]
+    else:
+        ccl_rates = bd["ccl_rate"] = set()
+
+    ccl_rates.add(Decimal(parts["ccl_rate"]) / Decimal("100"))
+    bd["ccl_gbp"] += Decimal(parts["ccl_gbp"]) / Decimal("100")
+
+
+def _handle_1460(headers, pre_record, record):
+    parts = _chop_record(record, unknown_1=1, net=12, vat_rate=6, vat=12)
+    net = Decimal(parts["net"]) / Decimal(100)
+    vat_rate = int(Decimal(parts["vat_rate"]))
+    vat = Decimal(parts["vat"]) / Decimal(100)
+
+    vat_breakdown = headers["breakdown"]["vat"]
+    try:
+        vat_bd = vat_breakdown[vat_rate]
+    except KeyError:
+        vat_bd = vat_breakdown[vat_rate] = {"vat": Decimal("0"), "net": Decimal("0")}
+
+    vat_bd["vat"] += vat
+    vat_bd["net"] += net
+
+
+def _handle_1500(headers, pre_record, record):
+    parts = _chop_record(
+        record,
+        unknown_1=8,
+        unknown_2=10,
+        unknown_3=10,
+        unknown_4=10,
+        unknown_5=20,
+        unknown_6=10,
+        unknown_7=10,
+        unknown_8=20,
+        gross=12,
+        net=12,
+        vat=12,
+    )
+    return {
+        "bill_type_code": "N",
+        "mpan_core": headers["mpan_core"],
+        "account": headers["account"],
+        "reference": headers["reference"],
+        "issue_date": headers["issue_date"],
+        "start_date": headers["start_date"],
+        "finish_date": headers["finish_date"],
+        "kwh": headers["kwh"],
+        "net": Decimal("0.00") + Decimal(parts["net"]) / Decimal("100"),
+        "vat": Decimal("0.00") + Decimal(parts["vat"]) / Decimal("100"),
+        "gross": Decimal("0.00") + Decimal(parts["gross"]) / Decimal("100"),
+        "breakdown": headers["breakdown"],
+        "reads": headers["reads"],
+    }
+
+
+def _handle_2000(headers, pre_record, record):
+    pass
+
+
+def _handle_9999(headers, pre_record, record):
+    pass
+
+
+LINE_HANDLERS = {
+    "0000": _handle_0000,
+    "0050": _handle_0050,
+    "0051": _handle_0051,
+    "0100": _handle_0100,
+    "0101": _handle_0101,
+    "0460": _handle_0460,
+    "0461": _handle_0461,
+    "0470": _handle_0470,
+    "1455": _handle_1455,
+    "1460": _handle_1460,
+    "1500": _handle_1500,
+    "2000": _handle_2000,
+    "9999": _handle_9999,
+}
 
 
 class Parser:
@@ -16,41 +271,28 @@ class Parser:
 
     def make_raw_bills(self):
         raw_bills = []
-        for i, line in enumerate(self.f):
-            self.line_number = i
-            record_type = line[62:66]
-            if record_type == "0100":
-                account = line[33:41]
-                reference = line[41:46]
-                start_date = None
-                finish_date = None
-                net = Decimal(0)
-                vat = Decimal(0)
-                mpan_core = None
-            elif record_type == "1460":
-                net += Decimal(line[67:79]) / Decimal(100)
-                vat += Decimal(line[85:97]) / Decimal(100)
-            elif record_type == "0461":
-                mpan_core = parse_mpan_core(line[135:148])
-            elif record_type == "0101":
-                start_date = parse_date(line[66:74])
-                finish_date = parse_date(line[74:82])
-            elif record_type == "1500":
-                raw_bill = {
-                    "bill_type_code": "N",
-                    "mpan_core": mpan_core,
-                    "account": account,
-                    "reference": reference,
-                    "issue_date": start_date,
-                    "start_date": start_date,
-                    "finish_date": finish_date,
-                    "kwh": Decimal("0.00"),
-                    "net": net,
-                    "vat": vat,
-                    "gross": Decimal("0.00"),
-                    "breakdown": {},
-                    "reads": [],
-                }
-                raw_bills.append(raw_bill)
+        headers = {}
+        for self.line_number, line in enumerate(self.f, 1):
+            pre_record, record_type, record = line[:80], line[80:84], line[84:]
+            try:
+                handler = LINE_HANDLERS[record_type]
+            except KeyError:
+                raise BadRequest(
+                    f"Record type {record_type} not recognized on line "
+                    f"{self.line_number} {line}"
+                )
+
+            try:
+                bill = handler(headers, pre_record, record)
+            except BadRequest as e:
+                raise BadRequest(
+                    f"Problem at line {self.line_number} {line}: {e.description}"
+                )
+            except BaseException as e:
+                raise Exception(
+                    f"Problem at line {self.line_number} {line}: {e}"
+                ) from e
+            if bill is not None:
+                raw_bills.append(bill)
 
         return raw_bills

chellow/e/views.py

@@ -5252,6 +5252,10 @@ def supplier_bill_import_get(import_id):
             fields["successful_max_registers"] = max(
                 len(bill["reads"]) for bill in imp_fields["successful_bills"]
             )
+        if "failed_bills" in imp_fields and len(imp_fields["failed_bills"]) > 0:
+            fields["failed_max_registers"] = max(
+                len(bill["reads"]) for bill in imp_fields["failed_bills"]
+            )
         fields.update(imp_fields)
         fields["status"] = importer.status()
     return render_template(

chellow/templates/e/supplier_bill_import.html

@@ -50,7 +50,19 @@
 					<th>VAT</th>
 					<th>Gross</th>
 					<th>Breakdown</th>
-					<th>Reads</th>
+					{% for i in range(failed_max_registers) %}
+						<th>R{{loop.index}} MPAN</th>
+						<th>R{{loop.index}} Meter Serial Number</th>
+						<th>R{{loop.index}} Coefficient</th>
+						<th>R{{loop.index}} Units</th>
+						<th>R{{loop.index}} TPR</th>
+						<th>R{{loop.index}} Previous Read Date</th>
+						<th>R{{loop.index}} Previous Read Value</th>
+						<th>R{{loop.index}} Previous Read Type</th>
+						<th>R{{loop.index}} Present Read Date</th>
+						<th>R{{loop.index}} Present Read Value</th>
+						<th>R{{loop.index}} Present Read Type</th>
+					{% endfor %}
 				</tr>
 			</thead>
 			<tbody>
@@ -87,7 +99,19 @@
 								<pre>{{bill.breakdown|dumps}}</pre>
 							{% endif %}
 						</td>
-						<td>{{bill.reads}}</td>
+						{% for read in bill.reads %}
+							<td>{{read.mpan}}</td>
+							<td>{{read.msn}}</td>
+							<td>{{read.coefficient}}</td>
+							<td>{{read.units}}</td>
+							<td>{{read.tpr_code}}</td>
+							<td>{{read.prev_date|hh_format}}</td>
+							<td>{{read.prev_value}}</td>
+							<td>{{read.prev_type_code}}</td>
+							<td>{{read.pres_date|hh_format}}</td>
+							<td>{{read.pres_value}}</td>
+							<td>{{read.pres_type_code}}</td>
+						{% endfor %}
 					</tr>
 				{% endfor %}
 			</tbody>

{chellow-1716452334.0.0.dist-info → chellow-1716903746.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: chellow
-Version: 1716452334.0.0
+Version: 1716903746.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)

{chellow-1716452334.0.0.dist-info → chellow-1716903746.0.0.dist-info}/RECORD

@@ -14,7 +14,7 @@ chellow/utils.py,sha256=32qPWEGzCPKPhSM7ztpzcR6ZG74sVZanScDIdK50Rq4,19308
 chellow/views.py,sha256=eDvTQM_PUqRvrCQvBdlF5q7MEs7w2yJmRjcC8WDbHtE,79584
 chellow/e/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 chellow/e/aahedc.py,sha256=d2usudp7KYWpU6Pk3fal5EQ47EbvkvKeaFGylnb3NWw,606
-chellow/e/bill_importer.py,sha256=y1bpn49xDwltj0PRFowWsjAVamcD8eyxUbCTdGxEZiE,7389
+chellow/e/bill_importer.py,sha256=7UcnqNlKbJc2GhW9gy8sDp9GuqambJVpZLvbafOZztA,7411
 chellow/e/bmarketidx.py,sha256=0JCBBnP3xfLq2eFjXO_u-gzHyTizab4Pp6PRK6OZLcw,7134
 chellow/e/bsuos.py,sha256=hdP9vnOJSuZl46OAkJeUg1XJYvYIBj4J6Sqce1Hy9Vs,15542
 chellow/e/ccl.py,sha256=30dh_SvlgzsTQPPAJNZWILaMvbeDsv9-P-S1JxS5_SQ,3184
@@ -38,7 +38,7 @@ chellow/e/system_price.py,sha256=6w5J7bzwFAZubE2zdOFRiS8IIrVP8hkoIOaG2yCt-Ic,623
 chellow/e/tlms.py,sha256=M33D6YpMixu2KkwSCzDRM3kThLgShg8exp63Obo75l8,8905
 chellow/e/tnuos.py,sha256=XseYztPUsQXNKuBmystO2kzzwAG9ehCZgpGBTdgSk-A,4313
 chellow/e/triad.py,sha256=lIQj7EdUrcFwEqleuHZXYU_bfzIwNOqUVVxB3NPQt4A,13710
-chellow/e/views.py,sha256=es1lQWgZjCcHQ7AR_98SM9hLJjX8UUarePuBHnkkJ_4,215037
+chellow/e/views.py,sha256=RQLiAzzWm5RUumvhR1rub1yFTMZ9uksqJIc1IShnXsQ,215256
 chellow/e/bill_parsers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 chellow/e/bill_parsers/activity_mop_stark_xlsx.py,sha256=UgWXDPzQkQghyj_lfgBqoSJpHB-t-qOdSaB8qY6GLog,4071
 chellow/e/bill_parsers/annual_mop_stark_xlsx.py,sha256=-HMoIfa_utXYKA44RuC0Xqv3vd2HLeQU_4P0iBUd3WA,4219
@@ -51,7 +51,7 @@ chellow/e/bill_parsers/gdf_csv.py,sha256=ZfK3Oc6oP28p_P9DIevLNB_zW2WLcEJ3Lvb1gL3
 chellow/e/bill_parsers/haven_csv.py,sha256=0uENq8IgVNqdxfBQMBxLTSZWCOuDHXZC0xzk52SbfyE,13652
 chellow/e/bill_parsers/haven_edi.py,sha256=YGPHRxPOhje9s32jqPHHELni2tooOYj3cMC_qaZVPq4,16107
 chellow/e/bill_parsers/haven_edi_tprs.py,sha256=ZVX9CCqUybsot_Z0BEOJPvl9x5kSr7fEWyuJXvZDcz4,11841
-chellow/e/bill_parsers/mm.py,sha256=P4CdkskDrjmFMSDp0ehE_ichTBHGywbT_dckMxuAqQQ,1929
+chellow/e/bill_parsers/mm.py,sha256=DEy-W-Z03TdfUjtFUNk48h5nEAaXKV3etfrPZ-z001Q,8494
 chellow/e/bill_parsers/nonsettlement_dc_stark_xlsx.py,sha256=yogXTuQHGRL7IiqvRWr2C9V24ez1j9Yx0128UygPE_k,4723
 chellow/e/bill_parsers/settlement_dc_stark_xlsx.py,sha256=gKeYMdUO4bVycV8n1lWs5AIfF3bHZLkM6tkasD4rhHs,6239
 chellow/e/bill_parsers/sse_edi.py,sha256=L85DOfNkqexeEIEr8pCBn_2sHJI-zEaw6cogpE3YyYM,15204
@@ -299,7 +299,7 @@ chellow/templates/e/supplier_batches.html,sha256=ij8LuHc2HzxovBn34aHCMra4pm3Himk
 chellow/templates/e/supplier_bill.html,sha256=7q79XfUSFPLR5npIXhRpTciQ9rGtNHhp9ryFZWXBhPc,4142
 chellow/templates/e/supplier_bill_add.html,sha256=BsD-Zh7d9auiqJ61VPHiQrP8u8rTcw09mEKuKH9dxaA,2100
 chellow/templates/e/supplier_bill_edit.html,sha256=oxZrMcMwrvluJSPxD4yfM9mWNeugoguAwT_ai9Ynl88,2732
-chellow/templates/e/supplier_bill_import.html,sha256=XSDURJKvXvI3ee15VqYoUpmF8Ng61Puybwexzk1AFF0,4291
+chellow/templates/e/supplier_bill_import.html,sha256=2_VvBoNE838UwuN0AiMwIrzqxQmWBo5DGY0lY833Bpk,5265
 chellow/templates/e/supplier_bill_imports.html,sha256=9iTNGWKn9XjQTBP1Sepbo0QVNlgKh7EfizXapam7I6s,9356
 chellow/templates/e/supplier_contract.html,sha256=oVGrYGIbCb-7MmpFQaXeTNRKBKy8GoGCWICWgo5E4xI,2863
 chellow/templates/e/supplier_contract_add.html,sha256=gsozEtF24lzYi_Bb4LTenvh62tCt7dQ4CwaIz7rFck4,899
@@ -364,6 +364,6 @@ chellow/templates/g/supply_note_edit.html,sha256=6UQf_qbhFDys3cVsTp-c7ABWZpggW9R
 chellow/templates/g/supply_notes.html,sha256=WR3YwGh_qqTklSJ7JqWX6BKBc9rk_jMff4RiWZiF2CM,936
 chellow/templates/g/unit.html,sha256=KouNVU0-i84afANkLQ_heJ0uDfJ9H5A05PuLqb8iCN8,438
 chellow/templates/g/units.html,sha256=p5Nd-lAIboKPEOO6N451hx1bcKxMg4BDODnZ-43MmJc,441
-chellow-1716452334.0.0.dist-info/METADATA,sha256=pjgx-WKJRzgK4cjpaGC2sDSgRDU2AQS0gxnApoP6NnI,12205
-chellow-1716452334.0.0.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
-chellow-1716452334.0.0.dist-info/RECORD,,
+chellow-1716903746.0.0.dist-info/METADATA,sha256=1b3az19NrZfzrNS1rPZPOjvDnuejOViqXe0ubM4A0-Q,12205
+chellow-1716903746.0.0.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87
+chellow-1716903746.0.0.dist-info/RECORD,,