chellow 1716370405.0.0__tar.gz → 1716564585.0.0__tar.gz

{chellow-1716370405.0.0 → chellow-1716564585.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: chellow
-Version: 1716370405.0.0
+Version: 1716564585.0.0
 Summary: Web Application for checking UK energy bills.
 Project-URL: Homepage, https://github.com/WessexWater/chellow
 Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
@@ -9,7 +9,7 @@ Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.9
 Requires-Dist: flask-restx==1.2.0
 Requires-Dist: flask==2.3.3
-Requires-Dist: odio==0.0.22
+Requires-Dist: odio==0.0.23
 Requires-Dist: openpyxl==3.1.2
 Requires-Dist: paramiko==3.4.0
 Requires-Dist: pep3143daemon==0.0.6
@@ -21,7 +21,7 @@ Requires-Dist: pypdf==3.17.0
 Requires-Dist: python-dateutil==2.8.2
 Requires-Dist: pytz==2022.6
 Requires-Dist: requests==2.32.2
-Requires-Dist: sqlalchemy==2.0.28
+Requires-Dist: sqlalchemy==2.0.30
 Requires-Dist: waitress==2.1.2
 Requires-Dist: xlrd==2.0.1
 Requires-Dist: zish==0.1.11

chellow-1716564585.0.0/VERSION

@@ -0,0 +1 @@
+__version__ = "1716564585.0.0"

chellow-1716564585.0.0/chellow/e/bill_parsers/mm.py

@@ -0,0 +1,253 @@
+import datetime
+from collections import defaultdict
+from decimal import Decimal
+from io import StringIO
+
+from werkzeug.exceptions import BadRequest
+
+from chellow.utils import HH, to_ct, to_utc
+
+
+def parse_date(date_str):
+    return to_utc(to_ct(parse_date_naive(date_str)))
+
+
+def parse_date_naive(date_str):
+    return datetime.datetime.strptime(date_str, "%Y%m%d")
+
+
+def _chop_record(record, **kwargs):
+    parts = {}
+    idx = 0
+    for name, length in kwargs.items():
+        parts[name] = record[idx : idx + length]
+        idx += length
+    return parts
+
+
+DATE_LENGTH = 8
+
+
+def _handle_0000(headers, pre_record, record):
+    parts = _chop_record(record, unknown_1=9, issue_date=DATE_LENGTH)
+    headers["issue_date"] = parse_date(parts["issue_date"])
+
+
+def _handle_0050(headers, pre_record, record):
+    pass
+
+
+def _handle_0051(headers, pre_record, record):
+    pass
+
+
+def _handle_0100(headers, pre_record, record):
+    issue_date = headers["issue_date"]
+    headers.clear()
+    headers["issue_date"] = issue_date
+    headers["account"] = pre_record[33:41]
+    headers["reference"] = pre_record[41:46]
+    headers["kwh"] = Decimal("0")
+    headers["breakdown"] = defaultdict(int, {"vat": {}})
+    headers["reads"] = []
+
+
+def _handle_0101(headers, pre_record, record):
+    parts = _chop_record(record, start_date=DATE_LENGTH, finish_date=DATE_LENGTH)
+    headers["start_date"] = parse_date(parts["start_date"])
+    headers["finish_date"] = to_utc(to_ct(parse_date_naive(parts["finish_date"]) - HH))
+
+
+CHARGE_LOOKUP = {
+    "STDG": "days",
+    "UNIT": "kwh",
+}
+
+ELEMENT_LOOKUP = {"10ANNUAL": "standing", "20RS0108": "0001", "9WANNUAL": "site_fee"}
+
+
+def _handle_0460(headers, pre_record, record):
+    parts = _chop_record(
+        record,
+        unknown_1=12,
+        gbp=12,
+        code=8,
+        quantity=12,
+        charge=4,
+        unknown_2=18,
+        rate=16,
+        unknown_date=DATE_LENGTH,
+        another_gbp=12,
+        charge_description=35,
+    )
+    units = CHARGE_LOOKUP[parts["charge"]]
+    gbp = Decimal(parts["gbp"]) / 100
+    quantity = Decimal(parts["quantity"])
+    rate = Decimal(parts["rate"])
+    element_name = ELEMENT_LOOKUP[parts["code"]]
+    breakdown = headers["breakdown"]
+    breakdown[f"{element_name}-{units}"] += quantity
+    rate_name = f"{element_name}-rate"
+    if rate_name in breakdown:
+        rates = breakdown[rate_name]
+    else:
+        rates = breakdown[rate_name] = set()
+
+    rates.add(rate)
+    breakdown[f"{element_name}-gbp"] += gbp
+
+
+UNITS_LOOKUP = {"KWH": "kwh"}
+
+REGISTER_CODE_LOOKUP = {"000001": "0001"}
+
+
+def _handle_0461(headers, pre_record, record):
+    parts = _chop_record(
+        record,
+        msn=11,
+        unknown_1=2,
+        prev_read_value=12,
+        pres_read_value=12,
+        register_code=6,
+        units=6,
+        quantity=12,
+        charge=6,
+        prev_read_type=1,
+        pres_read_type=1,
+        mpan_core=13,
+        mpan_top=8,
+        tariff_code=19,
+        pres_read_date=DATE_LENGTH,
+        prev_read_date=DATE_LENGTH,
+    )
+    mpan_core = parts["mpan_core"]
+    headers["mpan_core"] = mpan_core
+    units = UNITS_LOOKUP[parts["units"].strip()]
+    if units == "kwh":
+        headers["kwh"] += Decimal(parts["quantity"])
+    tpr_code = REGISTER_CODE_LOOKUP[parts["register_code"]]
+
+    headers["reads"].append(
+        {
+            "msn": parts["msn"].strip(),
+            "mpan": f"{parts['mpan_top']} {mpan_core}",
+            "coefficient": 1,
+            "units": units,
+            "tpr_code": tpr_code,
+            "prev_date": parse_date(parts["prev_read_date"]),
+            "prev_value": Decimal(parts["prev_read_value"]),
+            "prev_type_code": parts["prev_read_type"],
+            "pres_date": parse_date(parts["pres_read_date"]),
+            "pres_value": Decimal(parts["pres_read_value"]),
+            "pres_type_code": parts["pres_read_type"],
+        }
+    )
+
+
+def _handle_0470(headers, pre_record, record):
+    pass
+
+
+def _handle_1460(headers, pre_record, record):
+    parts = _chop_record(record, unknown_1=1, net=12, vat_rate=5, vat=12)
+    net = Decimal(parts["net"]) / Decimal(100)
+    vat_rate = int(Decimal(parts["vat_rate"]))
+    vat = Decimal(parts["vat"]) / Decimal(100)
+
+    vat_breakdown = headers["breakdown"]["vat"]
+    try:
+        vat_bd = vat_breakdown[vat_rate]
+    except KeyError:
+        vat_bd = vat_breakdown[vat_rate] = {"vat": Decimal("0"), "net": Decimal("0")}
+
+    vat_bd["vat"] += vat
+    vat_bd["net"] += net
+
+
+def _handle_1500(headers, pre_record, record):
+    parts = _chop_record(
+        record,
+        unknown_1=8,
+        unknown_2=10,
+        net=10,
+        unknown_3=10,
+        unknown_4=20,
+        vat=10,
+        unknown_5=10,
+        unknown_6=20,
+        gross=12,
+    )
+    return {
+        "bill_type_code": "N",
+        "mpan_core": headers["mpan_core"],
+        "account": headers["account"],
+        "reference": headers["reference"],
+        "issue_date": headers["issue_date"],
+        "start_date": headers["start_date"],
+        "finish_date": headers["finish_date"],
+        "kwh": headers["kwh"],
+        "net": Decimal(parts["net"]),
+        "vat": Decimal(parts["vat"]),
+        "gross": Decimal(parts["gross"]),
+        "breakdown": headers["breakdown"],
+        "reads": headers["reads"],
+    }
+
+
+def _handle_2000(headers, pre_record, record):
+    pass
+
+
+def _handle_9999(headers, pre_record, record):
+    pass
+
+
+LINE_HANDLERS = {
+    "0000": _handle_0000,
+    "0050": _handle_0050,
+    "0051": _handle_0051,
+    "0100": _handle_0100,
+    "0101": _handle_0101,
+    "0460": _handle_0460,
+    "0461": _handle_0461,
+    "0470": _handle_0470,
+    "1460": _handle_1460,
+    "1500": _handle_1500,
+    "2000": _handle_2000,
+    "9999": _handle_9999,
+}
+
+
+class Parser:
+    def __init__(self, f):
+        self.f = StringIO(str(f.read(), "utf-8", errors="ignore"))
+        self.line_number = None
+
+    def make_raw_bills(self):
+        raw_bills = []
+        headers = {}
+        for self.line_number, line in enumerate(self.f, 1):
+            pre_record, record_type, record = line[:80], line[80:84], line[84:]
+            try:
+                handler = LINE_HANDLERS[record_type]
+            except KeyError:
+                raise BadRequest(
+                    f"Record type {record_type} not recognized on line "
+                    f"{self.line_number} {line}"
+                )
+
+            try:
+                bill = handler(headers, pre_record, record)
+            except BadRequest as e:
+                raise BadRequest(
+                    f"Problem at line {self.line_number} {line}: {e.description}"
+                )
+            except BaseException as e:
+                raise Exception(
+                    f"Problem at line {self.line_number} {line}: {e}"
+                ) from e
+            if bill is not None:
+                raw_bills.append(bill)
+
+        return raw_bills

{chellow-1716370405.0.0 → chellow-1716564585.0.0}/chellow/e/views.py

@@ -5252,6 +5252,10 @@ def supplier_bill_import_get(import_id):
             fields["successful_max_registers"] = max(
                 len(bill["reads"]) for bill in imp_fields["successful_bills"]
             )
+        if "failed_bills" in imp_fields and len(imp_fields["failed_bills"]) > 0:
+            fields["failed_max_registers"] = max(
+                len(bill["reads"]) for bill in imp_fields["failed_bills"]
+            )
         fields.update(imp_fields)
         fields["status"] = importer.status()
     return render_template(

{chellow-1716370405.0.0 → chellow-1716564585.0.0}/chellow/models.py

@@ -6400,8 +6400,23 @@ class ReportRun(Base, PersistentClass):
     def insert(sess, name, user, title, data):
         report_run = ReportRun(name, user, title, data)
         sess.add(report_run)
+        sess.flush()
         return report_run
 
+    @staticmethod
+    def w_update(report_run_id, state):
+        with Session() as wsess:
+            report_run = ReportRun.get_by_id(wsess, report_run_id)
+            report_run.update(state)
+            wsess.commit()
+
+    @staticmethod
+    def w_insert_row(report_run_id, tab, titles, values, properties):
+        with Session() as wsess:
+            report_run = ReportRun.get_by_id(wsess, report_run_id)
+            report_run.insert_row(wsess, tab, titles, values, properties)
+            wsess.commit()
+
 
 def _jsonize(val):
     if isinstance(val, dict):

{chellow-1716370405.0.0 → chellow-1716564585.0.0}/chellow/reports/report_111.py

@@ -29,9 +29,9 @@ from chellow.models import (
     Llfc,
     MarketRole,
     MtcParticipant,
+    RSession,
     RegisterRead,
     ReportRun,
-    Session,
     Site,
     SiteEra,
     Supply,
@@ -102,19 +102,17 @@ def content(
     report_run_id,
 ):
     caches = {}
-    tmp_file = sess = supply_id = report_run = None
+    tmp_file = sess = supply_id = None
     forecast_date = to_utc(Datetime.max)
 
     try:
-        with Session() as sess:
+        with RSession() as sess:
             user = User.get_by_id(sess, user_id)
             tmp_file = open_file(
                 f"bill_check_{fname_additional}.csv", user, mode="w", newline=""
             )
             writer = csv.writer(tmp_file, lineterminator="\n")
 
-            report_run = ReportRun.get_by_id(sess, report_run_id)
-
             bills = (
                 sess.query(Bill)
                 .order_by(Bill.supply_id, Bill.reference)
@@ -216,30 +214,26 @@ def content(
                     virtual_bill_titles,
                     writer,
                     titles,
-                    report_run,
+                    report_run_id,
                 )
-
-            report_run.update("finished")
-            sess.commit()
+        ReportRun.w_update(report_run_id, "finished")
 
     except BadRequest as e:
-        if report_run is not None:
-            report_run.update("problem")
         if supply_id is None:
             prefix = "Problem: "
         else:
             prefix = f"Problem with supply {supply_id}:"
         tmp_file.write(prefix + e.description)
+        ReportRun.w_update(report_run_id, "problem")
     except BaseException:
-        if report_run is not None:
-            report_run.update("interrupted")
+        msg = traceback.format_exc()
+        sys.stderr.write(msg + "\n")
         if supply_id is None:
             prefix = "Problem: "
         else:
             prefix = f"Problem with supply {supply_id}:"
-        msg = traceback.format_exc()
-        sys.stderr.write(msg + "\n")
         tmp_file.write(prefix + msg)
+        ReportRun.w_update(report_run_id, "interrupted")
     finally:
         if tmp_file is not None:
             tmp_file.close()
@@ -318,7 +312,7 @@ def _process_supply(
     virtual_bill_titles,
     writer,
     titles,
-    report_run,
+    report_run_id,
 ):
     gaps = {}
     data_sources = {}
@@ -761,8 +755,8 @@ def _process_supply(
         values["batch_id"] = bill.batch.id
         values["supply_id"] = supply.id
         values["site_id"] = None if site_code is None else site.id
-        report_run.insert_row(
-            sess, "", report_run_titles, values, {"is_checked": False}
+        ReportRun.w_insert_row(
+            report_run_id, "", report_run_titles, values, {"is_checked": False}
         )
 
         for bill in sess.query(Bill).filter(
@@ -783,7 +777,7 @@ def _process_supply(
                     )
 
         # Avoid long-running transactions
-        sess.commit()
+        sess.rollback()
 
     clumps = []
     for element, elgap in sorted(gaps.items()):
@@ -837,7 +831,7 @@ def _process_supply(
         vals["supply_id"] = supply.id
         vals["site_id"] = None if site_code is None else site.id
 
-        report_run.insert_row(sess, "", titles, vals, {"is_checked": False})
+        ReportRun.w_insert_row(report_run_id, "", titles, vals, {"is_checked": False})
 
     # Avoid long-running transactions
-    sess.commit()
+    sess.rollback()

{chellow-1716370405.0.0 → chellow-1716564585.0.0}/chellow/reports/report_387.py

@@ -10,7 +10,7 @@ from sqlalchemy.sql.expression import null, true
 
 from chellow.dloads import open_file
 from chellow.e.computer import SupplySource, forecast_date
-from chellow.models import Era, Session, Site, SiteEra, Supply
+from chellow.models import Era, RSession, Site, SiteEra, Supply
 from chellow.utils import (
     csv_make_val,
     hh_format,
@@ -25,7 +25,7 @@ def content(supply_id, start_date, finish_date, user):
     caches = {}
     f = None
     try:
-        with Session() as sess:
+        with RSession() as sess:
             supply = Supply.get_by_id(sess, supply_id)
 
             f_date = forecast_date()

{chellow-1716370405.0.0 → chellow-1716564585.0.0}/chellow/templates/e/supplier_bill_import.html

@@ -50,7 +50,19 @@
 					<th>VAT</th>
 					<th>Gross</th>
 					<th>Breakdown</th>
-					<th>Reads</th>
+					{% for i in range(failed_max_registers) %}
+						<th>R{{loop.index}} MPAN</th>
+						<th>R{{loop.index}} Meter Serial Number</th>
+						<th>R{{loop.index}} Coefficient</th>
+						<th>R{{loop.index}} Units</th>
+						<th>R{{loop.index}} TPR</th>
+						<th>R{{loop.index}} Previous Read Date</th>
+						<th>R{{loop.index}} Previous Read Value</th>
+						<th>R{{loop.index}} Previous Read Type</th>
+						<th>R{{loop.index}} Present Read Date</th>
+						<th>R{{loop.index}} Present Read Value</th>
+						<th>R{{loop.index}} Present Read Type</th>
+					{% endfor %}
 				</tr>
 			</thead>
 			<tbody>
@@ -87,7 +99,19 @@
 								<pre>{{bill.breakdown|dumps}}</pre>
 							{% endif %}
 						</td>
-						<td>{{bill.reads}}</td>
+						{% for read in bill.reads %}
+							<td>{{read.mpan}}</td>
+							<td>{{read.msn}}</td>
+							<td>{{read.coefficient}}</td>
+							<td>{{read.units}}</td>
+							<td>{{read.tpr_code}}</td>
+							<td>{{read.prev_date|hh_format}}</td>
+							<td>{{read.prev_value}}</td>
+							<td>{{read.prev_type_code}}</td>
+							<td>{{read.pres_date|hh_format}}</td>
+							<td>{{read.pres_value}}</td>
+							<td>{{read.pres_type_code}}</td>
+						{% endfor %}
 					</tr>
 				{% endfor %}
 			</tbody>

{chellow-1716370405.0.0 → chellow-1716564585.0.0}/pyproject.toml

@@ -14,11 +14,11 @@ classifiers = [
     "Operating System :: OS Independent",
 ]
 dependencies = [
-	"odio==0.0.22",
+	"odio==0.0.23",
 	"pypdf==3.17.0",
 	"pg8000==1.31.1",
 	"Flask==2.3.3",
-	"SQLAlchemy==2.0.28",
+	"SQLAlchemy==2.0.30",
 	"flask-restx==1.2.0",
 	"openpyxl==3.1.2",
 	"python-dateutil==2.8.2",

{chellow-1716370405.0.0 → chellow-1716564585.0.0}/test/e/test_bill_importer.py

@@ -17,8 +17,12 @@ def test_supplier_bill_import_html(mocker, app):
             g.config = {}
             batch = mocker.Mock()
             failed_bills = [{"error": "MPAN not found"}]
+            failed_max_registers = 0
             render_template(
-                "/e/supplier_bill_import.html", batch=batch, failed_bills=failed_bills
+                "/e/supplier_bill_import.html",
+                batch=batch,
+                failed_bills=failed_bills,
+                failed_max_registers=failed_max_registers,
             )